feat(image): pass reference images to GPT image model for edits
Server: forward canvas/reference images to the image model so extract, continue, and edit requests stay faithful to the source product. - Add ImageRequestOptions.Images and route requests with inputs to the /v1/images/edits endpoint, via multipart upload (default) or JSON image_url payload, selectable through Agent.Image.InputImageTransport. - Extract reference URLs from prompt directives, inline @image tokens, and chat message image/mask contents; fold them into the idempotency key and prepend a reference-fidelity instruction to the prompt. Frontend: - Shrink CanvasToast to a compact top pill. - Paste an internal transparent clipboard node instead of re-uploading when it matches the clipboard image. - Keep agent-project polling alive on stream error while streaming.
This commit is contained in:
@@ -3,8 +3,10 @@ package piagent
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -64,6 +66,19 @@ func TestNewImageClientDisablesKeepAliveReuse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewImageClientDefaultsInputImagesToFileTransport(t *testing.T) {
|
||||
client := NewImageClient(ImageClientOptions{
|
||||
BaseURL: "http://example.com",
|
||||
APIKey: "test-key",
|
||||
})
|
||||
if client == nil {
|
||||
t.Fatal("expected image client")
|
||||
}
|
||||
if client.inputImageTransport != "file" {
|
||||
t.Fatalf("expected file input image transport, got %q", client.inputImageTransport)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageClientSendsTransportDefaultSizeWhenUnset(t *testing.T) {
|
||||
var requestBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -125,6 +140,155 @@ func TestImageClientSendsExplicitSize(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageClientUploadsDataReferenceImagesAsMultipart(t *testing.T) {
|
||||
var requestPath string
|
||||
var requestContentType string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestPath = r.URL.Path
|
||||
requestContentType = r.Header.Get("Content-Type")
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
t.Errorf("parse multipart form: %v", err)
|
||||
return
|
||||
}
|
||||
file, header, err := r.FormFile("image")
|
||||
if err != nil {
|
||||
t.Errorf("expected image file field: %v", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
data, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
t.Errorf("read image file: %v", err)
|
||||
return
|
||||
}
|
||||
if header.Filename != "reference-1.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "reference bytes" {
|
||||
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/image.png"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key"})
|
||||
_, err := client.Generate(context.Background(), "extract product from reference", ImageRequestOptions{
|
||||
Model: "gpt-image-2",
|
||||
Images: []string{"data:image/png;base64,cmVmZXJlbmNlIGJ5dGVz"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if requestPath != "/v1/images/edits" {
|
||||
t.Fatalf("expected image edit endpoint for references, got %s", requestPath)
|
||||
}
|
||||
if !strings.HasPrefix(requestContentType, "multipart/form-data") {
|
||||
t.Fatalf("expected multipart edit request, got %s", requestContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageClientSendsReferenceImagesByURLWhenConfigured(t *testing.T) {
|
||||
var requestBody map[string]any
|
||||
var requestPath string
|
||||
var requestContentType string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestPath = r.URL.Path
|
||||
requestContentType = r.Header.Get("Content-Type")
|
||||
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/image.png"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key", InputImageTransport: "url"})
|
||||
_, err := client.Generate(context.Background(), "extract product from reference", ImageRequestOptions{
|
||||
Model: "gpt-image-2",
|
||||
Images: []string{" https://example.com/reference.png ", "", "https://example.com/reference.png", "https://example.com/second.webp"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if requestPath != "/v1/images/edits" {
|
||||
t.Fatalf("expected image edit endpoint for references, got %s", requestPath)
|
||||
}
|
||||
if requestContentType != "application/json" {
|
||||
t.Fatalf("expected url transport to use JSON, got %s", requestContentType)
|
||||
}
|
||||
images, ok := requestBody["images"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected images array in request, got %#v", requestBody["images"])
|
||||
}
|
||||
if len(images) != 2 {
|
||||
t.Fatalf("expected normalized reference images, got %#v", images)
|
||||
}
|
||||
first, ok := images[0].(map[string]any)
|
||||
if !ok || first["image_url"] != "https://example.com/reference.png" {
|
||||
t.Fatalf("expected first image_url reference, got %#v", images[0])
|
||||
}
|
||||
second, ok := images[1].(map[string]any)
|
||||
if !ok || second["image_url"] != "https://example.com/second.webp" {
|
||||
t.Fatalf("expected second image_url reference, got %#v", images[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageClientUploadsInternalReferenceImagesAsMultipart(t *testing.T) {
|
||||
var sawEdit bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/reference.png":
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write([]byte("internal image bytes"))
|
||||
case "/v1/images/edits":
|
||||
sawEdit = true
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
|
||||
t.Errorf("expected multipart edit request, got %s", r.Header.Get("Content-Type"))
|
||||
}
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
t.Errorf("parse multipart form: %v", err)
|
||||
return
|
||||
}
|
||||
if r.FormValue("model") != "gpt-image-2" || r.FormValue("prompt") != "edit the internal image" || r.FormValue("size") != "1024x1024" {
|
||||
t.Errorf("unexpected multipart fields: model=%q prompt=%q size=%q", r.FormValue("model"), r.FormValue("prompt"), r.FormValue("size"))
|
||||
}
|
||||
file, header, err := r.FormFile("image")
|
||||
if err != nil {
|
||||
t.Errorf("expected image file field: %v", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
data, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
t.Errorf("read image file: %v", err)
|
||||
return
|
||||
}
|
||||
if header.Filename != "reference.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "internal image bytes" {
|
||||
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/edited.png"}]}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key"})
|
||||
image, err := client.Generate(context.Background(), "edit the internal image", ImageRequestOptions{
|
||||
Model: "gpt-image-2",
|
||||
Size: "1024x1024",
|
||||
Images: []string{server.URL + "/reference.png"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if image.URL != "https://example.com/edited.png" {
|
||||
t.Fatalf("expected edited image, got %#v", image)
|
||||
}
|
||||
if !sawEdit {
|
||||
t.Fatal("expected image edit request")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageClientParsesAlternateImageResponseShapes(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
Reference in New Issue
Block a user