feat(image): pass reference images to GPT image model for edits

Server: forward canvas/reference images to the image model so extract,
continue, and edit requests stay faithful to the source product.
- Add ImageRequestOptions.Images and route requests with inputs to the
  /v1/images/edits endpoint, via multipart upload (default) or JSON
  image_url payload, selectable through Agent.Image.InputImageTransport.
- Extract reference URLs from prompt directives, inline @image tokens,
  and chat message image/mask contents; fold them into the idempotency
  key and prepend a reference-fidelity instruction to the prompt.

Frontend:
- Shrink CanvasToast to a compact top pill.
- Paste an internal transparent clipboard node instead of re-uploading
  when it matches the clipboard image.
- Keep agent-project polling alive on stream error while streaming.
This commit is contained in:
2026-07-08 01:19:20 +08:00
parent bb7fce9171
commit 2f5291a0f9
11 changed files with 614 additions and 63 deletions
@@ -1,27 +1,30 @@
.canvas-toast {
position: fixed;
left: 50%;
top: 48px;
top: 14px;
z-index: 120;
width: min(1040px, calc(100vw - 140px));
min-height: 126px;
box-sizing: border-box;
width: fit-content;
min-width: 220px;
max-width: min(480px, calc(100vw - 32px));
min-height: 54px;
display: flex;
align-items: center;
gap: 28px;
padding: 0 44px;
gap: 12px;
padding: 12px 18px;
border: 1px solid rgba(224, 229, 238, 0.96);
border-radius: 24px;
border-radius: 14px;
color: #1f2329;
background: rgba(255, 255, 255, 0.98);
box-shadow: 0 38px 90px rgba(17, 24, 39, 0.16);
box-shadow: 0 14px 36px rgba(17, 24, 39, 0.12);
pointer-events: none;
transform: translateX(-50%);
animation: canvas-toast-enter 180ms cubic-bezier(0.2, 0, 0.2, 1);
}
.canvas-toast-icon {
width: 44px;
height: 44px;
width: 30px;
height: 30px;
display: grid;
place-items: center;
flex: 0 0 auto;
@@ -29,6 +32,11 @@
color: #ffffff;
}
.canvas-toast-icon svg {
width: 18px;
height: 18px;
}
.canvas-toast.success .canvas-toast-icon {
background: #45d27c;
}
@@ -40,9 +48,9 @@
.canvas-toast strong {
min-width: 0;
overflow: hidden;
font-size: 32px;
font-weight: 850;
line-height: 1.18;
font-size: 15px;
font-weight: 750;
line-height: 1.32;
text-overflow: ellipsis;
white-space: normal;
word-break: break-word;
@@ -62,26 +70,28 @@
@media (max-width: 760px) {
.canvas-toast {
top: 18px;
width: calc(100vw - 32px);
min-height: 76px;
gap: 14px;
padding: 16px 18px;
border-radius: 18px;
top: 0;
width: fit-content;
min-width: 0;
max-width: calc(100vw - 28px);
min-height: 50px;
gap: 10px;
padding: 11px 14px;
border-radius: 13px;
}
.canvas-toast-icon {
width: 32px;
height: 32px;
width: 28px;
height: 28px;
}
.canvas-toast-icon svg {
width: 20px;
height: 20px;
width: 17px;
height: 17px;
}
.canvas-toast strong {
font-size: 18px;
font-size: 14px;
line-height: 1.28;
}
}
@@ -233,8 +233,9 @@ export function useGenerationStream({
closeRef.current = null;
},
onError: (nextError) => {
if (closeAgentProjectPollRef.current) {
if (streamingRef.current === streamKey) {
closeRef.current = null;
startAgentProjectPolling(nextProjectId, threadId);
return;
}
completeChannel(channel);
@@ -243,7 +244,6 @@ export function useGenerationStream({
closeRef.current = null;
}
});
startAgentProjectPolling(nextProjectId, threadId);
},
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
);
@@ -306,8 +306,9 @@ export function useGenerationStream({
closeRef.current = null;
},
onError: (nextError) => {
if (closeAgentProjectPollRef.current) {
if (streamingRef.current === streamKey) {
closeRef.current = null;
startAgentProjectPolling(nextProjectId, payload.threadId);
return;
}
completeChannel(channel);
@@ -316,7 +317,6 @@ export function useGenerationStream({
closeRef.current = null;
}
});
startAgentProjectPolling(nextProjectId, payload.threadId);
},
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
);
@@ -1843,9 +1843,9 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
.catch(() => showCanvasToast(t("clipboardCopyFailed"), "error"));
};
const pasteNode = () => {
if (!clipboardNode) return;
const copy = cloneCanvasNode(clipboardNode, cryptoFallbackId());
const pasteNode = (sourceNode = clipboardNode) => {
if (!sourceNode) return;
const copy = cloneCanvasNode(sourceNode, cryptoFallbackId());
setNodes((current) => [...current, copy]);
selectOnlyNode(copy.id);
setDirty(true);
@@ -2145,11 +2145,25 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
if (isEditableShortcutTarget(event.target)) return;
const imageFiles = event.clipboardData ? dataTransferImageFiles(event.clipboardData, "clipboard-image") : [];
if (imageFiles.length > 0) {
event.preventDefault();
const pasteImageFiles = () => {
const pointer = lastCanvasPointerRef.current;
const anchor = pointer ? canvasPointFromClient(pointer.clientX, pointer.clientY) : canvasCenterPoint();
void uploadImagesToCanvas(imageFiles, anchor);
};
if (clipboardNode && imageFiles.length > 0 && isTransparentClipboardNode(clipboardNode)) {
event.preventDefault();
void shouldUseInternalClipboardNode(clipboardNode, imageFiles).then((useInternalNode) => {
if (useInternalNode) {
pasteNode(clipboardNode);
return;
}
pasteImageFiles();
});
return;
}
if (imageFiles.length > 0) {
event.preventDefault();
pasteImageFiles();
return;
}
+3
View File
@@ -58,6 +58,7 @@ Agent:
Image:
BaseURL: http://154.21.87.3:8080
APIKeyEnv: GPT_IMAGE_API_KEY
InputImageTransport: file
TimeoutSeconds: 600
ObjectStorage:
Driver: minio
@@ -71,6 +72,8 @@ ObjectStorage:
ExpiresIn: 900
```
`Agent.Image.InputImageTransport` controls how input/reference images are sent to the image model for edit/image-to-image calls: `file` uploads image bytes with multipart form data, while `url` sends `images: [{"image_url": "..."}]` for gateways that can fetch public URLs directly.
Start local Docker dependencies for debugging:
```bash
+1
View File
@@ -104,6 +104,7 @@ Agent:
BaseURL: http://154.21.87.3:8080
APIKeyEnv: GPT_IMAGE_API_KEY
APIKey: "sk-0af1835a4e855198ec6e1ff8fb120844c146dd09b0b25b05b30ed8fa5c7be568"
InputImageTransport: file
TimeoutSeconds: 600
ObjectStorage:
Driver: minio
+5 -4
View File
@@ -123,10 +123,11 @@ type AgentConfig struct {
}
type ImageGenerationConfig struct {
BaseURL string `json:",optional"`
APIKey string `json:",optional"`
APIKeyEnv string `json:",default=GPT_IMAGE_API_KEY"`
TimeoutSeconds int `json:",default=600"`
BaseURL string `json:",optional"`
APIKey string `json:",optional"`
APIKeyEnv string `json:",default=GPT_IMAGE_API_KEY"`
InputImageTransport string `json:",default=file"`
TimeoutSeconds int `json:",default=600"`
}
type TextExtractionConfig struct {
@@ -283,6 +283,9 @@ func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prom
}
func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) {
referenceImages := referenceImageURLs(req)
prompt = withReferenceImageInstruction(prompt, len(referenceImages))
fallback = withReferenceImageInstruction(fallback, len(referenceImages))
prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes)
fallback = prepareGPTImagePrompt(fallback, imageGenerationFallbackMaxRunes)
if imagePromptTooLarge(prompt) && strings.TrimSpace(fallback) != "" {
@@ -293,6 +296,7 @@ func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.
Model: req.ImageModel,
Size: req.ImageSize,
Count: 1,
Images: referenceImages,
IdempotencyKey: imageGenerationIdempotencyKey(req, index, prompt, false),
})
if err == nil {
@@ -308,6 +312,7 @@ func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.
Model: req.ImageModel,
Size: req.ImageSize,
Count: 1,
Images: referenceImages,
IdempotencyKey: imageGenerationIdempotencyKey(req, index, fallback, true),
})
if fallbackErr == nil {
@@ -329,12 +334,79 @@ func imageGenerationIdempotencyKey(req design.AgentRequest, index int, prompt st
strings.TrimSpace(req.ImageSize),
strconv.Itoa(index),
kind,
strings.Join(referenceImageURLs(req), "\n"),
strings.TrimSpace(prompt),
}, "\n")
sum := sha256.Sum256([]byte(seed))
return "img-gen-" + kind + "-" + hex.EncodeToString(sum[:16])
}
func referenceImageURLs(req design.AgentRequest) []string {
urls := make([]string, 0)
seen := make(map[string]struct{})
add := func(value string) {
value = strings.TrimSpace(strings.TrimRight(value, ",。,."))
if value == "" {
return
}
if _, ok := seen[value]; ok {
return
}
seen[value] = struct{}{}
urls = append(urls, value)
}
addFromText := func(text string) {
for _, match := range referenceImageDirectiveURLPattern.FindAllStringSubmatch(text, -1) {
if len(match) > 1 {
add(match[1])
}
}
for _, match := range promptImageTokenURLPattern.FindAllStringSubmatch(text, -1) {
if len(match) > 1 && strings.TrimSpace(match[1]) != "" {
add(match[1])
continue
}
if len(match) > 2 {
add(match[2])
}
}
}
addFromText(req.Prompt)
for _, message := range req.Messages {
for _, content := range message.Contents {
switch strings.ToLower(strings.TrimSpace(content.Type)) {
case "image", "mask":
add(content.ImageURL)
case "text":
addFromText(content.Text)
}
if len(urls) >= 16 {
return urls[:16]
}
}
}
if len(urls) > 16 {
return urls[:16]
}
return urls
}
func withReferenceImageInstruction(prompt string, imageCount int) string {
prompt = strings.TrimSpace(prompt)
if prompt == "" || imageCount <= 0 {
return prompt
}
lower := strings.ToLower(prompt)
if strings.Contains(prompt, "附带参考图") || strings.Contains(lower, "attached reference image") || strings.Contains(lower, "input reference image") {
return prompt
}
if prefersEnglish(prompt) {
return "Input reference image(s) are attached. Use them as the visual source of truth for product shape, fabric, color, structure, and identity; when extracting, continuing, or editing a product, do not invent a different item.\n" + prompt
}
return "已随请求附带参考图;生成时必须以参考图中的商品外形、材质、颜色、结构细节和身份为准,拆出、延续或编辑商品时不要另造不同商品。\n" + prompt
}
func imagePromptTooLarge(prompt string) bool {
return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes
}
@@ -404,6 +476,8 @@ var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×|
var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`)
var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`)
var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[(][^)\n]+[)])?\s*[:]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`)
var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`)
func fullFrameImagePrompt(prompt string) string {
prompt = strings.TrimSpace(prompt)
@@ -214,6 +214,52 @@ func TestDesignAgentGeneratesImageThroughPiAgent(t *testing.T) {
}
}
func TestDesignAgentPassesInlineReferenceImagesToImageGenerator(t *testing.T) {
imageGenerator := &fakeImageGenerator{}
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
_, err := agent.Plan(context.Background(), design.AgentRequest{
Prompt: "把参考图里的灰裤子拆成单件商品图",
Mode: "image",
ImageModel: "gpt-image-2",
Messages: []design.AgentChatMessage{
{
Role: "user",
Contents: []design.AgentContent{
{Type: "image", ImageURL: "https://example.com/pants-rack.png", ImageWidth: 1200, ImageHeight: 900},
{Type: "text", Text: "把参考图里的灰裤子拆成单件商品图", TextSource: "input"},
},
},
},
})
if err != nil {
t.Fatal(err)
}
if len(imageGenerator.opts.Images) != 1 || imageGenerator.opts.Images[0] != "https://example.com/pants-rack.png" {
t.Fatalf("expected reference image to be passed to GPT Image 2, got %#v", imageGenerator.opts.Images)
}
if !strings.Contains(imageGenerator.prompt, "参考图") {
t.Fatalf("expected prompt to tell image model to use attached reference, got %s", imageGenerator.prompt)
}
}
func TestDesignAgentPassesPromptDirectiveReferenceImagesToImageGenerator(t *testing.T) {
imageGenerator := &fakeImageGenerator{}
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
_, err := agent.Plan(context.Background(), design.AgentRequest{
Prompt: "参考图 1image):https://example.com/pants-rack.png\n灰裤子拆成单件衣服,为了后面电商配图。",
Mode: "image",
ImageModel: "gpt-image-2",
})
if err != nil {
t.Fatal(err)
}
if len(imageGenerator.opts.Images) != 1 || imageGenerator.opts.Images[0] != "https://example.com/pants-rack.png" {
t.Fatalf("expected directive reference image to be passed to GPT Image 2, got %#v", imageGenerator.opts.Images)
}
}
func TestDesignAgentUsesDefaultImageSizeWithoutExplicitRequest(t *testing.T) {
imageGenerator := &fakeImageGenerator{}
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
@@ -3,11 +3,18 @@ package piagent
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"net/textproto"
"net/url"
"path/filepath"
"strconv"
"strings"
"time"
"unicode/utf8"
@@ -35,19 +42,22 @@ type ImageRequestOptions struct {
Model string
Size string
Count int
Images []string
IdempotencyKey string
}
type ImageClientOptions struct {
BaseURL string
APIKey string
TimeoutSeconds int
BaseURL string
APIKey string
InputImageTransport string
TimeoutSeconds int
}
type ImageClient struct {
baseURL string
apiKey string
client *http.Client
baseURL string
apiKey string
inputImageTransport string
client *http.Client
}
func NewImageClient(opts ImageClientOptions) *ImageClient {
@@ -63,8 +73,9 @@ func NewImageClient(opts ImageClientOptions) *ImageClient {
}
return &ImageClient{
baseURL: strings.TrimRight(opts.BaseURL, "/"),
apiKey: opts.APIKey,
baseURL: strings.TrimRight(opts.BaseURL, "/"),
apiKey: opts.APIKey,
inputImageTransport: normalizeInputImageTransport(opts.InputImageTransport),
client: &http.Client{
Timeout: timeout,
Transport: &http.Transport{
@@ -79,6 +90,15 @@ func NewImageClient(opts ImageClientOptions) *ImageClient {
}
}
func normalizeInputImageTransport(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "url", "image_url", "image-url":
return "url"
default:
return "file"
}
}
func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageRequestOptions) (GeneratedImage, error) {
prompt = strings.TrimSpace(prompt)
if prompt == "" {
@@ -99,20 +119,10 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
if count > 10 {
count = 10
}
images := normalizeImageInputs(opts.Images)
requestPayload := map[string]any{
"model": model,
"prompt": prompt,
}
if count > 1 {
requestPayload["n"] = count
}
if size != "" {
requestPayload["size"] = size
}
idempotencyKey := strings.TrimSpace(opts.IdempotencyKey)
body, err := json.Marshal(requestPayload)
body, contentType, err := c.imageRequestBody(ctx, model, prompt, size, count, images)
if err != nil {
return GeneratedImage{}, err
}
@@ -121,7 +131,7 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
startedAt := time.Now()
promptRunes := utf8.RuneCountInString(prompt)
for attempt := 0; attempt < 3; attempt++ {
image, err := c.generateOnce(ctx, body, idempotencyKey)
image, err := c.generateOnce(ctx, body, contentType, idempotencyKey, len(images) > 0)
if err == nil {
logx.Infof(
"gpt image generation succeeded model=%s size=%s count=%d prompt_runes=%d attempt=%d elapsed=%s timeout=%s images=%d idempotency_key=%s",
@@ -166,13 +176,240 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
return GeneratedImage{}, lastErr
}
func (c *ImageClient) generateOnce(ctx context.Context, body []byte, idempotencyKey string) (GeneratedImage, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/images/generations", bytes.NewReader(body))
func (c *ImageClient) imageRequestBody(ctx context.Context, model string, prompt string, size string, count int, images []string) ([]byte, string, error) {
if len(images) > 0 {
if c.inputImageTransport == "url" {
return jsonImageEditBody(model, prompt, size, count, images)
}
return c.multipartImageEditBody(ctx, model, prompt, size, count, images)
}
requestPayload := map[string]any{
"model": model,
"prompt": prompt,
}
if count > 1 {
requestPayload["n"] = count
}
if size != "" {
requestPayload["size"] = size
}
body, err := json.Marshal(requestPayload)
return body, "application/json", err
}
func jsonImageEditBody(model string, prompt string, size string, count int, images []string) ([]byte, string, error) {
requestPayload := map[string]any{
"model": model,
"prompt": prompt,
"images": imageReferencePayload(images),
}
if count > 1 {
requestPayload["n"] = count
}
if size != "" {
requestPayload["size"] = size
}
body, err := json.Marshal(requestPayload)
return body, "application/json", err
}
func (c *ImageClient) multipartImageEditBody(ctx context.Context, model string, prompt string, size string, count int, images []string) ([]byte, string, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
fields := map[string]string{
"model": model,
"prompt": prompt,
}
if size != "" {
fields["size"] = size
}
if count > 1 {
fields["n"] = strconv.Itoa(count)
}
for key, value := range fields {
if err := writer.WriteField(key, value); err != nil {
return nil, "", err
}
}
fieldName := "image"
if len(images) > 1 {
fieldName = "image[]"
}
for index, image := range images {
input, err := c.imageInputFile(ctx, image, index)
if err != nil {
_ = writer.Close()
return nil, "", err
}
part, err := createImageFormFile(writer, fieldName, input)
if err != nil {
_ = writer.Close()
return nil, "", err
}
if _, err := part.Write(input.Data); err != nil {
_ = writer.Close()
return nil, "", err
}
}
if err := writer.Close(); err != nil {
return nil, "", err
}
return body.Bytes(), writer.FormDataContentType(), nil
}
func createImageFormFile(writer *multipart.Writer, fieldName string, input imageInputFile) (io.Writer, error) {
header := make(textproto.MIMEHeader)
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeMultipartValue(fieldName), escapeMultipartValue(input.FileName)))
contentType := strings.TrimSpace(input.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
}
header.Set("Content-Type", contentType)
return writer.CreatePart(header)
}
func escapeMultipartValue(value string) string {
return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(value)
}
type imageInputFile struct {
FileName string
ContentType string
Data []byte
}
func (c *ImageClient) imageInputFile(ctx context.Context, value string, index int) (imageInputFile, error) {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "data:image/") {
header, payload, ok := strings.Cut(value, ",")
if !ok {
return imageInputFile{}, fmt.Errorf("invalid image data url")
}
contentType := strings.TrimPrefix(strings.Split(header, ";")[0], "data:")
data, err := base64.StdEncoding.DecodeString(payload)
if err != nil {
return imageInputFile{}, err
}
return imageInputFile{
FileName: fmt.Sprintf("reference-%d%s", index+1, imageExtension(contentType, "")),
ContentType: contentType,
Data: data,
}, nil
}
if !strings.HasPrefix(value, "http://") && !strings.HasPrefix(value, "https://") {
return imageInputFile{}, fmt.Errorf("unsupported image input %q", value)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, value, nil)
if err != nil {
return imageInputFile{}, err
}
req.Header.Set("User-Agent", "img-infinite-canvas/1.0")
resp, err := c.client.Do(req)
if err != nil {
return imageInputFile{}, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return imageInputFile{}, fmt.Errorf("download reference image failed: %s", resp.Status)
}
contentType := strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0])
data, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
if err != nil {
return imageInputFile{}, err
}
if len(data) == 0 {
return imageInputFile{}, fmt.Errorf("download reference image returned empty body")
}
if contentType == "" || !strings.HasPrefix(contentType, "image/") {
contentType = http.DetectContentType(data)
}
if !strings.HasPrefix(contentType, "image/") {
return imageInputFile{}, fmt.Errorf("reference input is not an image: %s", contentType)
}
return imageInputFile{
FileName: imageFileName(value, contentType, index),
ContentType: contentType,
Data: data,
}, nil
}
func normalizeImageInputs(images []string) []string {
if len(images) == 0 {
return nil
}
normalized := make([]string, 0, len(images))
seen := make(map[string]struct{})
for _, image := range images {
image = strings.TrimSpace(image)
if image == "" {
continue
}
if _, ok := seen[image]; ok {
continue
}
seen[image] = struct{}{}
normalized = append(normalized, image)
if len(normalized) >= 16 {
break
}
}
return normalized
}
func imageReferencePayload(images []string) []map[string]string {
items := make([]map[string]string, 0, len(images))
for _, image := range images {
items = append(items, map[string]string{"image_url": image})
}
return items
}
func imageFileName(value string, contentType string, index int) string {
name := ""
if parsed, err := url.Parse(value); err == nil {
name = filepath.Base(parsed.Path)
}
if name == "" || name == "." || name == "/" {
name = fmt.Sprintf("reference-%d%s", index+1, imageExtension(contentType, ""))
}
if filepath.Ext(name) == "" {
name += imageExtension(contentType, ".png")
}
return name
}
func imageExtension(contentType string, fallback string) string {
switch strings.ToLower(strings.TrimSpace(contentType)) {
case "image/png":
return ".png"
case "image/jpeg", "image/jpg":
return ".jpg"
case "image/webp":
return ".webp"
case "image/gif":
return ".gif"
default:
if fallback != "" {
return fallback
}
return ".png"
}
}
func (c *ImageClient) generateOnce(ctx context.Context, body []byte, contentType string, idempotencyKey string, edit bool) (GeneratedImage, error) {
endpoint := "/v1/images/generations"
if edit {
endpoint = "/v1/images/edits"
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+endpoint, bytes.NewReader(body))
if err != nil {
return GeneratedImage{}, err
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Type", contentType)
req.Header.Set("Connection", "close")
if idempotencyKey != "" {
req.Header.Set("Idempotency-Key", idempotencyKey)
@@ -3,8 +3,10 @@ package piagent
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
@@ -64,6 +66,19 @@ func TestNewImageClientDisablesKeepAliveReuse(t *testing.T) {
}
}
func TestNewImageClientDefaultsInputImagesToFileTransport(t *testing.T) {
client := NewImageClient(ImageClientOptions{
BaseURL: "http://example.com",
APIKey: "test-key",
})
if client == nil {
t.Fatal("expected image client")
}
if client.inputImageTransport != "file" {
t.Fatalf("expected file input image transport, got %q", client.inputImageTransport)
}
}
func TestImageClientSendsTransportDefaultSizeWhenUnset(t *testing.T) {
var requestBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -125,6 +140,155 @@ func TestImageClientSendsExplicitSize(t *testing.T) {
}
}
func TestImageClientUploadsDataReferenceImagesAsMultipart(t *testing.T) {
var requestPath string
var requestContentType string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestPath = r.URL.Path
requestContentType = r.Header.Get("Content-Type")
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse multipart form: %v", err)
return
}
file, header, err := r.FormFile("image")
if err != nil {
t.Errorf("expected image file field: %v", err)
return
}
defer file.Close()
data, err := io.ReadAll(file)
if err != nil {
t.Errorf("read image file: %v", err)
return
}
if header.Filename != "reference-1.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "reference bytes" {
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/image.png"}]}`))
}))
defer server.Close()
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key"})
_, err := client.Generate(context.Background(), "extract product from reference", ImageRequestOptions{
Model: "gpt-image-2",
Images: []string{"data:image/png;base64,cmVmZXJlbmNlIGJ5dGVz"},
})
if err != nil {
t.Fatal(err)
}
if requestPath != "/v1/images/edits" {
t.Fatalf("expected image edit endpoint for references, got %s", requestPath)
}
if !strings.HasPrefix(requestContentType, "multipart/form-data") {
t.Fatalf("expected multipart edit request, got %s", requestContentType)
}
}
func TestImageClientSendsReferenceImagesByURLWhenConfigured(t *testing.T) {
var requestBody map[string]any
var requestPath string
var requestContentType string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestPath = r.URL.Path
requestContentType = r.Header.Get("Content-Type")
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
t.Fatal(err)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/image.png"}]}`))
}))
defer server.Close()
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key", InputImageTransport: "url"})
_, err := client.Generate(context.Background(), "extract product from reference", ImageRequestOptions{
Model: "gpt-image-2",
Images: []string{" https://example.com/reference.png ", "", "https://example.com/reference.png", "https://example.com/second.webp"},
})
if err != nil {
t.Fatal(err)
}
if requestPath != "/v1/images/edits" {
t.Fatalf("expected image edit endpoint for references, got %s", requestPath)
}
if requestContentType != "application/json" {
t.Fatalf("expected url transport to use JSON, got %s", requestContentType)
}
images, ok := requestBody["images"].([]any)
if !ok {
t.Fatalf("expected images array in request, got %#v", requestBody["images"])
}
if len(images) != 2 {
t.Fatalf("expected normalized reference images, got %#v", images)
}
first, ok := images[0].(map[string]any)
if !ok || first["image_url"] != "https://example.com/reference.png" {
t.Fatalf("expected first image_url reference, got %#v", images[0])
}
second, ok := images[1].(map[string]any)
if !ok || second["image_url"] != "https://example.com/second.webp" {
t.Fatalf("expected second image_url reference, got %#v", images[1])
}
}
func TestImageClientUploadsInternalReferenceImagesAsMultipart(t *testing.T) {
var sawEdit bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/reference.png":
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write([]byte("internal image bytes"))
case "/v1/images/edits":
sawEdit = true
if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
t.Errorf("expected multipart edit request, got %s", r.Header.Get("Content-Type"))
}
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse multipart form: %v", err)
return
}
if r.FormValue("model") != "gpt-image-2" || r.FormValue("prompt") != "edit the internal image" || r.FormValue("size") != "1024x1024" {
t.Errorf("unexpected multipart fields: model=%q prompt=%q size=%q", r.FormValue("model"), r.FormValue("prompt"), r.FormValue("size"))
}
file, header, err := r.FormFile("image")
if err != nil {
t.Errorf("expected image file field: %v", err)
return
}
defer file.Close()
data, err := io.ReadAll(file)
if err != nil {
t.Errorf("read image file: %v", err)
return
}
if header.Filename != "reference.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "internal image bytes" {
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/edited.png"}]}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key"})
image, err := client.Generate(context.Background(), "edit the internal image", ImageRequestOptions{
Model: "gpt-image-2",
Size: "1024x1024",
Images: []string{server.URL + "/reference.png"},
})
if err != nil {
t.Fatal(err)
}
if image.URL != "https://example.com/edited.png" {
t.Fatalf("expected edited image, got %#v", image)
}
if !sawEdit {
t.Fatal("expected image edit request")
}
}
func TestImageClientParsesAlternateImageResponseShapes(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
+4 -3
View File
@@ -304,9 +304,10 @@ func newImageGenerator(c config.Config) piagent.ImageGenerator {
opts := c.Agent.Image
apiKey := resolveImageAPIKey(opts)
client := piagent.NewImageClient(piagent.ImageClientOptions{
BaseURL: opts.BaseURL,
APIKey: apiKey,
TimeoutSeconds: opts.TimeoutSeconds,
BaseURL: opts.BaseURL,
APIKey: apiKey,
InputImageTransport: opts.InputImageTransport,
TimeoutSeconds: opts.TimeoutSeconds,
})
if client == nil {
return nil