feat(image): pass reference images to GPT image model for edits
Server: forward canvas/reference images to the image model so extract, continue, and edit requests stay faithful to the source product. - Add ImageRequestOptions.Images and route requests with inputs to the /v1/images/edits endpoint, via multipart upload (default) or JSON image_url payload, selectable through Agent.Image.InputImageTransport. - Extract reference URLs from prompt directives, inline @image tokens, and chat message image/mask contents; fold them into the idempotency key and prepend a reference-fidelity instruction to the prompt. Frontend: - Shrink CanvasToast to a compact top pill. - Paste an internal transparent clipboard node instead of re-uploading when it matches the clipboard image. - Keep agent-project polling alive on stream error while streaming.
This commit is contained in:
@@ -1,27 +1,30 @@
|
|||||||
.canvas-toast {
|
.canvas-toast {
|
||||||
position: fixed;
|
position: fixed;
|
||||||
left: 50%;
|
left: 50%;
|
||||||
top: 48px;
|
top: 14px;
|
||||||
z-index: 120;
|
z-index: 120;
|
||||||
width: min(1040px, calc(100vw - 140px));
|
box-sizing: border-box;
|
||||||
min-height: 126px;
|
width: fit-content;
|
||||||
|
min-width: 220px;
|
||||||
|
max-width: min(480px, calc(100vw - 32px));
|
||||||
|
min-height: 54px;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 28px;
|
gap: 12px;
|
||||||
padding: 0 44px;
|
padding: 12px 18px;
|
||||||
border: 1px solid rgba(224, 229, 238, 0.96);
|
border: 1px solid rgba(224, 229, 238, 0.96);
|
||||||
border-radius: 24px;
|
border-radius: 14px;
|
||||||
color: #1f2329;
|
color: #1f2329;
|
||||||
background: rgba(255, 255, 255, 0.98);
|
background: rgba(255, 255, 255, 0.98);
|
||||||
box-shadow: 0 38px 90px rgba(17, 24, 39, 0.16);
|
box-shadow: 0 14px 36px rgba(17, 24, 39, 0.12);
|
||||||
pointer-events: none;
|
pointer-events: none;
|
||||||
transform: translateX(-50%);
|
transform: translateX(-50%);
|
||||||
animation: canvas-toast-enter 180ms cubic-bezier(0.2, 0, 0.2, 1);
|
animation: canvas-toast-enter 180ms cubic-bezier(0.2, 0, 0.2, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
.canvas-toast-icon {
|
.canvas-toast-icon {
|
||||||
width: 44px;
|
width: 30px;
|
||||||
height: 44px;
|
height: 30px;
|
||||||
display: grid;
|
display: grid;
|
||||||
place-items: center;
|
place-items: center;
|
||||||
flex: 0 0 auto;
|
flex: 0 0 auto;
|
||||||
@@ -29,6 +32,11 @@
|
|||||||
color: #ffffff;
|
color: #ffffff;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.canvas-toast-icon svg {
|
||||||
|
width: 18px;
|
||||||
|
height: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
.canvas-toast.success .canvas-toast-icon {
|
.canvas-toast.success .canvas-toast-icon {
|
||||||
background: #45d27c;
|
background: #45d27c;
|
||||||
}
|
}
|
||||||
@@ -40,9 +48,9 @@
|
|||||||
.canvas-toast strong {
|
.canvas-toast strong {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
font-size: 32px;
|
font-size: 15px;
|
||||||
font-weight: 850;
|
font-weight: 750;
|
||||||
line-height: 1.18;
|
line-height: 1.32;
|
||||||
text-overflow: ellipsis;
|
text-overflow: ellipsis;
|
||||||
white-space: normal;
|
white-space: normal;
|
||||||
word-break: break-word;
|
word-break: break-word;
|
||||||
@@ -62,26 +70,28 @@
|
|||||||
|
|
||||||
@media (max-width: 760px) {
|
@media (max-width: 760px) {
|
||||||
.canvas-toast {
|
.canvas-toast {
|
||||||
top: 18px;
|
top: 0;
|
||||||
width: calc(100vw - 32px);
|
width: fit-content;
|
||||||
min-height: 76px;
|
min-width: 0;
|
||||||
gap: 14px;
|
max-width: calc(100vw - 28px);
|
||||||
padding: 16px 18px;
|
min-height: 50px;
|
||||||
border-radius: 18px;
|
gap: 10px;
|
||||||
|
padding: 11px 14px;
|
||||||
|
border-radius: 13px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.canvas-toast-icon {
|
.canvas-toast-icon {
|
||||||
width: 32px;
|
width: 28px;
|
||||||
height: 32px;
|
height: 28px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.canvas-toast-icon svg {
|
.canvas-toast-icon svg {
|
||||||
width: 20px;
|
width: 17px;
|
||||||
height: 20px;
|
height: 17px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.canvas-toast strong {
|
.canvas-toast strong {
|
||||||
font-size: 18px;
|
font-size: 14px;
|
||||||
line-height: 1.28;
|
line-height: 1.28;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -233,8 +233,9 @@ export function useGenerationStream({
|
|||||||
closeRef.current = null;
|
closeRef.current = null;
|
||||||
},
|
},
|
||||||
onError: (nextError) => {
|
onError: (nextError) => {
|
||||||
if (closeAgentProjectPollRef.current) {
|
if (streamingRef.current === streamKey) {
|
||||||
closeRef.current = null;
|
closeRef.current = null;
|
||||||
|
startAgentProjectPolling(nextProjectId, threadId);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
completeChannel(channel);
|
completeChannel(channel);
|
||||||
@@ -243,7 +244,6 @@ export function useGenerationStream({
|
|||||||
closeRef.current = null;
|
closeRef.current = null;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
startAgentProjectPolling(nextProjectId, threadId);
|
|
||||||
},
|
},
|
||||||
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
|
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
|
||||||
);
|
);
|
||||||
@@ -306,8 +306,9 @@ export function useGenerationStream({
|
|||||||
closeRef.current = null;
|
closeRef.current = null;
|
||||||
},
|
},
|
||||||
onError: (nextError) => {
|
onError: (nextError) => {
|
||||||
if (closeAgentProjectPollRef.current) {
|
if (streamingRef.current === streamKey) {
|
||||||
closeRef.current = null;
|
closeRef.current = null;
|
||||||
|
startAgentProjectPolling(nextProjectId, payload.threadId);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
completeChannel(channel);
|
completeChannel(channel);
|
||||||
@@ -316,7 +317,6 @@ export function useGenerationStream({
|
|||||||
closeRef.current = null;
|
closeRef.current = null;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
startAgentProjectPolling(nextProjectId, payload.threadId);
|
|
||||||
},
|
},
|
||||||
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
|
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1843,9 +1843,9 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
|
|||||||
.catch(() => showCanvasToast(t("clipboardCopyFailed"), "error"));
|
.catch(() => showCanvasToast(t("clipboardCopyFailed"), "error"));
|
||||||
};
|
};
|
||||||
|
|
||||||
const pasteNode = () => {
|
const pasteNode = (sourceNode = clipboardNode) => {
|
||||||
if (!clipboardNode) return;
|
if (!sourceNode) return;
|
||||||
const copy = cloneCanvasNode(clipboardNode, cryptoFallbackId());
|
const copy = cloneCanvasNode(sourceNode, cryptoFallbackId());
|
||||||
setNodes((current) => [...current, copy]);
|
setNodes((current) => [...current, copy]);
|
||||||
selectOnlyNode(copy.id);
|
selectOnlyNode(copy.id);
|
||||||
setDirty(true);
|
setDirty(true);
|
||||||
@@ -2145,11 +2145,25 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
|
|||||||
if (isEditableShortcutTarget(event.target)) return;
|
if (isEditableShortcutTarget(event.target)) return;
|
||||||
|
|
||||||
const imageFiles = event.clipboardData ? dataTransferImageFiles(event.clipboardData, "clipboard-image") : [];
|
const imageFiles = event.clipboardData ? dataTransferImageFiles(event.clipboardData, "clipboard-image") : [];
|
||||||
if (imageFiles.length > 0) {
|
const pasteImageFiles = () => {
|
||||||
event.preventDefault();
|
|
||||||
const pointer = lastCanvasPointerRef.current;
|
const pointer = lastCanvasPointerRef.current;
|
||||||
const anchor = pointer ? canvasPointFromClient(pointer.clientX, pointer.clientY) : canvasCenterPoint();
|
const anchor = pointer ? canvasPointFromClient(pointer.clientX, pointer.clientY) : canvasCenterPoint();
|
||||||
void uploadImagesToCanvas(imageFiles, anchor);
|
void uploadImagesToCanvas(imageFiles, anchor);
|
||||||
|
};
|
||||||
|
if (clipboardNode && imageFiles.length > 0 && isTransparentClipboardNode(clipboardNode)) {
|
||||||
|
event.preventDefault();
|
||||||
|
void shouldUseInternalClipboardNode(clipboardNode, imageFiles).then((useInternalNode) => {
|
||||||
|
if (useInternalNode) {
|
||||||
|
pasteNode(clipboardNode);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pasteImageFiles();
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (imageFiles.length > 0) {
|
||||||
|
event.preventDefault();
|
||||||
|
pasteImageFiles();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ Agent:
|
|||||||
Image:
|
Image:
|
||||||
BaseURL: http://154.21.87.3:8080
|
BaseURL: http://154.21.87.3:8080
|
||||||
APIKeyEnv: GPT_IMAGE_API_KEY
|
APIKeyEnv: GPT_IMAGE_API_KEY
|
||||||
|
InputImageTransport: file
|
||||||
TimeoutSeconds: 600
|
TimeoutSeconds: 600
|
||||||
ObjectStorage:
|
ObjectStorage:
|
||||||
Driver: minio
|
Driver: minio
|
||||||
@@ -71,6 +72,8 @@ ObjectStorage:
|
|||||||
ExpiresIn: 900
|
ExpiresIn: 900
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`Agent.Image.InputImageTransport` controls how input/reference images are sent to the image model for edit/image-to-image calls: `file` uploads image bytes with multipart form data, while `url` sends `images: [{"image_url": "..."}]` for gateways that can fetch public URLs directly.
|
||||||
|
|
||||||
Start local Docker dependencies for debugging:
|
Start local Docker dependencies for debugging:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ Agent:
|
|||||||
BaseURL: http://154.21.87.3:8080
|
BaseURL: http://154.21.87.3:8080
|
||||||
APIKeyEnv: GPT_IMAGE_API_KEY
|
APIKeyEnv: GPT_IMAGE_API_KEY
|
||||||
APIKey: "sk-0af1835a4e855198ec6e1ff8fb120844c146dd09b0b25b05b30ed8fa5c7be568"
|
APIKey: "sk-0af1835a4e855198ec6e1ff8fb120844c146dd09b0b25b05b30ed8fa5c7be568"
|
||||||
|
InputImageTransport: file
|
||||||
TimeoutSeconds: 600
|
TimeoutSeconds: 600
|
||||||
ObjectStorage:
|
ObjectStorage:
|
||||||
Driver: minio
|
Driver: minio
|
||||||
|
|||||||
@@ -123,10 +123,11 @@ type AgentConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ImageGenerationConfig struct {
|
type ImageGenerationConfig struct {
|
||||||
BaseURL string `json:",optional"`
|
BaseURL string `json:",optional"`
|
||||||
APIKey string `json:",optional"`
|
APIKey string `json:",optional"`
|
||||||
APIKeyEnv string `json:",default=GPT_IMAGE_API_KEY"`
|
APIKeyEnv string `json:",default=GPT_IMAGE_API_KEY"`
|
||||||
TimeoutSeconds int `json:",default=600"`
|
InputImageTransport string `json:",default=file"`
|
||||||
|
TimeoutSeconds int `json:",default=600"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TextExtractionConfig struct {
|
type TextExtractionConfig struct {
|
||||||
|
|||||||
@@ -283,6 +283,9 @@ func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prom
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) {
|
func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) {
|
||||||
|
referenceImages := referenceImageURLs(req)
|
||||||
|
prompt = withReferenceImageInstruction(prompt, len(referenceImages))
|
||||||
|
fallback = withReferenceImageInstruction(fallback, len(referenceImages))
|
||||||
prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes)
|
prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes)
|
||||||
fallback = prepareGPTImagePrompt(fallback, imageGenerationFallbackMaxRunes)
|
fallback = prepareGPTImagePrompt(fallback, imageGenerationFallbackMaxRunes)
|
||||||
if imagePromptTooLarge(prompt) && strings.TrimSpace(fallback) != "" {
|
if imagePromptTooLarge(prompt) && strings.TrimSpace(fallback) != "" {
|
||||||
@@ -293,6 +296,7 @@ func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.
|
|||||||
Model: req.ImageModel,
|
Model: req.ImageModel,
|
||||||
Size: req.ImageSize,
|
Size: req.ImageSize,
|
||||||
Count: 1,
|
Count: 1,
|
||||||
|
Images: referenceImages,
|
||||||
IdempotencyKey: imageGenerationIdempotencyKey(req, index, prompt, false),
|
IdempotencyKey: imageGenerationIdempotencyKey(req, index, prompt, false),
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -308,6 +312,7 @@ func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.
|
|||||||
Model: req.ImageModel,
|
Model: req.ImageModel,
|
||||||
Size: req.ImageSize,
|
Size: req.ImageSize,
|
||||||
Count: 1,
|
Count: 1,
|
||||||
|
Images: referenceImages,
|
||||||
IdempotencyKey: imageGenerationIdempotencyKey(req, index, fallback, true),
|
IdempotencyKey: imageGenerationIdempotencyKey(req, index, fallback, true),
|
||||||
})
|
})
|
||||||
if fallbackErr == nil {
|
if fallbackErr == nil {
|
||||||
@@ -329,12 +334,79 @@ func imageGenerationIdempotencyKey(req design.AgentRequest, index int, prompt st
|
|||||||
strings.TrimSpace(req.ImageSize),
|
strings.TrimSpace(req.ImageSize),
|
||||||
strconv.Itoa(index),
|
strconv.Itoa(index),
|
||||||
kind,
|
kind,
|
||||||
|
strings.Join(referenceImageURLs(req), "\n"),
|
||||||
strings.TrimSpace(prompt),
|
strings.TrimSpace(prompt),
|
||||||
}, "\n")
|
}, "\n")
|
||||||
sum := sha256.Sum256([]byte(seed))
|
sum := sha256.Sum256([]byte(seed))
|
||||||
return "img-gen-" + kind + "-" + hex.EncodeToString(sum[:16])
|
return "img-gen-" + kind + "-" + hex.EncodeToString(sum[:16])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceImageURLs(req design.AgentRequest) []string {
|
||||||
|
urls := make([]string, 0)
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
add := func(value string) {
|
||||||
|
value = strings.TrimSpace(strings.TrimRight(value, ",。,."))
|
||||||
|
if value == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, ok := seen[value]; ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
urls = append(urls, value)
|
||||||
|
}
|
||||||
|
addFromText := func(text string) {
|
||||||
|
for _, match := range referenceImageDirectiveURLPattern.FindAllStringSubmatch(text, -1) {
|
||||||
|
if len(match) > 1 {
|
||||||
|
add(match[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, match := range promptImageTokenURLPattern.FindAllStringSubmatch(text, -1) {
|
||||||
|
if len(match) > 1 && strings.TrimSpace(match[1]) != "" {
|
||||||
|
add(match[1])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(match) > 2 {
|
||||||
|
add(match[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addFromText(req.Prompt)
|
||||||
|
for _, message := range req.Messages {
|
||||||
|
for _, content := range message.Contents {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(content.Type)) {
|
||||||
|
case "image", "mask":
|
||||||
|
add(content.ImageURL)
|
||||||
|
case "text":
|
||||||
|
addFromText(content.Text)
|
||||||
|
}
|
||||||
|
if len(urls) >= 16 {
|
||||||
|
return urls[:16]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(urls) > 16 {
|
||||||
|
return urls[:16]
|
||||||
|
}
|
||||||
|
return urls
|
||||||
|
}
|
||||||
|
|
||||||
|
func withReferenceImageInstruction(prompt string, imageCount int) string {
|
||||||
|
prompt = strings.TrimSpace(prompt)
|
||||||
|
if prompt == "" || imageCount <= 0 {
|
||||||
|
return prompt
|
||||||
|
}
|
||||||
|
lower := strings.ToLower(prompt)
|
||||||
|
if strings.Contains(prompt, "附带参考图") || strings.Contains(lower, "attached reference image") || strings.Contains(lower, "input reference image") {
|
||||||
|
return prompt
|
||||||
|
}
|
||||||
|
if prefersEnglish(prompt) {
|
||||||
|
return "Input reference image(s) are attached. Use them as the visual source of truth for product shape, fabric, color, structure, and identity; when extracting, continuing, or editing a product, do not invent a different item.\n" + prompt
|
||||||
|
}
|
||||||
|
return "已随请求附带参考图;生成时必须以参考图中的商品外形、材质、颜色、结构细节和身份为准,拆出、延续或编辑商品时不要另造不同商品。\n" + prompt
|
||||||
|
}
|
||||||
|
|
||||||
func imagePromptTooLarge(prompt string) bool {
|
func imagePromptTooLarge(prompt string) bool {
|
||||||
return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes
|
return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes
|
||||||
}
|
}
|
||||||
@@ -404,6 +476,8 @@ var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×|
|
|||||||
var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`)
|
var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`)
|
||||||
|
|
||||||
var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`)
|
var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`)
|
||||||
|
var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[((][^))\n]+[))])?\s*[::]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`)
|
||||||
|
var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`)
|
||||||
|
|
||||||
func fullFrameImagePrompt(prompt string) string {
|
func fullFrameImagePrompt(prompt string) string {
|
||||||
prompt = strings.TrimSpace(prompt)
|
prompt = strings.TrimSpace(prompt)
|
||||||
|
|||||||
@@ -214,6 +214,52 @@ func TestDesignAgentGeneratesImageThroughPiAgent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDesignAgentPassesInlineReferenceImagesToImageGenerator(t *testing.T) {
|
||||||
|
imageGenerator := &fakeImageGenerator{}
|
||||||
|
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
|
||||||
|
|
||||||
|
_, err := agent.Plan(context.Background(), design.AgentRequest{
|
||||||
|
Prompt: "把参考图里的灰裤子拆成单件商品图",
|
||||||
|
Mode: "image",
|
||||||
|
ImageModel: "gpt-image-2",
|
||||||
|
Messages: []design.AgentChatMessage{
|
||||||
|
{
|
||||||
|
Role: "user",
|
||||||
|
Contents: []design.AgentContent{
|
||||||
|
{Type: "image", ImageURL: "https://example.com/pants-rack.png", ImageWidth: 1200, ImageHeight: 900},
|
||||||
|
{Type: "text", Text: "把参考图里的灰裤子拆成单件商品图", TextSource: "input"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imageGenerator.opts.Images) != 1 || imageGenerator.opts.Images[0] != "https://example.com/pants-rack.png" {
|
||||||
|
t.Fatalf("expected reference image to be passed to GPT Image 2, got %#v", imageGenerator.opts.Images)
|
||||||
|
}
|
||||||
|
if !strings.Contains(imageGenerator.prompt, "参考图") {
|
||||||
|
t.Fatalf("expected prompt to tell image model to use attached reference, got %s", imageGenerator.prompt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDesignAgentPassesPromptDirectiveReferenceImagesToImageGenerator(t *testing.T) {
|
||||||
|
imageGenerator := &fakeImageGenerator{}
|
||||||
|
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
|
||||||
|
|
||||||
|
_, err := agent.Plan(context.Background(), design.AgentRequest{
|
||||||
|
Prompt: "参考图 1(image):https://example.com/pants-rack.png\n灰裤子拆成单件衣服,为了后面电商配图。",
|
||||||
|
Mode: "image",
|
||||||
|
ImageModel: "gpt-image-2",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imageGenerator.opts.Images) != 1 || imageGenerator.opts.Images[0] != "https://example.com/pants-rack.png" {
|
||||||
|
t.Fatalf("expected directive reference image to be passed to GPT Image 2, got %#v", imageGenerator.opts.Images)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDesignAgentUsesDefaultImageSizeWithoutExplicitRequest(t *testing.T) {
|
func TestDesignAgentUsesDefaultImageSizeWithoutExplicitRequest(t *testing.T) {
|
||||||
imageGenerator := &fakeImageGenerator{}
|
imageGenerator := &fakeImageGenerator{}
|
||||||
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
|
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
|
||||||
|
|||||||
@@ -3,11 +3,18 @@ package piagent
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/textproto"
|
||||||
|
"net/url"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
@@ -35,19 +42,22 @@ type ImageRequestOptions struct {
|
|||||||
Model string
|
Model string
|
||||||
Size string
|
Size string
|
||||||
Count int
|
Count int
|
||||||
|
Images []string
|
||||||
IdempotencyKey string
|
IdempotencyKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
type ImageClientOptions struct {
|
type ImageClientOptions struct {
|
||||||
BaseURL string
|
BaseURL string
|
||||||
APIKey string
|
APIKey string
|
||||||
TimeoutSeconds int
|
InputImageTransport string
|
||||||
|
TimeoutSeconds int
|
||||||
}
|
}
|
||||||
|
|
||||||
type ImageClient struct {
|
type ImageClient struct {
|
||||||
baseURL string
|
baseURL string
|
||||||
apiKey string
|
apiKey string
|
||||||
client *http.Client
|
inputImageTransport string
|
||||||
|
client *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewImageClient(opts ImageClientOptions) *ImageClient {
|
func NewImageClient(opts ImageClientOptions) *ImageClient {
|
||||||
@@ -63,8 +73,9 @@ func NewImageClient(opts ImageClientOptions) *ImageClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &ImageClient{
|
return &ImageClient{
|
||||||
baseURL: strings.TrimRight(opts.BaseURL, "/"),
|
baseURL: strings.TrimRight(opts.BaseURL, "/"),
|
||||||
apiKey: opts.APIKey,
|
apiKey: opts.APIKey,
|
||||||
|
inputImageTransport: normalizeInputImageTransport(opts.InputImageTransport),
|
||||||
client: &http.Client{
|
client: &http.Client{
|
||||||
Timeout: timeout,
|
Timeout: timeout,
|
||||||
Transport: &http.Transport{
|
Transport: &http.Transport{
|
||||||
@@ -79,6 +90,15 @@ func NewImageClient(opts ImageClientOptions) *ImageClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeInputImageTransport(value string) string {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||||
|
case "url", "image_url", "image-url":
|
||||||
|
return "url"
|
||||||
|
default:
|
||||||
|
return "file"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageRequestOptions) (GeneratedImage, error) {
|
func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageRequestOptions) (GeneratedImage, error) {
|
||||||
prompt = strings.TrimSpace(prompt)
|
prompt = strings.TrimSpace(prompt)
|
||||||
if prompt == "" {
|
if prompt == "" {
|
||||||
@@ -99,20 +119,10 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
|
|||||||
if count > 10 {
|
if count > 10 {
|
||||||
count = 10
|
count = 10
|
||||||
}
|
}
|
||||||
|
images := normalizeImageInputs(opts.Images)
|
||||||
|
|
||||||
requestPayload := map[string]any{
|
|
||||||
"model": model,
|
|
||||||
"prompt": prompt,
|
|
||||||
}
|
|
||||||
if count > 1 {
|
|
||||||
requestPayload["n"] = count
|
|
||||||
}
|
|
||||||
if size != "" {
|
|
||||||
requestPayload["size"] = size
|
|
||||||
}
|
|
||||||
idempotencyKey := strings.TrimSpace(opts.IdempotencyKey)
|
idempotencyKey := strings.TrimSpace(opts.IdempotencyKey)
|
||||||
|
body, contentType, err := c.imageRequestBody(ctx, model, prompt, size, count, images)
|
||||||
body, err := json.Marshal(requestPayload)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return GeneratedImage{}, err
|
return GeneratedImage{}, err
|
||||||
}
|
}
|
||||||
@@ -121,7 +131,7 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
|
|||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
promptRunes := utf8.RuneCountInString(prompt)
|
promptRunes := utf8.RuneCountInString(prompt)
|
||||||
for attempt := 0; attempt < 3; attempt++ {
|
for attempt := 0; attempt < 3; attempt++ {
|
||||||
image, err := c.generateOnce(ctx, body, idempotencyKey)
|
image, err := c.generateOnce(ctx, body, contentType, idempotencyKey, len(images) > 0)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
logx.Infof(
|
logx.Infof(
|
||||||
"gpt image generation succeeded model=%s size=%s count=%d prompt_runes=%d attempt=%d elapsed=%s timeout=%s images=%d idempotency_key=%s",
|
"gpt image generation succeeded model=%s size=%s count=%d prompt_runes=%d attempt=%d elapsed=%s timeout=%s images=%d idempotency_key=%s",
|
||||||
@@ -166,13 +176,240 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
|
|||||||
return GeneratedImage{}, lastErr
|
return GeneratedImage{}, lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *ImageClient) generateOnce(ctx context.Context, body []byte, idempotencyKey string) (GeneratedImage, error) {
|
func (c *ImageClient) imageRequestBody(ctx context.Context, model string, prompt string, size string, count int, images []string) ([]byte, string, error) {
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/images/generations", bytes.NewReader(body))
|
if len(images) > 0 {
|
||||||
|
if c.inputImageTransport == "url" {
|
||||||
|
return jsonImageEditBody(model, prompt, size, count, images)
|
||||||
|
}
|
||||||
|
return c.multipartImageEditBody(ctx, model, prompt, size, count, images)
|
||||||
|
}
|
||||||
|
|
||||||
|
requestPayload := map[string]any{
|
||||||
|
"model": model,
|
||||||
|
"prompt": prompt,
|
||||||
|
}
|
||||||
|
if count > 1 {
|
||||||
|
requestPayload["n"] = count
|
||||||
|
}
|
||||||
|
if size != "" {
|
||||||
|
requestPayload["size"] = size
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(requestPayload)
|
||||||
|
return body, "application/json", err
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonImageEditBody(model string, prompt string, size string, count int, images []string) ([]byte, string, error) {
|
||||||
|
requestPayload := map[string]any{
|
||||||
|
"model": model,
|
||||||
|
"prompt": prompt,
|
||||||
|
"images": imageReferencePayload(images),
|
||||||
|
}
|
||||||
|
if count > 1 {
|
||||||
|
requestPayload["n"] = count
|
||||||
|
}
|
||||||
|
if size != "" {
|
||||||
|
requestPayload["size"] = size
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(requestPayload)
|
||||||
|
return body, "application/json", err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ImageClient) multipartImageEditBody(ctx context.Context, model string, prompt string, size string, count int, images []string) ([]byte, string, error) {
|
||||||
|
var body bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&body)
|
||||||
|
fields := map[string]string{
|
||||||
|
"model": model,
|
||||||
|
"prompt": prompt,
|
||||||
|
}
|
||||||
|
if size != "" {
|
||||||
|
fields["size"] = size
|
||||||
|
}
|
||||||
|
if count > 1 {
|
||||||
|
fields["n"] = strconv.Itoa(count)
|
||||||
|
}
|
||||||
|
for key, value := range fields {
|
||||||
|
if err := writer.WriteField(key, value); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fieldName := "image"
|
||||||
|
if len(images) > 1 {
|
||||||
|
fieldName = "image[]"
|
||||||
|
}
|
||||||
|
for index, image := range images {
|
||||||
|
input, err := c.imageInputFile(ctx, image, index)
|
||||||
|
if err != nil {
|
||||||
|
_ = writer.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
part, err := createImageFormFile(writer, fieldName, input)
|
||||||
|
if err != nil {
|
||||||
|
_ = writer.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if _, err := part.Write(input.Data); err != nil {
|
||||||
|
_ = writer.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := writer.Close(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
return body.Bytes(), writer.FormDataContentType(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func createImageFormFile(writer *multipart.Writer, fieldName string, input imageInputFile) (io.Writer, error) {
|
||||||
|
header := make(textproto.MIMEHeader)
|
||||||
|
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeMultipartValue(fieldName), escapeMultipartValue(input.FileName)))
|
||||||
|
contentType := strings.TrimSpace(input.ContentType)
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
header.Set("Content-Type", contentType)
|
||||||
|
return writer.CreatePart(header)
|
||||||
|
}
|
||||||
|
|
||||||
|
func escapeMultipartValue(value string) string {
|
||||||
|
return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
type imageInputFile struct {
|
||||||
|
FileName string
|
||||||
|
ContentType string
|
||||||
|
Data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ImageClient) imageInputFile(ctx context.Context, value string, index int) (imageInputFile, error) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if strings.HasPrefix(value, "data:image/") {
|
||||||
|
header, payload, ok := strings.Cut(value, ",")
|
||||||
|
if !ok {
|
||||||
|
return imageInputFile{}, fmt.Errorf("invalid image data url")
|
||||||
|
}
|
||||||
|
contentType := strings.TrimPrefix(strings.Split(header, ";")[0], "data:")
|
||||||
|
data, err := base64.StdEncoding.DecodeString(payload)
|
||||||
|
if err != nil {
|
||||||
|
return imageInputFile{}, err
|
||||||
|
}
|
||||||
|
return imageInputFile{
|
||||||
|
FileName: fmt.Sprintf("reference-%d%s", index+1, imageExtension(contentType, "")),
|
||||||
|
ContentType: contentType,
|
||||||
|
Data: data,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(value, "http://") && !strings.HasPrefix(value, "https://") {
|
||||||
|
return imageInputFile{}, fmt.Errorf("unsupported image input %q", value)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, value, nil)
|
||||||
|
if err != nil {
|
||||||
|
return imageInputFile{}, err
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "img-infinite-canvas/1.0")
|
||||||
|
resp, err := c.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return imageInputFile{}, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return imageInputFile{}, fmt.Errorf("download reference image failed: %s", resp.Status)
|
||||||
|
}
|
||||||
|
contentType := strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0])
|
||||||
|
data, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
|
||||||
|
if err != nil {
|
||||||
|
return imageInputFile{}, err
|
||||||
|
}
|
||||||
|
if len(data) == 0 {
|
||||||
|
return imageInputFile{}, fmt.Errorf("download reference image returned empty body")
|
||||||
|
}
|
||||||
|
if contentType == "" || !strings.HasPrefix(contentType, "image/") {
|
||||||
|
contentType = http.DetectContentType(data)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(contentType, "image/") {
|
||||||
|
return imageInputFile{}, fmt.Errorf("reference input is not an image: %s", contentType)
|
||||||
|
}
|
||||||
|
return imageInputFile{
|
||||||
|
FileName: imageFileName(value, contentType, index),
|
||||||
|
ContentType: contentType,
|
||||||
|
Data: data,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeImageInputs(images []string) []string {
|
||||||
|
if len(images) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
normalized := make([]string, 0, len(images))
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
for _, image := range images {
|
||||||
|
image = strings.TrimSpace(image)
|
||||||
|
if image == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[image]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[image] = struct{}{}
|
||||||
|
normalized = append(normalized, image)
|
||||||
|
if len(normalized) >= 16 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
func imageReferencePayload(images []string) []map[string]string {
|
||||||
|
items := make([]map[string]string, 0, len(images))
|
||||||
|
for _, image := range images {
|
||||||
|
items = append(items, map[string]string{"image_url": image})
|
||||||
|
}
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
|
func imageFileName(value string, contentType string, index int) string {
|
||||||
|
name := ""
|
||||||
|
if parsed, err := url.Parse(value); err == nil {
|
||||||
|
name = filepath.Base(parsed.Path)
|
||||||
|
}
|
||||||
|
if name == "" || name == "." || name == "/" {
|
||||||
|
name = fmt.Sprintf("reference-%d%s", index+1, imageExtension(contentType, ""))
|
||||||
|
}
|
||||||
|
if filepath.Ext(name) == "" {
|
||||||
|
name += imageExtension(contentType, ".png")
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
func imageExtension(contentType string, fallback string) string {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(contentType)) {
|
||||||
|
case "image/png":
|
||||||
|
return ".png"
|
||||||
|
case "image/jpeg", "image/jpg":
|
||||||
|
return ".jpg"
|
||||||
|
case "image/webp":
|
||||||
|
return ".webp"
|
||||||
|
case "image/gif":
|
||||||
|
return ".gif"
|
||||||
|
default:
|
||||||
|
if fallback != "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return ".png"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ImageClient) generateOnce(ctx context.Context, body []byte, contentType string, idempotencyKey string, edit bool) (GeneratedImage, error) {
|
||||||
|
endpoint := "/v1/images/generations"
|
||||||
|
if edit {
|
||||||
|
endpoint = "/v1/images/edits"
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+endpoint, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return GeneratedImage{}, err
|
return GeneratedImage{}, err
|
||||||
}
|
}
|
||||||
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", contentType)
|
||||||
req.Header.Set("Connection", "close")
|
req.Header.Set("Connection", "close")
|
||||||
if idempotencyKey != "" {
|
if idempotencyKey != "" {
|
||||||
req.Header.Set("Idempotency-Key", idempotencyKey)
|
req.Header.Set("Idempotency-Key", idempotencyKey)
|
||||||
|
|||||||
@@ -3,8 +3,10 @@ package piagent
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -64,6 +66,19 @@ func TestNewImageClientDisablesKeepAliveReuse(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNewImageClientDefaultsInputImagesToFileTransport(t *testing.T) {
|
||||||
|
client := NewImageClient(ImageClientOptions{
|
||||||
|
BaseURL: "http://example.com",
|
||||||
|
APIKey: "test-key",
|
||||||
|
})
|
||||||
|
if client == nil {
|
||||||
|
t.Fatal("expected image client")
|
||||||
|
}
|
||||||
|
if client.inputImageTransport != "file" {
|
||||||
|
t.Fatalf("expected file input image transport, got %q", client.inputImageTransport)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestImageClientSendsTransportDefaultSizeWhenUnset(t *testing.T) {
|
func TestImageClientSendsTransportDefaultSizeWhenUnset(t *testing.T) {
|
||||||
var requestBody map[string]any
|
var requestBody map[string]any
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -125,6 +140,155 @@ func TestImageClientSendsExplicitSize(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestImageClientUploadsDataReferenceImagesAsMultipart(t *testing.T) {
|
||||||
|
var requestPath string
|
||||||
|
var requestContentType string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
requestPath = r.URL.Path
|
||||||
|
requestContentType = r.Header.Get("Content-Type")
|
||||||
|
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||||
|
t.Errorf("parse multipart form: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
file, header, err := r.FormFile("image")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("expected image file field: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
data, err := io.ReadAll(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("read image file: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if header.Filename != "reference-1.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "reference bytes" {
|
||||||
|
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/image.png"}]}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key"})
|
||||||
|
_, err := client.Generate(context.Background(), "extract product from reference", ImageRequestOptions{
|
||||||
|
Model: "gpt-image-2",
|
||||||
|
Images: []string{"data:image/png;base64,cmVmZXJlbmNlIGJ5dGVz"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if requestPath != "/v1/images/edits" {
|
||||||
|
t.Fatalf("expected image edit endpoint for references, got %s", requestPath)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(requestContentType, "multipart/form-data") {
|
||||||
|
t.Fatalf("expected multipart edit request, got %s", requestContentType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageClientSendsReferenceImagesByURLWhenConfigured(t *testing.T) {
|
||||||
|
var requestBody map[string]any
|
||||||
|
var requestPath string
|
||||||
|
var requestContentType string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
requestPath = r.URL.Path
|
||||||
|
requestContentType = r.Header.Get("Content-Type")
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/image.png"}]}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key", InputImageTransport: "url"})
|
||||||
|
_, err := client.Generate(context.Background(), "extract product from reference", ImageRequestOptions{
|
||||||
|
Model: "gpt-image-2",
|
||||||
|
Images: []string{" https://example.com/reference.png ", "", "https://example.com/reference.png", "https://example.com/second.webp"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if requestPath != "/v1/images/edits" {
|
||||||
|
t.Fatalf("expected image edit endpoint for references, got %s", requestPath)
|
||||||
|
}
|
||||||
|
if requestContentType != "application/json" {
|
||||||
|
t.Fatalf("expected url transport to use JSON, got %s", requestContentType)
|
||||||
|
}
|
||||||
|
images, ok := requestBody["images"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected images array in request, got %#v", requestBody["images"])
|
||||||
|
}
|
||||||
|
if len(images) != 2 {
|
||||||
|
t.Fatalf("expected normalized reference images, got %#v", images)
|
||||||
|
}
|
||||||
|
first, ok := images[0].(map[string]any)
|
||||||
|
if !ok || first["image_url"] != "https://example.com/reference.png" {
|
||||||
|
t.Fatalf("expected first image_url reference, got %#v", images[0])
|
||||||
|
}
|
||||||
|
second, ok := images[1].(map[string]any)
|
||||||
|
if !ok || second["image_url"] != "https://example.com/second.webp" {
|
||||||
|
t.Fatalf("expected second image_url reference, got %#v", images[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageClientUploadsInternalReferenceImagesAsMultipart(t *testing.T) {
|
||||||
|
var sawEdit bool
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/reference.png":
|
||||||
|
w.Header().Set("Content-Type", "image/png")
|
||||||
|
_, _ = w.Write([]byte("internal image bytes"))
|
||||||
|
case "/v1/images/edits":
|
||||||
|
sawEdit = true
|
||||||
|
if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
|
||||||
|
t.Errorf("expected multipart edit request, got %s", r.Header.Get("Content-Type"))
|
||||||
|
}
|
||||||
|
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||||
|
t.Errorf("parse multipart form: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.FormValue("model") != "gpt-image-2" || r.FormValue("prompt") != "edit the internal image" || r.FormValue("size") != "1024x1024" {
|
||||||
|
t.Errorf("unexpected multipart fields: model=%q prompt=%q size=%q", r.FormValue("model"), r.FormValue("prompt"), r.FormValue("size"))
|
||||||
|
}
|
||||||
|
file, header, err := r.FormFile("image")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("expected image file field: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
data, err := io.ReadAll(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("read image file: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if header.Filename != "reference.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "internal image bytes" {
|
||||||
|
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/edited.png"}]}`))
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key"})
|
||||||
|
image, err := client.Generate(context.Background(), "edit the internal image", ImageRequestOptions{
|
||||||
|
Model: "gpt-image-2",
|
||||||
|
Size: "1024x1024",
|
||||||
|
Images: []string{server.URL + "/reference.png"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if image.URL != "https://example.com/edited.png" {
|
||||||
|
t.Fatalf("expected edited image, got %#v", image)
|
||||||
|
}
|
||||||
|
if !sawEdit {
|
||||||
|
t.Fatal("expected image edit request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestImageClientParsesAlternateImageResponseShapes(t *testing.T) {
|
func TestImageClientParsesAlternateImageResponseShapes(t *testing.T) {
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|||||||
@@ -304,9 +304,10 @@ func newImageGenerator(c config.Config) piagent.ImageGenerator {
|
|||||||
opts := c.Agent.Image
|
opts := c.Agent.Image
|
||||||
apiKey := resolveImageAPIKey(opts)
|
apiKey := resolveImageAPIKey(opts)
|
||||||
client := piagent.NewImageClient(piagent.ImageClientOptions{
|
client := piagent.NewImageClient(piagent.ImageClientOptions{
|
||||||
BaseURL: opts.BaseURL,
|
BaseURL: opts.BaseURL,
|
||||||
APIKey: apiKey,
|
APIKey: apiKey,
|
||||||
TimeoutSeconds: opts.TimeoutSeconds,
|
InputImageTransport: opts.InputImageTransport,
|
||||||
|
TimeoutSeconds: opts.TimeoutSeconds,
|
||||||
})
|
})
|
||||||
if client == nil {
|
if client == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
Reference in New Issue
Block a user