feat(image): pass reference images to GPT image model for edits

Server: forward canvas/reference images to the image model so extract, continue, and edit requests stay faithful to the source product. - Add ImageRequestOptions.Images and route requests with inputs to the /v1/images/edits endpoint, via multipart upload (default) or JSON image_url payload, selectable through Agent.Image.InputImageTransport. - Extract reference URLs from prompt directives, inline @image tokens, and chat message image/mask contents; fold them into the idempotency key and prepend a reference-fidelity instruction to the prompt. Frontend: - Shrink CanvasToast to a compact top pill. - Paste an internal transparent clipboard node instead of re-uploading when it matches the clipboard image. - Keep agent-project polling alive on stream error while streaming.
2026-07-08 01:19:20 +08:00
parent bb7fce9171
commit 2f5291a0f9
11 changed files with 614 additions and 63 deletions
@@ -283,6 +283,9 @@ func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prom
 }

 func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) {
+	referenceImages := referenceImageURLs(req)
+	prompt = withReferenceImageInstruction(prompt, len(referenceImages))
+	fallback = withReferenceImageInstruction(fallback, len(referenceImages))
 	prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes)
 	fallback = prepareGPTImagePrompt(fallback, imageGenerationFallbackMaxRunes)
 	if imagePromptTooLarge(prompt) && strings.TrimSpace(fallback) != "" {
@@ -293,6 +296,7 @@ func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.
 		Model:          req.ImageModel,
 		Size:           req.ImageSize,
 		Count:          1,
+		Images:         referenceImages,
 		IdempotencyKey: imageGenerationIdempotencyKey(req, index, prompt, false),
 	})
 	if err == nil {
@@ -308,6 +312,7 @@ func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.
 		Model:          req.ImageModel,
 		Size:           req.ImageSize,
 		Count:          1,
+		Images:         referenceImages,
 		IdempotencyKey: imageGenerationIdempotencyKey(req, index, fallback, true),
 	})
 	if fallbackErr == nil {
@@ -329,12 +334,79 @@ func imageGenerationIdempotencyKey(req design.AgentRequest, index int, prompt st
 		strings.TrimSpace(req.ImageSize),
 		strconv.Itoa(index),
 		kind,
+		strings.Join(referenceImageURLs(req), "\n"),
 		strings.TrimSpace(prompt),
 	}, "\n")
 	sum := sha256.Sum256([]byte(seed))
 	return "img-gen-" + kind + "-" + hex.EncodeToString(sum[:16])
 }

+func referenceImageURLs(req design.AgentRequest) []string {
+	urls := make([]string, 0)
+	seen := make(map[string]struct{})
+	add := func(value string) {
+		value = strings.TrimSpace(strings.TrimRight(value, "，。,."))
+		if value == "" {
+			return
+		}
+		if _, ok := seen[value]; ok {
+			return
+		}
+		seen[value] = struct{}{}
+		urls = append(urls, value)
+	}
+	addFromText := func(text string) {
+		for _, match := range referenceImageDirectiveURLPattern.FindAllStringSubmatch(text, -1) {
+			if len(match) > 1 {
+				add(match[1])
+			}
+		}
+		for _, match := range promptImageTokenURLPattern.FindAllStringSubmatch(text, -1) {
+			if len(match) > 1 && strings.TrimSpace(match[1]) != "" {
+				add(match[1])
+				continue
+			}
+			if len(match) > 2 {
+				add(match[2])
+			}
+		}
+	}
+
+	addFromText(req.Prompt)
+	for _, message := range req.Messages {
+		for _, content := range message.Contents {
+			switch strings.ToLower(strings.TrimSpace(content.Type)) {
+			case "image", "mask":
+				add(content.ImageURL)
+			case "text":
+				addFromText(content.Text)
+			}
+			if len(urls) >= 16 {
+				return urls[:16]
+			}
+		}
+	}
+	if len(urls) > 16 {
+		return urls[:16]
+	}
+	return urls
+}
+
+func withReferenceImageInstruction(prompt string, imageCount int) string {
+	prompt = strings.TrimSpace(prompt)
+	if prompt == "" || imageCount <= 0 {
+		return prompt
+	}
+	lower := strings.ToLower(prompt)
+	if strings.Contains(prompt, "附带参考图") || strings.Contains(lower, "attached reference image") || strings.Contains(lower, "input reference image") {
+		return prompt
+	}
+	if prefersEnglish(prompt) {
+		return "Input reference image(s) are attached. Use them as the visual source of truth for product shape, fabric, color, structure, and identity; when extracting, continuing, or editing a product, do not invent a different item.\n" + prompt
+	}
+	return "已随请求附带参考图；生成时必须以参考图中的商品外形、材质、颜色、结构细节和身份为准，拆出、延续或编辑商品时不要另造不同商品。\n" + prompt
+}
+
 func imagePromptTooLarge(prompt string) bool {
 	return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes
 }
@@ -404,6 +476,8 @@ var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×|
 var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`)

 var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`)
+var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[（(][^）)\n]+[）)])?\s*[:：]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`)
+var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`)

 func fullFrameImagePrompt(prompt string) string {
 	prompt = strings.TrimSpace(prompt)