From 68427ad91a79afc7220b5d1179877458b224f7bd Mon Sep 17 00:00:00 2001 From: liangxu Date: Wed, 8 Jul 2026 16:17:52 +0800 Subject: [PATCH] feat(piagent): give image generator inline reference placement context Derive an ordered map of each inline reference image to the surrounding prompt text describing its role, and feed it into the decision, research, task-plan, conversation and image-prompt builders. The image prompt now carries this placement so reference roles are not swapped. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../infrastructure/piagent/creative_agent.go | 15 + .../infrastructure/piagent/design_agent.go | 24 +- .../piagent/design_agent_test.go | 29 ++ .../piagent/reference_context.go | 273 ++++++++++++++++++ 4 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 server/internal/infrastructure/piagent/reference_context.go diff --git a/server/internal/infrastructure/piagent/creative_agent.go b/server/internal/infrastructure/piagent/creative_agent.go index 023e70e..e50721e 100644 --- a/server/internal/infrastructure/piagent/creative_agent.go +++ b/server/internal/infrastructure/piagent/creative_agent.go @@ -577,6 +577,9 @@ func buildDecisionPrompt(req design.AgentDecisionRequest) string { "", "Input summary:", agentMessageContentSummary(req.Messages), + "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), }, "\n") } @@ -598,6 +601,9 @@ func buildResearchDecisionPrompt(req design.AgentResearchDecisionRequest) string req.Prompt, agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "Mode: " + req.Mode, }, "\n") } @@ -624,6 +630,9 @@ func buildAgentTaskPlanPrompt(req design.AgentTaskPlanRequest) string { req.Prompt, agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "Mode: " + req.Mode, "Web search: " + searchState, "", @@ -648,6 +657,9 @@ func buildConversationPrompt(req design.AgentConversationRequest) string { "Inline references:", agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "User message:", req.Prompt, "", @@ -697,6 +709,9 @@ func buildImagePromptPlannerInput(req design.AgentImagePromptRequest) string { req.Prompt, agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "Mode: " + req.Mode, }, "\n") } diff --git a/server/internal/infrastructure/piagent/design_agent.go b/server/internal/infrastructure/piagent/design_agent.go index d07aeb9..2a125aa 100644 --- a/server/internal/infrastructure/piagent/design_agent.go +++ b/server/internal/infrastructure/piagent/design_agent.go @@ -284,6 +284,8 @@ func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prom func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) { referenceImages := referenceImageURLs(req) + prompt = withReferenceImageContext(prompt, req, len(referenceImages)) + fallback = withReferenceImageContext(fallback, req, len(referenceImages)) prompt = withReferenceImageInstruction(prompt, len(referenceImages)) fallback = withReferenceImageInstruction(fallback, len(referenceImages)) prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes) @@ -345,7 +347,7 @@ func referenceImageURLs(req design.AgentRequest) []string { urls := make([]string, 0) seen := make(map[string]struct{}) add := func(value string) { - value = strings.TrimSpace(strings.TrimRight(value, "，。,.")) + value = sanitizeReferenceURL(value) if value == "" { return } @@ -407,6 +409,24 @@ func withReferenceImageInstruction(prompt string, imageCount int) string { return "已随请求附带参考图；只按用户原话说明的用途使用每张图，例如商品/源图、Logo、风格、色彩、构图或编辑对象。需要编辑或延续源商品时保持其身份，不要仅凭图片内容臆测未说明的角色。\n" + prompt } +func withReferenceImageContext(prompt string, req design.AgentRequest, imageCount int) string { + prompt = strings.TrimSpace(prompt) + if prompt == "" || imageCount <= 0 { + return prompt + } + if strings.Contains(prompt, "Inline reference placement:") || strings.Contains(prompt, "参考图位置关系") { + return prompt + } + context := inlineReferencePlacementContext(req.Prompt, req.Messages) + if context == "" || context == "none" { + return prompt + } + if prefersEnglish(prompt + "\n" + req.Prompt) { + return "Inline reference placement (attached image inputs follow this order; nearby text before or after each image defines its role, so do not swap reference roles):\n" + context + "\n\n" + prompt + } + return "参考图位置关系（附带图片按此顺序输入；每张图前后的相邻文字代表它的用途/角色，不要调换参考图角色）：\n" + context + "\n\n" + prompt +} + func imagePromptTooLarge(prompt string) bool { return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes } @@ -476,7 +496,7 @@ var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×| var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`) var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`) -var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[（(][^）)\n]+[）)])?\s*[:：]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`) +var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[（(][^）)\n]+[）)])?\s*[:：]\s*(https?://[^\s\]，。]+|data:image/[^\s\]，。]+|/[^\s\]，。]+)`) var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`) func fullFrameImagePrompt(prompt string) string { diff --git a/server/internal/infrastructure/piagent/design_agent_test.go b/server/internal/infrastructure/piagent/design_agent_test.go index 5ea3da4..3c0a7b1 100644 --- a/server/internal/infrastructure/piagent/design_agent_test.go +++ b/server/internal/infrastructure/piagent/design_agent_test.go @@ -260,6 +260,35 @@ func TestDesignAgentPassesPromptDirectiveReferenceImagesToImageGenerator(t *test } } +func TestDesignAgentPreservesInlineReferencePlacementForImageGenerator(t *testing.T) { + imageGenerator := &fakeImageGenerator{} + agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{}) + prompt := "参考图 1（0.jpg）：https://example.com/mood.webp 为服装品牌 stillday 创建情绪板 logo 设计 参考图 2（image.png）：https://example.com/logo.png，包含材质、色彩、摄影姿态。" + + _, err := agent.Plan(context.Background(), design.AgentRequest{ + Prompt: prompt, + Mode: "image", + ImageModel: "gpt-image-2", + Messages: []design.AgentChatMessage{ + { + Role: "user", + Contents: []design.AgentContent{ + {Type: "text", Text: prompt, TextSource: "input"}, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if len(imageGenerator.opts.Images) != 2 || imageGenerator.opts.Images[0] != "https://example.com/mood.webp" || imageGenerator.opts.Images[1] != "https://example.com/logo.png" { + t.Fatalf("expected ordered reference images, got %#v", imageGenerator.opts.Images) + } + if !containsAll(imageGenerator.prompt, []string{"参考图 2", "image.png", "logo 设计"}) { + t.Fatalf("expected image prompt to carry inline placement semantics, got %s", imageGenerator.prompt) + } +} + func TestDesignAgentUsesDefaultImageSizeWithoutExplicitRequest(t *testing.T) { imageGenerator := &fakeImageGenerator{} agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{}) diff --git a/server/internal/infrastructure/piagent/reference_context.go b/server/internal/infrastructure/piagent/reference_context.go new file mode 100644 index 0000000..f139842 --- /dev/null +++ b/server/internal/infrastructure/piagent/reference_context.go @@ -0,0 +1,273 @@ +package piagent + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "img_infinite_canvas/internal/domain/design" +) + +type inlineReferenceMatch struct { + start int + end int + index int + name string + url string +} + +var ( + inlineReferenceDirectivePattern = regexp.MustCompile(`(?i)(参考图|reference image)\s*(\d+)?\s*(?:[（(]([^）)\n]+)[）)])?\s*[:：]\s*(https?://[^\s\]，。]+|data:image/[^\s\]，。]+|/[^\s\]，。]+|attachment://reference-\d+)`) + inlinePromptImageTokenPattern = regexp.MustCompile(`\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]`) +) + +func inlineReferencePlacementContext(prompt string, messages []design.AgentChatMessage) string { + messageText := agentMessageInlineReferenceText(messages) + source := prompt + if len(collectInlineReferenceMatches(messageText)) > 0 && (len(collectInlineReferenceMatches(prompt)) == 0 || utf8.RuneCountInString(messageText) >= utf8.RuneCountInString(prompt)) { + source = messageText + } + return inlineReferencePlacementContextFromText(source, prefersEnglish(prompt+"\n"+messageText)) +} + +func inlineReferencePlacementContextFromText(text string, english bool) string { + matches := collectInlineReferenceMatches(text) + if len(matches) == 0 { + return "none" + } + if len(matches) > 16 { + matches = matches[:16] + } + + lines := make([]string, 0, len(matches)+1) + if english { + lines = append(lines, "Image input order follows the inline reference order. Nearby text before or after a reference is semantic role/context for that image.") + } else { + lines = append(lines, "图片输入顺序按参考图出现顺序；某张图前后的相邻文字是这张图的角色/用途线索。") + } + for order, match := range matches { + beforeStart := 0 + if order > 0 { + beforeStart = matches[order-1].end + } + afterEnd := len(text) + if order+1 < len(matches) { + afterEnd = matches[order+1].start + } + before := compactReferenceNeighbor(text[beforeStart:match.start], true) + after := compactReferenceNeighbor(text[match.end:afterEnd], false) + if english { + lines = append(lines, englishReferencePlacementLine(order+1, match, before, after)) + continue + } + lines = append(lines, chineseReferencePlacementLine(order+1, match, before, after)) + } + return strings.Join(lines, "\n") +} + +func agentMessageInlineReferenceText(messages []design.AgentChatMessage) string { + var builder strings.Builder + imageIndex := 0 + writeText := func(text string) { + text = strings.TrimSpace(text) + if text == "" { + return + } + if builder.Len() > 0 { + builder.WriteString(" ") + } + builder.WriteString(text) + } + for _, message := range messages { + if strings.TrimSpace(message.Role) != "" && strings.ToLower(strings.TrimSpace(message.Role)) != "user" { + continue + } + for _, content := range message.Contents { + switch strings.ToLower(strings.TrimSpace(content.Type)) { + case "text": + if strings.EqualFold(strings.TrimSpace(content.TextSource), "settings") { + continue + } + writeText(stripInlineImageGeneratorDirectiveLines(content.Text)) + case "image", "mask": + imageIndex++ + name := compactInlineReferenceName(content.ImageName) + if name == "" { + name = "image" + } + url := sanitizeReferenceURL(content.ImageURL) + if url == "" { + url = fmt.Sprintf("attachment://reference-%d", imageIndex) + } + writeText(fmt.Sprintf("参考图 %d（%s）：%s", imageIndex, name, url)) + } + } + } + return builder.String() +} + +func collectInlineReferenceMatches(text string) []inlineReferenceMatch { + if strings.TrimSpace(text) == "" { + return nil + } + matches := make([]inlineReferenceMatch, 0) + + for _, indexes := range inlinePromptImageTokenPattern.FindAllStringSubmatchIndex(text, -1) { + match := inlineReferenceMatch{start: indexes[0], end: indexes[1]} + match.index = atoiSubmatch(text, indexes, 2) + match.name = submatch(text, indexes, 4) + match.url = sanitizeReferenceURL(firstNonEmptySubmatch(text, indexes, 8, 10)) + if match.url != "" { + matches = append(matches, match) + } + } + + for _, indexes := range inlineReferenceDirectivePattern.FindAllStringSubmatchIndex(text, -1) { + match := inlineReferenceMatch{start: indexes[0], end: indexes[1]} + match.index = atoiSubmatch(text, indexes, 4) + match.name = submatch(text, indexes, 6) + match.url = sanitizeReferenceURL(submatch(text, indexes, 8)) + if match.url != "" { + matches = append(matches, match) + } + } + + sort.SliceStable(matches, func(i, j int) bool { + if matches[i].start == matches[j].start { + return matches[i].end > matches[j].end + } + return matches[i].start < matches[j].start + }) + + deduped := make([]inlineReferenceMatch, 0, len(matches)) + for _, match := range matches { + overlaps := false + for _, existing := range deduped { + if match.start < existing.end && match.end > existing.start { + overlaps = true + break + } + } + if !overlaps { + deduped = append(deduped, match) + } + } + return deduped +} + +func chineseReferencePlacementLine(order int, match inlineReferenceMatch, before string, after string) string { + parts := []string{fmt.Sprintf("- %s", referenceDisplayName(order, match, false))} + if before != "" { + parts = append(parts, fmt.Sprintf("前文“%s”", before)) + } + if after != "" { + parts = append(parts, fmt.Sprintf("后文“%s”", after)) + } + if len(parts) == 1 { + parts = append(parts, "无相邻文字") + } + return strings.Join(parts, "：") +} + +func englishReferencePlacementLine(order int, match inlineReferenceMatch, before string, after string) string { + parts := []string{fmt.Sprintf("- %s", referenceDisplayName(order, match, true))} + if before != "" { + parts = append(parts, fmt.Sprintf("previous text %q", before)) + } + if after != "" { + parts = append(parts, fmt.Sprintf("next text %q", after)) + } + if len(parts) == 1 { + parts = append(parts, "no adjacent text") + } + return strings.Join(parts, ": ") +} + +func referenceDisplayName(order int, match inlineReferenceMatch, english bool) string { + name := compactInlineReferenceName(match.name) + if name == "" { + name = "image" + } + if english { + label := fmt.Sprintf("Reference image %d", order) + if match.index > 0 && match.index != order { + label += fmt.Sprintf(" / user label %d", match.index) + } + return fmt.Sprintf("%s (%s)", label, name) + } + label := fmt.Sprintf("参考图 %d", order) + if match.index > 0 && match.index != order { + label += fmt.Sprintf(" / 用户标注 %d", match.index) + } + return fmt.Sprintf("%s（%s）", label, name) +} + +func compactReferenceNeighbor(value string, keepTail bool) string { + value = strings.ReplaceAll(value, "\u00a0", " ") + value = imagePromptURLPattern.ReplaceAllString(value, "[参考图]") + value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ") + value = strings.Trim(value, " \t\r\n，。,.；;、") + if value == "" { + return "" + } + const maxRunes = 72 + if utf8.RuneCountInString(value) <= maxRunes { + return value + } + runes := []rune(value) + if keepTail { + return "..." + string(runes[len(runes)-maxRunes:]) + } + return string(runes[:maxRunes]) + "..." +} + +func sanitizeReferenceURL(value string) string { + return strings.TrimSpace(strings.TrimRight(value, "，。,.；;、")) +} + +func compactInlineReferenceName(value string) string { + return strings.Join(strings.Fields(value), " ") +} + +func stripInlineImageGeneratorDirectiveLines(text string) string { + lines := strings.Split(text, "\n") + filtered := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "image generator target node:") { + continue + } + filtered = append(filtered, line) + } + return strings.TrimSpace(strings.Join(filtered, "\n")) +} + +func submatch(text string, indexes []int, offset int) string { + if offset+1 >= len(indexes) || indexes[offset] < 0 || indexes[offset+1] < 0 { + return "" + } + return text[indexes[offset]:indexes[offset+1]] +} + +func firstNonEmptySubmatch(text string, indexes []int, offsets ...int) string { + for _, offset := range offsets { + if value := submatch(text, indexes, offset); strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func atoiSubmatch(text string, indexes []int, offset int) int { + value := strings.TrimSpace(submatch(text, indexes, offset)) + if value == "" { + return 0 + } + number, err := strconv.Atoi(value) + if err != nil { + return 0 + } + return number +}