diff --git a/server/internal/infrastructure/piagent/creative_agent.go b/server/internal/infrastructure/piagent/creative_agent.go index 023e70e..e50721e 100644 --- a/server/internal/infrastructure/piagent/creative_agent.go +++ b/server/internal/infrastructure/piagent/creative_agent.go @@ -577,6 +577,9 @@ func buildDecisionPrompt(req design.AgentDecisionRequest) string { "", "Input summary:", agentMessageContentSummary(req.Messages), + "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), }, "\n") } @@ -598,6 +601,9 @@ func buildResearchDecisionPrompt(req design.AgentResearchDecisionRequest) string req.Prompt, agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "Mode: " + req.Mode, }, "\n") } @@ -624,6 +630,9 @@ func buildAgentTaskPlanPrompt(req design.AgentTaskPlanRequest) string { req.Prompt, agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "Mode: " + req.Mode, "Web search: " + searchState, "", @@ -648,6 +657,9 @@ func buildConversationPrompt(req design.AgentConversationRequest) string { "Inline references:", agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "User message:", req.Prompt, "", @@ -697,6 +709,9 @@ func buildImagePromptPlannerInput(req design.AgentImagePromptRequest) string { req.Prompt, agentMessageContentSummary(req.Messages), "", + "Inline reference placement:", + inlineReferencePlacementContext(req.Prompt, req.Messages), + "", "Mode: " + req.Mode, }, "\n") } diff --git a/server/internal/infrastructure/piagent/design_agent.go b/server/internal/infrastructure/piagent/design_agent.go index d07aeb9..2a125aa 100644 --- a/server/internal/infrastructure/piagent/design_agent.go +++ b/server/internal/infrastructure/piagent/design_agent.go @@ -284,6 +284,8 @@ func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prom func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) { referenceImages := referenceImageURLs(req) + prompt = withReferenceImageContext(prompt, req, len(referenceImages)) + fallback = withReferenceImageContext(fallback, req, len(referenceImages)) prompt = withReferenceImageInstruction(prompt, len(referenceImages)) fallback = withReferenceImageInstruction(fallback, len(referenceImages)) prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes) @@ -345,7 +347,7 @@ func referenceImageURLs(req design.AgentRequest) []string { urls := make([]string, 0) seen := make(map[string]struct{}) add := func(value string) { - value = strings.TrimSpace(strings.TrimRight(value, "，。,.")) + value = sanitizeReferenceURL(value) if value == "" { return } @@ -407,6 +409,24 @@ func withReferenceImageInstruction(prompt string, imageCount int) string { return "已随请求附带参考图；只按用户原话说明的用途使用每张图，例如商品/源图、Logo、风格、色彩、构图或编辑对象。需要编辑或延续源商品时保持其身份，不要仅凭图片内容臆测未说明的角色。\n" + prompt } +func withReferenceImageContext(prompt string, req design.AgentRequest, imageCount int) string { + prompt = strings.TrimSpace(prompt) + if prompt == "" || imageCount <= 0 { + return prompt + } + if strings.Contains(prompt, "Inline reference placement:") || strings.Contains(prompt, "参考图位置关系") { + return prompt + } + context := inlineReferencePlacementContext(req.Prompt, req.Messages) + if context == "" || context == "none" { + return prompt + } + if prefersEnglish(prompt + "\n" + req.Prompt) { + return "Inline reference placement (attached image inputs follow this order; nearby text before or after each image defines its role, so do not swap reference roles):\n" + context + "\n\n" + prompt + } + return "参考图位置关系（附带图片按此顺序输入；每张图前后的相邻文字代表它的用途/角色，不要调换参考图角色）：\n" + context + "\n\n" + prompt +} + func imagePromptTooLarge(prompt string) bool { return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes } @@ -476,7 +496,7 @@ var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×| var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`) var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`) -var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[（(][^）)\n]+[）)])?\s*[:：]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`) +var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[（(][^）)\n]+[）)])?\s*[:：]\s*(https?://[^\s\]，。]+|data:image/[^\s\]，。]+|/[^\s\]，。]+)`) var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`) func fullFrameImagePrompt(prompt string) string { diff --git a/server/internal/infrastructure/piagent/design_agent_test.go b/server/internal/infrastructure/piagent/design_agent_test.go index 5ea3da4..3c0a7b1 100644 --- a/server/internal/infrastructure/piagent/design_agent_test.go +++ b/server/internal/infrastructure/piagent/design_agent_test.go @@ -260,6 +260,35 @@ func TestDesignAgentPassesPromptDirectiveReferenceImagesToImageGenerator(t *test } } +func TestDesignAgentPreservesInlineReferencePlacementForImageGenerator(t *testing.T) { + imageGenerator := &fakeImageGenerator{} + agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{}) + prompt := "参考图 1（0.jpg）：https://example.com/mood.webp 为服装品牌 stillday 创建情绪板 logo 设计 参考图 2（image.png）：https://example.com/logo.png，包含材质、色彩、摄影姿态。" + + _, err := agent.Plan(context.Background(), design.AgentRequest{ + Prompt: prompt, + Mode: "image", + ImageModel: "gpt-image-2", + Messages: []design.AgentChatMessage{ + { + Role: "user", + Contents: []design.AgentContent{ + {Type: "text", Text: prompt, TextSource: "input"}, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if len(imageGenerator.opts.Images) != 2 || imageGenerator.opts.Images[0] != "https://example.com/mood.webp" || imageGenerator.opts.Images[1] != "https://example.com/logo.png" { + t.Fatalf("expected ordered reference images, got %#v", imageGenerator.opts.Images) + } + if !containsAll(imageGenerator.prompt, []string{"参考图 2", "image.png", "logo 设计"}) { + t.Fatalf("expected image prompt to carry inline placement semantics, got %s", imageGenerator.prompt) + } +} + func TestDesignAgentUsesDefaultImageSizeWithoutExplicitRequest(t *testing.T) { imageGenerator := &fakeImageGenerator{} agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{}) diff --git a/server/internal/infrastructure/piagent/reference_context.go b/server/internal/infrastructure/piagent/reference_context.go new file mode 100644 index 0000000..f139842 --- /dev/null +++ b/server/internal/infrastructure/piagent/reference_context.go @@ -0,0 +1,273 @@ +package piagent + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "img_infinite_canvas/internal/domain/design" +) + +type inlineReferenceMatch struct { + start int + end int + index int + name string + url string +} + +var ( + inlineReferenceDirectivePattern = regexp.MustCompile(`(?i)(参考图|reference image)\s*(\d+)?\s*(?:[（(]([^）)\n]+)[）)])?\s*[:：]\s*(https?://[^\s\]，。]+|data:image/[^\s\]，。]+|/[^\s\]，。]+|attachment://reference-\d+)`) + inlinePromptImageTokenPattern = regexp.MustCompile(`\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]`) +) + +func inlineReferencePlacementContext(prompt string, messages []design.AgentChatMessage) string { + messageText := agentMessageInlineReferenceText(messages) + source := prompt + if len(collectInlineReferenceMatches(messageText)) > 0 && (len(collectInlineReferenceMatches(prompt)) == 0 || utf8.RuneCountInString(messageText) >= utf8.RuneCountInString(prompt)) { + source = messageText + } + return inlineReferencePlacementContextFromText(source, prefersEnglish(prompt+"\n"+messageText)) +} + +func inlineReferencePlacementContextFromText(text string, english bool) string { + matches := collectInlineReferenceMatches(text) + if len(matches) == 0 { + return "none" + } + if len(matches) > 16 { + matches = matches[:16] + } + + lines := make([]string, 0, len(matches)+1) + if english { + lines = append(lines, "Image input order follows the inline reference order. Nearby text before or after a reference is semantic role/context for that image.") + } else { + lines = append(lines, "图片输入顺序按参考图出现顺序；某张图前后的相邻文字是这张图的角色/用途线索。") + } + for order, match := range matches { + beforeStart := 0 + if order > 0 { + beforeStart = matches[order-1].end + } + afterEnd := len(text) + if order+1 < len(matches) { + afterEnd = matches[order+1].start + } + before := compactReferenceNeighbor(text[beforeStart:match.start], true) + after := compactReferenceNeighbor(text[match.end:afterEnd], false) + if english { + lines = append(lines, englishReferencePlacementLine(order+1, match, before, after)) + continue + } + lines = append(lines, chineseReferencePlacementLine(order+1, match, before, after)) + } + return strings.Join(lines, "\n") +} + +func agentMessageInlineReferenceText(messages []design.AgentChatMessage) string { + var builder strings.Builder + imageIndex := 0 + writeText := func(text string) { + text = strings.TrimSpace(text) + if text == "" { + return + } + if builder.Len() > 0 { + builder.WriteString(" ") + } + builder.WriteString(text) + } + for _, message := range messages { + if strings.TrimSpace(message.Role) != "" && strings.ToLower(strings.TrimSpace(message.Role)) != "user" { + continue + } + for _, content := range message.Contents { + switch strings.ToLower(strings.TrimSpace(content.Type)) { + case "text": + if strings.EqualFold(strings.TrimSpace(content.TextSource), "settings") { + continue + } + writeText(stripInlineImageGeneratorDirectiveLines(content.Text)) + case "image", "mask": + imageIndex++ + name := compactInlineReferenceName(content.ImageName) + if name == "" { + name = "image" + } + url := sanitizeReferenceURL(content.ImageURL) + if url == "" { + url = fmt.Sprintf("attachment://reference-%d", imageIndex) + } + writeText(fmt.Sprintf("参考图 %d（%s）：%s", imageIndex, name, url)) + } + } + } + return builder.String() +} + +func collectInlineReferenceMatches(text string) []inlineReferenceMatch { + if strings.TrimSpace(text) == "" { + return nil + } + matches := make([]inlineReferenceMatch, 0) + + for _, indexes := range inlinePromptImageTokenPattern.FindAllStringSubmatchIndex(text, -1) { + match := inlineReferenceMatch{start: indexes[0], end: indexes[1]} + match.index = atoiSubmatch(text, indexes, 2) + match.name = submatch(text, indexes, 4) + match.url = sanitizeReferenceURL(firstNonEmptySubmatch(text, indexes, 8, 10)) + if match.url != "" { + matches = append(matches, match) + } + } + + for _, indexes := range inlineReferenceDirectivePattern.FindAllStringSubmatchIndex(text, -1) { + match := inlineReferenceMatch{start: indexes[0], end: indexes[1]} + match.index = atoiSubmatch(text, indexes, 4) + match.name = submatch(text, indexes, 6) + match.url = sanitizeReferenceURL(submatch(text, indexes, 8)) + if match.url != "" { + matches = append(matches, match) + } + } + + sort.SliceStable(matches, func(i, j int) bool { + if matches[i].start == matches[j].start { + return matches[i].end > matches[j].end + } + return matches[i].start < matches[j].start + }) + + deduped := make([]inlineReferenceMatch, 0, len(matches)) + for _, match := range matches { + overlaps := false + for _, existing := range deduped { + if match.start < existing.end && match.end > existing.start { + overlaps = true + break + } + } + if !overlaps { + deduped = append(deduped, match) + } + } + return deduped +} + +func chineseReferencePlacementLine(order int, match inlineReferenceMatch, before string, after string) string { + parts := []string{fmt.Sprintf("- %s", referenceDisplayName(order, match, false))} + if before != "" { + parts = append(parts, fmt.Sprintf("前文“%s”", before)) + } + if after != "" { + parts = append(parts, fmt.Sprintf("后文“%s”", after)) + } + if len(parts) == 1 { + parts = append(parts, "无相邻文字") + } + return strings.Join(parts, "：") +} + +func englishReferencePlacementLine(order int, match inlineReferenceMatch, before string, after string) string { + parts := []string{fmt.Sprintf("- %s", referenceDisplayName(order, match, true))} + if before != "" { + parts = append(parts, fmt.Sprintf("previous text %q", before)) + } + if after != "" { + parts = append(parts, fmt.Sprintf("next text %q", after)) + } + if len(parts) == 1 { + parts = append(parts, "no adjacent text") + } + return strings.Join(parts, ": ") +} + +func referenceDisplayName(order int, match inlineReferenceMatch, english bool) string { + name := compactInlineReferenceName(match.name) + if name == "" { + name = "image" + } + if english { + label := fmt.Sprintf("Reference image %d", order) + if match.index > 0 && match.index != order { + label += fmt.Sprintf(" / user label %d", match.index) + } + return fmt.Sprintf("%s (%s)", label, name) + } + label := fmt.Sprintf("参考图 %d", order) + if match.index > 0 && match.index != order { + label += fmt.Sprintf(" / 用户标注 %d", match.index) + } + return fmt.Sprintf("%s（%s）", label, name) +} + +func compactReferenceNeighbor(value string, keepTail bool) string { + value = strings.ReplaceAll(value, "\u00a0", " ") + value = imagePromptURLPattern.ReplaceAllString(value, "[参考图]") + value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ") + value = strings.Trim(value, " \t\r\n，。,.；;、") + if value == "" { + return "" + } + const maxRunes = 72 + if utf8.RuneCountInString(value) <= maxRunes { + return value + } + runes := []rune(value) + if keepTail { + return "..." + string(runes[len(runes)-maxRunes:]) + } + return string(runes[:maxRunes]) + "..." +} + +func sanitizeReferenceURL(value string) string { + return strings.TrimSpace(strings.TrimRight(value, "，。,.；;、")) +} + +func compactInlineReferenceName(value string) string { + return strings.Join(strings.Fields(value), " ") +} + +func stripInlineImageGeneratorDirectiveLines(text string) string { + lines := strings.Split(text, "\n") + filtered := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "image generator target node:") { + continue + } + filtered = append(filtered, line) + } + return strings.TrimSpace(strings.Join(filtered, "\n")) +} + +func submatch(text string, indexes []int, offset int) string { + if offset+1 >= len(indexes) || indexes[offset] < 0 || indexes[offset+1] < 0 { + return "" + } + return text[indexes[offset]:indexes[offset+1]] +} + +func firstNonEmptySubmatch(text string, indexes []int, offsets ...int) string { + for _, offset := range offsets { + if value := submatch(text, indexes, offset); strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func atoiSubmatch(text string, indexes []int, offset int) int { + value := strings.TrimSpace(submatch(text, indexes, offset)) + if value == "" { + return 0 + } + number, err := strconv.Atoi(value) + if err != nil { + return 0 + } + return number +}