feat(agent): clarify vague briefs before generating and harden prompts
Let the planner return a clarify/chat intent that asks 2-4 targeted questions (imageCount 0) when essentials are missing, instead of forging ahead. Tighten prompt guidance to preserve user wording and stated reference-image roles, avoid inventing copy/specs, keep durable brand constraints, and treat vague taste feedback as clarification. Also fix the generation heartbeat goroutine to stop cleanly via defer, add research-query fallbacks, and refresh image-only capability copy. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1377,12 +1377,17 @@ func (s *DesignService) planAgentTask(ctx context.Context, project design.Projec
|
||||
}
|
||||
|
||||
func normalizeAgentTaskPlan(plan design.AgentTaskPlan, prompt string, mode string, messages []design.AgentChatMessage, webSearchEnabled bool, forceImage bool) design.AgentTaskPlan {
|
||||
plan.Intent = normalizeAgentIntent(plan.Intent)
|
||||
if forceImage || shouldGenerateForAgentPrompt(prompt, mode, messages) {
|
||||
rawIntent := strings.TrimSpace(plan.Intent)
|
||||
plan.Intent = normalizeAgentIntent(rawIntent)
|
||||
if forceImage || shouldForceImageMode(mode) {
|
||||
plan.Intent = "image"
|
||||
}
|
||||
if plan.Intent == "" {
|
||||
plan.Intent = "chat"
|
||||
if shouldGenerateForAgentPrompt(prompt, mode, messages) {
|
||||
plan.Intent = "image"
|
||||
} else {
|
||||
plan.Intent = "chat"
|
||||
}
|
||||
}
|
||||
if !webSearchEnabled {
|
||||
plan.ShouldSearch = false
|
||||
@@ -1434,9 +1439,9 @@ func normalizeAgentTaskPlan(plan design.AgentTaskPlan, prompt string, mode strin
|
||||
func fallbackPlanningResponse(prompt string, intent string) string {
|
||||
if normalizeAgentIntent(intent) == "image" {
|
||||
if prefersEnglishText(prompt) {
|
||||
return "I will break this into the right visual deliverables and generate them on the canvas."
|
||||
return "I will turn this into concrete image deliverables and generate them on the canvas."
|
||||
}
|
||||
return "我会先把需求拆成合适的视觉交付项,再在画布中生成。"
|
||||
return "我会先把需求拆成具体图片交付项,再在画布中生成。"
|
||||
}
|
||||
if prefersEnglishText(prompt) {
|
||||
return "I will answer based on the current canvas context."
|
||||
@@ -1573,9 +1578,9 @@ func (s *DesignService) updateAgentMessage(ctx context.Context, projectID string
|
||||
func fallbackAgentChatResponse(prompt string, project design.Project) string {
|
||||
if isCapabilityPrompt(strings.ToLower(strings.TrimSpace(prompt))) {
|
||||
if prefersEnglishText(prompt) {
|
||||
return "I mainly help with image and design-canvas work: generating visuals, editing images, shaping ecommerce pages, reading the current canvas, and suggesting the next concrete move. When you give me a clear visual task, I will enter the image-generation pipeline."
|
||||
return "I focus on image generation and image editing: brand/site visuals, ecommerce product images, posters, banners, social graphics, reference-based generation, background removal, vectorization, mockup placement, and multi-round canvas iteration. Give me a clear visual task and I will generate or edit images on the canvas."
|
||||
}
|
||||
return "我主要负责图片与设计画布:生成视觉稿、编辑图片、做电商页面方向、拆解当前画布并给下一步建议。你给我明确的作图目标时,我再进入生图链路。"
|
||||
return "我专注图片生成和图片编辑:品牌/官网视觉、电商产品图、海报 Banner、社媒图、参考图生图、去背景、矢量化、Mockup 贴图,以及在无限画布里多轮迭代。你给我明确的图片任务后,我会进入生图或修图链路。"
|
||||
}
|
||||
if strings.TrimSpace(project.Brief) != "" {
|
||||
if prefersEnglishText(prompt) {
|
||||
@@ -1594,7 +1599,9 @@ func normalizeAgentIntent(intent string) string {
|
||||
switch intent {
|
||||
case "image", "generate_image", "edit_image", "visual", "design":
|
||||
return "image"
|
||||
case "research":
|
||||
case "":
|
||||
return ""
|
||||
case "research", "clarify", "brief", "briefing", "question", "questions", "chat", "conversation":
|
||||
return "chat"
|
||||
default:
|
||||
return "chat"
|
||||
@@ -2571,7 +2578,9 @@ func (s *DesignService) completeCreateProjectGeneration(ctx context.Context, pro
|
||||
message.Type = "assistant"
|
||||
}
|
||||
if incremental {
|
||||
project.Nodes = mergeCompletedIncrementalGeneratedNodes(project.Nodes, plan.GeneratedNodes, "")
|
||||
project.Nodes = clearRemainingAgentGenerationPlaceholders(project.Nodes, "")
|
||||
project.Messages = appendMissingIncrementalArtifactMessages(project.Messages, threadID, prompt, plan.GeneratedNodes, now.Add(-time.Millisecond))
|
||||
} else {
|
||||
project.Nodes = mergeGeneratedNodesIntoTarget(nonGeneratingNodes(project.Nodes), plan.GeneratedNodes, "")
|
||||
project.Messages = append(project.Messages, toolArtifactsMessage(threadID, prompt, plan.GeneratedNodes, now.Add(-time.Millisecond)))
|
||||
@@ -2940,7 +2949,11 @@ func (s *DesignService) completeFollowupGeneration(ctx context.Context, projectI
|
||||
message.Type = "assistant"
|
||||
}
|
||||
if incremental {
|
||||
project.Nodes = mergeCompletedIncrementalGeneratedNodes(project.Nodes, plan.GeneratedNodes, targetNodeID)
|
||||
project.Nodes = clearRemainingAgentGenerationPlaceholders(project.Nodes, targetNodeID)
|
||||
if !isolated {
|
||||
project.Messages = appendMissingIncrementalArtifactMessages(project.Messages, threadID, prompt, plan.GeneratedNodes, now.Add(-time.Millisecond))
|
||||
}
|
||||
} else {
|
||||
baseNodes := nonGeneratingNodes(project.Nodes)
|
||||
if targetNodeID != "" {
|
||||
@@ -2986,7 +2999,15 @@ func (s *DesignService) planWithIncrementalImages(ctx context.Context, projectID
|
||||
|
||||
done := make(chan struct{})
|
||||
if appendArtifacts {
|
||||
go s.emitGenerationHeartbeat(ctx, projectID, threadID, req.Prompt, done)
|
||||
heartbeatStopped := make(chan struct{})
|
||||
go func() {
|
||||
defer close(heartbeatStopped)
|
||||
s.emitGenerationHeartbeat(ctx, projectID, threadID, req.Prompt, done)
|
||||
}()
|
||||
defer func() {
|
||||
close(done)
|
||||
<-heartbeatStopped
|
||||
}()
|
||||
}
|
||||
var mu sync.Mutex
|
||||
persistedNodes := make([]design.Node, 0, plannedGenerationResultCapacity(req))
|
||||
@@ -3004,9 +3025,6 @@ func (s *DesignService) planWithIncrementalImages(ctx context.Context, projectID
|
||||
persistedNodes = append(persistedNodes, node)
|
||||
return s.applyIncrementalGeneratedNode(ctx, projectID, threadID, prompt, targetNodeID, node, appendArtifacts)
|
||||
})
|
||||
if appendArtifacts {
|
||||
close(done)
|
||||
}
|
||||
if len(persistedNodes) > 0 {
|
||||
plan.GeneratedNodes = append([]design.Node(nil), persistedNodes...)
|
||||
}
|
||||
@@ -3039,6 +3057,32 @@ func completedGeneratedImageNodes(nodes []design.Node) []design.Node {
|
||||
return completed
|
||||
}
|
||||
|
||||
func mergeCompletedIncrementalGeneratedNodes(nodes []design.Node, generated []design.Node, targetNodeID string) []design.Node {
|
||||
completed := completedGeneratedImageNodes(generated)
|
||||
if len(completed) == 0 {
|
||||
return nodes
|
||||
}
|
||||
if strings.TrimSpace(targetNodeID) != "" {
|
||||
return mergeGeneratedNodesIntoTarget(nodes, completed, targetNodeID)
|
||||
}
|
||||
for _, node := range completed {
|
||||
nodes = upsertGeneratedNode(nodes, node)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func appendMissingIncrementalArtifactMessages(messages []design.Message, threadID string, prompt string, nodes []design.Node, createdAt time.Time) []design.Message {
|
||||
existing := generatedImageArtifactsByID(messages)
|
||||
for _, node := range completedGeneratedImageNodes(nodes) {
|
||||
if _, ok := existing[node.ID]; ok {
|
||||
continue
|
||||
}
|
||||
messages = append(messages, toolArtifactsMessage(threadID, prompt, []design.Node{node}, createdAt))
|
||||
existing[node.ID] = design.GeneratorTaskArtifact{}
|
||||
}
|
||||
return messages
|
||||
}
|
||||
|
||||
func plannedGenerationResultCapacity(req design.AgentRequest) int {
|
||||
count := req.TaskPlan.ImageCount
|
||||
if count <= 0 {
|
||||
@@ -3237,10 +3281,7 @@ func (s *DesignService) maybeAddResearchStep(ctx context.Context, projectID stri
|
||||
Messages: messages,
|
||||
Memory: buildAgentMemory(project, prompt, mode, messages),
|
||||
})
|
||||
if !decision.ShouldSearch {
|
||||
return nil
|
||||
}
|
||||
query := strings.TrimSpace(decision.Query)
|
||||
query := enabledResearchQuery(prompt, mode, decision)
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -3254,16 +3295,38 @@ func (s *DesignService) maybeAddPlannedResearchStep(ctx context.Context, project
|
||||
if strings.TrimSpace(plan.Intent) == "" {
|
||||
return s.maybeAddResearchStep(ctx, projectID, threadID, project, prompt, mode, messages, enabled)
|
||||
}
|
||||
if !plan.ShouldSearch {
|
||||
return nil
|
||||
}
|
||||
query := strings.TrimSpace(plan.SearchQuery)
|
||||
if query == "" && plan.ShouldSearch {
|
||||
query = fallbackResearchQuery(prompt, mode)
|
||||
}
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
return s.addResearchStep(ctx, projectID, threadID, query)
|
||||
}
|
||||
|
||||
func enabledResearchQuery(prompt string, mode string, decision design.AgentResearchDecision) string {
|
||||
if strings.TrimSpace(decision.Query) != "" {
|
||||
return strings.TrimSpace(decision.Query)
|
||||
}
|
||||
if !shouldSearchWhenEnabled(prompt, mode) {
|
||||
return ""
|
||||
}
|
||||
return fallbackResearchQuery(prompt, mode)
|
||||
}
|
||||
|
||||
func shouldSearchWhenEnabled(prompt string, mode string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(prompt))
|
||||
if text == "" || isConversationalPrompt(text) {
|
||||
return false
|
||||
}
|
||||
modeText := strings.ToLower(strings.TrimSpace(mode))
|
||||
if strings.Contains(modeText, "image-action") {
|
||||
return shouldRunResearchFallback(prompt, mode)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *DesignService) decideResearch(ctx context.Context, req design.AgentResearchDecisionRequest) design.AgentResearchDecision {
|
||||
if decider, ok := s.creativeAgent.(design.ResearchDecider); ok {
|
||||
if decision, err := decider.DecideResearch(ctx, req); err == nil {
|
||||
@@ -3366,6 +3429,11 @@ func shouldGenerateForAgentPrompt(prompt string, mode string, messages []design.
|
||||
return false
|
||||
}
|
||||
|
||||
func shouldForceImageMode(mode string) bool {
|
||||
modeText := strings.ToLower(strings.TrimSpace(mode))
|
||||
return strings.Contains(modeText, "image-action") || modeText == "image" || modeText == "pure-image"
|
||||
}
|
||||
|
||||
func hasAgentImageInput(messages []design.AgentChatMessage) bool {
|
||||
for _, message := range messages {
|
||||
for _, content := range message.Contents {
|
||||
@@ -5692,7 +5760,7 @@ func inspirationItems() []design.InspirationItem {
|
||||
return []design.InspirationItem{
|
||||
{ID: "luxury-spring", Title: "2026 春季系列设计趋势", Category: "Luxury Showcase", Description: "强对比摄影、低饱和色块与清晰购买动线。", Accent: "#111827"},
|
||||
{ID: "natural-ui", Title: "极简自然主义 UI", Category: "Mobile App", Description: "玻璃拟态信息卡与植物识别流程。", Accent: "#6b8f71"},
|
||||
{ID: "blue-geometry", Title: "蓝色 3D 几何广告", Category: "Campaign", Description: "冷感材质、漂浮图形和高亮 CTA。", Accent: "#2563eb"},
|
||||
{ID: "blue-geometry", Title: "蓝色立体几何广告", Category: "Campaign", Description: "冷感材质、漂浮图形和高亮 CTA。", Accent: "#2563eb"},
|
||||
{ID: "brand-kit", Title: "天然护肤品牌板", Category: "Brand Kit", Description: "字体、插画、包装和详情页模块统一沉淀。", Accent: "#c8a568"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,6 +319,37 @@ func (a incrementalCreateProjectAgent) Replay(ctx context.Context, project desig
|
||||
return design.AgentReplay{}, nil
|
||||
}
|
||||
|
||||
type staleOverwriteIncrementalCreateProjectAgent struct {
|
||||
repo design.Repository
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (a staleOverwriteIncrementalCreateProjectAgent) Plan(ctx context.Context, req design.AgentRequest) (design.AgentPlan, error) {
|
||||
return design.AgentPlan{}, errors.New("expected incremental generation")
|
||||
}
|
||||
|
||||
func (a staleOverwriteIncrementalCreateProjectAgent) PlanIncremental(ctx context.Context, req design.AgentRequest, onResult func(design.AgentImageGenerationResult) error) (design.AgentPlan, error) {
|
||||
if len(req.Project.Nodes) == 0 {
|
||||
a.t.Fatal("expected planned placeholder before image generation")
|
||||
}
|
||||
staleProject := req.Project
|
||||
node := design.Node{ID: req.Project.Nodes[0].ID, Type: design.NodeTypeImage, Title: "盲盒IP视觉体系", Content: "https://example.com/blind-box-system.png", Status: "success", Width: 1024, Height: 1024}
|
||||
if err := onResult(design.AgentImageGenerationResult{Index: 0, Node: node}); err != nil {
|
||||
return design.AgentPlan{}, err
|
||||
}
|
||||
if err := a.repo.Save(ctx, staleProject); err != nil {
|
||||
return design.AgentPlan{}, err
|
||||
}
|
||||
return design.AgentPlan{
|
||||
Message: design.Message{Role: "assistant", Type: "assistant", Title: "图片生成完成", Content: "盲盒IP视觉体系已生成。"},
|
||||
GeneratedNodes: []design.Node{node},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a staleOverwriteIncrementalCreateProjectAgent) Replay(ctx context.Context, project design.Project) (design.AgentReplay, error) {
|
||||
return design.AgentReplay{}, nil
|
||||
}
|
||||
|
||||
type partialFailingIncrementalCreateProjectAgent struct{}
|
||||
|
||||
func (a partialFailingIncrementalCreateProjectAgent) Plan(ctx context.Context, req design.AgentRequest) (design.AgentPlan, error) {
|
||||
@@ -541,6 +572,54 @@ func TestAgentChatCreatesPlaceholdersFromModelPlannedImageTasks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentChatAllowsPlannerToClarifyVisualBriefBeforeGenerating(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := memory.NewProjectRepository()
|
||||
queue := &fakeJobQueue{}
|
||||
service := NewDesignService(repo, nil, nil, nil, nil, fakeCreativeChatAgent{
|
||||
decision: design.AgentDecision{Intent: "image", Reason: "visual request"},
|
||||
taskPlan: design.AgentTaskPlan{
|
||||
Intent: "clarify",
|
||||
UserFacingResponse: "开始生图前我需要确认:Logo 是否已有?主视觉偏大胆色彩还是极简?首页要优先展示哪些品类?",
|
||||
ImageCount: 0,
|
||||
},
|
||||
response: "开始生图前我需要确认:Logo 是否已有?主视觉偏大胆色彩还是极简?首页要优先展示哪些品类?",
|
||||
})
|
||||
service.SetJobQueue(queue)
|
||||
project := design.Project{
|
||||
ID: "project-clarify-before-generate",
|
||||
Title: "Clarify before generate",
|
||||
Status: design.StatusReady,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
if err := repo.Save(ctx, project); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
thread, err := service.AgentChat(ctx, project.ID, design.AgentChatRequest{
|
||||
Messages: []design.AgentChatMessage{
|
||||
{
|
||||
Role: "user",
|
||||
Contents: []design.AgentContent{
|
||||
{Type: "text", Text: "still day 品牌官网设计,主打 mood 配色和舒适针织。"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(thread.Project.Nodes) != 0 {
|
||||
t.Fatalf("expected no image placeholders while clarifying, got %#v", thread.Project.Nodes)
|
||||
}
|
||||
if len(queue.jobs) != 1 || queue.jobs[0].Kind != design.JobAgentConversation {
|
||||
t.Fatalf("expected clarification to enqueue chat job, got %#v", queue.jobs)
|
||||
}
|
||||
if queue.jobs[0].Conversation.TaskPlan.Intent != "chat" {
|
||||
t.Fatalf("expected clarification plan normalized to chat, got %#v", queue.jobs[0].Conversation.TaskPlan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateProjectAsyncReturnsBeforeTaskPlanning(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := memory.NewProjectRepository()
|
||||
@@ -702,6 +781,62 @@ func TestCreateProjectGenerationStreamsEachImageAsItFinishes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateProjectGenerationFinalizesAfterStaleHeartbeatOverwrite(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := memory.NewProjectRepository()
|
||||
creative := fakeCreativeChatAgent{
|
||||
decision: design.AgentDecision{Intent: "image"},
|
||||
taskPlan: design.AgentTaskPlan{
|
||||
Intent: "image",
|
||||
UserFacingResponse: "我将生成一张盲盒IP视觉体系图。",
|
||||
ImageCount: 1,
|
||||
ImageTasks: []design.AgentImageTask{
|
||||
{Title: "盲盒IP视觉体系", Brief: "角色、包装、展示台、贴纸和发售海报总览"},
|
||||
},
|
||||
},
|
||||
}
|
||||
service := NewDesignService(repo, staleOverwriteIncrementalCreateProjectAgent{repo: repo, t: t}, nil, nil, nil, creative)
|
||||
service.SetJobQueue(&fakeJobQueue{})
|
||||
project, err := service.CreateProjectAsync(ctx, "", "设计一张盲盒IP视觉体系。", "gpt-image-2", "", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := service.ProcessJob(ctx, design.Job{
|
||||
Kind: design.JobCreateProjectGeneration,
|
||||
ProjectID: project.ID,
|
||||
UserID: project.UserID,
|
||||
ThreadID: project.LastThreadID,
|
||||
Prompt: project.Brief,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
saved, err := repo.Get(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.Status != design.StatusReady {
|
||||
t.Fatalf("expected project ready after stale heartbeat overwrite, got %s", saved.Status)
|
||||
}
|
||||
if len(saved.Nodes) != 1 || saved.Nodes[0].Status != "success" || saved.Nodes[0].Content != "https://example.com/blind-box-system.png" {
|
||||
t.Fatalf("expected final generated node restored, got %#v", saved.Nodes)
|
||||
}
|
||||
foundToolSuccess := false
|
||||
foundArtifact := false
|
||||
for _, message := range saved.Messages {
|
||||
if message.Title == "GPT Image 2" && message.ThreadID == project.LastThreadID && message.Status == "success" {
|
||||
foundToolSuccess = true
|
||||
}
|
||||
if message.Name == "generate_media" && strings.Contains(message.Content, "blind-box-system.png") {
|
||||
foundArtifact = true
|
||||
}
|
||||
}
|
||||
if !foundToolSuccess || !foundArtifact {
|
||||
t.Fatalf("expected final tool success and restored artifact message, got %#v", saved.Messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateProjectGenerationPartialFailurePreservesSuccessfulImagesAndClosesTask(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := memory.NewProjectRepository()
|
||||
@@ -1147,7 +1282,7 @@ func TestGenerationStepHeartbeatUpdatesLatestToolMessage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebSearchRunsOnlyWhenModelDecisionNeedsIt(t *testing.T) {
|
||||
func TestWebSearchRunsWhenEnabledEvenIfModelSkips(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := memory.NewProjectRepository()
|
||||
project := design.Project{
|
||||
@@ -1172,13 +1307,18 @@ func TestWebSearchRunsOnlyWhenModelDecisionNeedsIt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if searcher.called {
|
||||
t.Fatal("expected web search to stay off when model decision says no")
|
||||
if !searcher.called {
|
||||
t.Fatal("expected enabled web search to run even when model decision says no")
|
||||
}
|
||||
if searcher.query != fallbackResearchQuery("优化当前首页视觉", "design") {
|
||||
t.Fatalf("expected fallback search query, got %q", searcher.query)
|
||||
}
|
||||
if agent.memory.ShortTerm == "" || agent.memory.LongTerm == "" {
|
||||
t.Fatalf("expected short and long memory in research decision, got %#v", agent.memory)
|
||||
}
|
||||
|
||||
searcher.called = false
|
||||
searcher.query = ""
|
||||
agent.decision = design.AgentResearchDecision{ShouldSearch: true, Query: "still day ecommerce homepage reference", Reason: "needs external references"}
|
||||
err = service.maybeAddResearchStep(ctx, project.ID, "thread-1", project, "优化当前首页视觉", "design", nil, true)
|
||||
if err != nil {
|
||||
@@ -1262,7 +1402,7 @@ func TestAgentChatUsesPlannerSearchQueryInsteadOfRawPrompt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitialWebSearchDoesNotFallbackToRawPromptWhenModelSkips(t *testing.T) {
|
||||
func TestInitialWebSearchFallsBackToSearchQueryWhenModelSkips(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := memory.NewProjectRepository()
|
||||
threadID := "thread-initial-search"
|
||||
@@ -1288,15 +1428,18 @@ func TestInitialWebSearchDoesNotFallbackToRawPromptWhenModelSkips(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if searcher.called {
|
||||
t.Fatalf("expected no raw-prompt fallback search, got query=%q", searcher.query)
|
||||
if !searcher.called {
|
||||
t.Fatal("expected enabled initial web search to run")
|
||||
}
|
||||
if searcher.query != fallbackResearchQuery("做一个美漫风格分镜", "design") {
|
||||
t.Fatalf("expected fallback search query, got %q", searcher.query)
|
||||
}
|
||||
saved, err := repo.Get(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(saved.Messages) != len(project.Messages) {
|
||||
t.Fatalf("expected no research message, got %#v", saved.Messages)
|
||||
if len(saved.Messages) == len(project.Messages) || saved.Messages[len(saved.Messages)-1].Title != "Synthesize Skills" {
|
||||
t.Fatalf("expected research message, got %#v", saved.Messages)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@ func (c *CreativeAgent) Decide(ctx context.Context, req design.AgentDecisionRequ
|
||||
"Decide whether the user wants the agent to call the image generation/editing pipeline or just answer in chat.",
|
||||
"Return strict JSON only: {\"intent\":\"image\"|\"chat\",\"reason\":\"short reason\"}.",
|
||||
"Use image only for explicit visual production, image editing, ecommerce design output, page/banner/poster/product/mockup generation, or iterative visual changes.",
|
||||
"Use chat when the user asks what the agent can do, when critical brand/design requirements are missing, or when a brief needs 2-4 targeted clarification questions before generating.",
|
||||
"Use chat for greetings, thanks, capability questions, vague conversation, or clarification without a concrete visual action.",
|
||||
}, " "),
|
||||
},
|
||||
@@ -89,8 +90,10 @@ func (c *CreativeAgent) Respond(ctx context.Context, req design.AgentConversatio
|
||||
"Answer naturally in the user's language and locale, like a senior creative partner who understands image generation, ecommerce pages, product visuals, and canvas workflows.",
|
||||
"Use short memory for the current turn and canvas state; use long memory for durable project preferences and prior decisions.",
|
||||
"Do not use canned templates. Do not claim you generated an image unless an image tool actually ran.",
|
||||
"Do not mention unsupported video, audio, or 3D features. This product currently supports image generation/editing and infinite-canvas design workflows.",
|
||||
"For greetings, reply briefly and invite a concrete next step. For capability questions, explain useful image/design capabilities with current canvas context.",
|
||||
"Do not mention unsupported video, audio, or 3D model features. This product supports image generation, image editing, and infinite-canvas design workflows only.",
|
||||
"For greetings, reply briefly and invite a concrete next step. For capability questions, answer with image-only capabilities: brand/site visuals, ecommerce product images, posters/banners/social images, reference-based generation, image edits, background removal, vectorization, mockup placement, and multi-round canvas iteration.",
|
||||
"When a current project exists, connect capabilities to the canvas instead of giving a generic product brochure.",
|
||||
"When the task plan indicates missing requirements or contains user response questions, ask those questions directly and stop; do not pretend generation has started.",
|
||||
}, " "),
|
||||
},
|
||||
{
|
||||
@@ -109,8 +112,10 @@ func (c *CreativeAgent) StreamRespond(ctx context.Context, req design.AgentConve
|
||||
"Answer naturally in the user's language and locale, like a senior creative partner who understands image generation, ecommerce pages, product visuals, and canvas workflows.",
|
||||
"Use short memory for the current turn and canvas state; use long memory for durable project preferences and prior decisions.",
|
||||
"Do not use canned templates. Do not claim you generated an image unless an image tool actually ran.",
|
||||
"Do not mention unsupported video, audio, or 3D features. This product currently supports image generation/editing and infinite-canvas design workflows.",
|
||||
"For greetings, reply briefly and invite a concrete next step. For capability questions, explain useful image/design capabilities with current canvas context.",
|
||||
"Do not mention unsupported video, audio, or 3D model features. This product supports image generation, image editing, and infinite-canvas design workflows only.",
|
||||
"For greetings, reply briefly and invite a concrete next step. For capability questions, answer with image-only capabilities: brand/site visuals, ecommerce product images, posters/banners/social images, reference-based generation, image edits, background removal, vectorization, mockup placement, and multi-round canvas iteration.",
|
||||
"When a current project exists, connect capabilities to the canvas instead of giving a generic product brochure.",
|
||||
"When the task plan indicates missing requirements or contains user response questions, ask those questions directly and stop; do not pretend generation has started.",
|
||||
}, " "),
|
||||
},
|
||||
{
|
||||
@@ -150,15 +155,26 @@ func (c *CreativeAgent) PlanAgentTask(ctx context.Context, req design.AgentTaskP
|
||||
"content": strings.Join([]string{
|
||||
"You are the automatic planner for a Moteva-style creative agent.",
|
||||
"Read the user's request, canvas, inline references, short memory, and long memory, then decide the next workflow.",
|
||||
"Return strict JSON only: {\"intent\":\"image\"|\"chat\",\"reason\":\"short internal reason\",\"userFacingResponse\":\"one concise localized sentence explaining what you will do\",\"shouldSearch\":true|false,\"searchQuery\":\"query or empty\",\"searchReason\":\"short reason\",\"imageCount\":1-10,\"imageTasks\":[{\"title\":\"localized short title\",\"brief\":\"specific output brief\"}]}",
|
||||
"Return strict JSON only: {\"intent\":\"image\"|\"chat\",\"reason\":\"short internal reason\",\"userFacingResponse\":\"one concise localized sentence explaining what you will do\",\"shouldSearch\":true|false,\"searchQuery\":\"query or empty\",\"searchReason\":\"short reason\",\"imageCount\":0-10,\"imageTasks\":[{\"title\":\"localized short title\",\"brief\":\"specific output brief\"}]}",
|
||||
"Intent image means the user wants visual production, image editing, ecommerce/page/poster/product/character/illustration/output generation, or concrete canvas creation.",
|
||||
"Intent chat means greetings, thanks, vague discussion, capability questions, or missing critical information.",
|
||||
"For open-ended brand, ecommerce, homepage, logo, or campaign requests with missing essentials, choose intent chat and ask 2-4 targeted questions before generation. Essentials can include brand/logo assets, visual direction, product/category scope, audience, key sections, slogan/copy, deliverable format, and must-keep constraints.",
|
||||
"For homepage or website requests, treat static visual drafts as image tasks and runnable HTML/code as out of scope. If the user asks for HTML/code, clarify whether they want a static visual draft or image assets instead.",
|
||||
"If the user asks for visible page modules, slogans, navigation labels, button copy, or other text but does not provide the content, choose intent chat unless they explicitly authorize you to decide.",
|
||||
"For multiple uploaded reference images with no stated roles, choose intent chat and ask which images are product/source, logo, style, color, or composition references. Do not infer roles just by looking at the images.",
|
||||
"When the user asks for multiple images of the same new person, character, model, or product and no consistency anchor exists, choose intent chat and explain that an anchor/reference must be confirmed first. Multi-panel sheets inside one image do not need separate image count.",
|
||||
"If the user already supplied enough signal or explicitly asks to proceed, choose image and make reasonable art-direction decisions.",
|
||||
"When the user gives vague taste feedback such as not stylish enough, more premium, more big-company, treat it as feedback rather than an edit instruction. Ask one focused clarification question with likely directions unless the user explicitly authorizes your judgment or provides a reference image/brand.",
|
||||
"If the user explicitly authorizes your judgment, keep durable constraints from memory and translate vague feedback into concrete visual actions in imageTasks: composition, color, typography, product/category structure, spacing, and ecommerce hierarchy.",
|
||||
"If web search is disabled, shouldSearch must be false. If enabled, choose search only when external references will improve the work: public brand/product facts, current trends, competitor/style references, niche cultural or visual references, or explicit research/search requests.",
|
||||
"When shouldSearch is true, searchQuery must be your analyzed search keywords, not the raw user prompt. Keep it focused on reference material such as design drafts, case studies, visual identity, design process, moodboards, and style references.",
|
||||
"When shouldSearch is true, keep the user's named brands, artists, products, and style terms intact. Add only minimal retrieval terms such as case study, visual identity, moodboard, official site, or lookbook; do not replace the user's wording with your own aesthetic interpretation.",
|
||||
"For logo tasks, avoid logo-maker/tool results: include terms like logo design case study, brand identity, visual identity, design process, moodboard, and negative terms such as -maker -generator -free -template when useful.",
|
||||
"When research is used, treat it as reference collection. Do not turn unconfirmed visual inferences from search images into prompt text; use researched facts and user-confirmed references only.",
|
||||
"Image count must match explicit deliverables, not just numeric counts. Examples: front view + side view + details + color scheme = 4 images; three poster options = 3; one pure edit = 1.",
|
||||
"Count final independent image files, not internal panels, grid cells, or edit actions. A nine-grid character sheet, website long-page visual draft, or front-side-back board can still be one image if the user asked for one composed artifact.",
|
||||
"Image tasks must be concrete, ordered, and aligned with the count. For character/design work, separate views, detail sheets, palette/style sheets, and final composite boards when requested.",
|
||||
"Do not expose hidden chain-of-thought. The userFacingResponse is a short planning summary, not a long explanation.",
|
||||
"If intent is chat for clarification, userFacingResponse should be the exact concise questions to ask and imageCount must be 0.",
|
||||
"Use the user's language and locale automatically.",
|
||||
}, " "),
|
||||
},
|
||||
@@ -216,7 +232,8 @@ func (c *CreativeAgent) SummarizeImage(ctx context.Context, req design.AgentImag
|
||||
"You are a senior ecommerce creative director reviewing a just-generated image result.",
|
||||
"Write a concise, non-generic response in the user's language and locale.",
|
||||
"Never use a fixed version phrase like 'v5 出来了' unless the user explicitly asked for that version.",
|
||||
"Do not invent exact visual details you cannot see. Tie the critique to the prompt, likely ecommerce/page/composition goals, and give 2-4 concrete next actions.",
|
||||
"Do not invent exact visual details you cannot see. Tie the response to the prompt and canvas goal.",
|
||||
"When the user authorized your judgment or asked for an iteration, state the main change you attempted and what constraint it preserves. Mention a next step only when it is useful or required by the workflow.",
|
||||
"Sound like a professional art director, not a template.",
|
||||
}, " "),
|
||||
},
|
||||
@@ -236,21 +253,29 @@ func (c *CreativeAgent) BuildImagePrompt(ctx context.Context, req design.AgentIm
|
||||
"First distill the user request, current canvas context, recent conversation, memory, web research, and reference images into a compact visual intent summary; then write the exact standalone image-generation brief for GPT Image 2.",
|
||||
"Use short memory for the current working state and latest turn; use long memory for durable brand, style, product, and user preference decisions.",
|
||||
"When short memory conflicts with long memory, the latest explicit user instruction wins.",
|
||||
"Treat durable brand constraints as binding: product/category scope, mood/style preferences, logo/brand assets, color decisions, and future expansion requirements must carry into the brief unless the latest user instruction changes them.",
|
||||
"Preserve the user's own wording as the core of the final prompt. Add only requirements grounded in explicit user text, confirmed memory, current canvas artifacts, image generation settings, or tool constraints.",
|
||||
"Return strict JSON only: {\"title\":\"short localized artifact title\",\"prompt\":\"detailed image generation prompt\"}.",
|
||||
"The title must be content-specific, not a model name. Use the user's language and locale. Keep it under 18 CJK characters or 48 Latin characters.",
|
||||
"Use this proven prompt framework as a thinking guide, not a rigid template: task type, subject description, style definition, technical parameters, output specification.",
|
||||
"Only format the final prompt with explicit labels such as 任务类型/主体描述/风格定义/技术参数/输出规格 when that improves clarity. Do not force empty or irrelevant fields.",
|
||||
"Write directly in the user's language; Chinese prompts are preferred when the user writes Chinese. Do not translate Chinese requests into English by default.",
|
||||
"If the image must contain visible text, include the exact text content in 主体描述.",
|
||||
"Use 风格定义 to anchor references such as 参考 XX 风格 when the user gave a style reference.",
|
||||
"Use 风格定义 to anchor references such as 参考 XX 风格 only when the user gave that style reference or confirmed a searched reference.",
|
||||
"For multiple reference images, preserve the roles stated by the user such as product/source, logo, style, color, or composition. If roles were not stated, do not invent them in the prompt.",
|
||||
"For website/homepage visual drafts, include page modules, slogans, navigation labels, button copy, and body copy only if the user provided them or explicitly authorized you to decide. Do not silently add nav/hero/category/footer/copy just because they are common.",
|
||||
"When the user says previous version, last version, v2, v3, or similar, use only clear recent conversation, canvas title/order, or explicit user references to decide the source image. If the source is ambiguous, the planner should have asked instead of generating.",
|
||||
"If user feedback recognizes different versions by dimension, preserve those dimensions separately: for example, use v2 for overall direction and v3 for color only when the user said so.",
|
||||
"If the task edits an existing image, write the prompt as concrete change actions plus what must remain unchanged. If the task only uses a reference for style/color/composition, write it as a new generation task using the reference role from the user's words.",
|
||||
"The prompt must preserve only the relevant distilled context from previous canvas work, current brand/product/page direction, referenced images, and requested iteration.",
|
||||
"If web research results are available, use them as evidence and cite their visual implications inside the prompt without fabricating facts.",
|
||||
"If web research results are available, use factual context such as official brand/product/category facts. Do not describe visual traits from unconfirmed search images as if the user requested them.",
|
||||
"Never copy raw conversation history, raw canvas dumps, image URLs, storage URLs, JSON, IDs, coordinates, or internal event text into the final prompt.",
|
||||
"The final prompt sent to GPT Image 2 must be self-contained and compact; it should read like an art-director brief, not like application context.",
|
||||
"Do not add generic beautifiers, lens words, color guesses, or premium-style cliches merely to improve the image. Ask for clarification in the planning step when those choices are subjective and not authorized.",
|
||||
"Keep important subjects, products, furniture, people, text, logos, buttons, and decorative elements fully inside the visible frame with safe margins; avoid accidental edge cropping.",
|
||||
"Include ratio/resolution such as 3:4, 9:16, or 4K only when the user explicitly requested it. Otherwise do not invent output specs.",
|
||||
"Keep the final prompt generation-friendly: under 700 CJK characters or 1100 Latin characters, with one clear composition and no over-constrained checklist.",
|
||||
"Be concrete about composition, ecommerce hierarchy, art direction, typography, product treatment, colors, and what to avoid. Do not mention internal tools.",
|
||||
"Be concrete about composition, ecommerce hierarchy, typography, product treatment, colors, and what to avoid only when those choices are grounded in the request, memory, canvas, or confirmed references. Do not mention internal tools.",
|
||||
}, " "),
|
||||
},
|
||||
{
|
||||
@@ -377,7 +402,7 @@ func (c *CreativeAgent) DecideResearch(ctx context.Context, req design.AgentRese
|
||||
"Use search only when the task materially depends on current facts, public brand/product/site details, competitors, trends, market references, unfamiliar proper nouns, or an explicit request to research the web.",
|
||||
"Do not search for greetings, capability questions, pure canvas operations, ordinary image edits, or style refinements that can be handled from the canvas and conversation context.",
|
||||
"If searching, write one focused keyword query in the user's language when suitable; use English only when it will likely find stronger global results.",
|
||||
"The query must be your analyzed retrieval keywords, not the raw user prompt. Aim for design references: drafts, case studies, visual identity, design process, moodboards, competitors, and style systems.",
|
||||
"Keep named references and user style terms intact. Add only minimal retrieval terms such as official site, case study, visual identity, moodboard, lookbook, competitors, or style system; do not replace the user's words with your own aesthetic interpretation.",
|
||||
"For logo tasks, avoid logo-maker/tool results: include logo design case study, brand identity, visual identity, design process, moodboard, and negative terms such as -maker -generator -free -template when useful.",
|
||||
}, " "),
|
||||
},
|
||||
@@ -883,6 +908,7 @@ func researchContext(messages []design.Message) string {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
Content string `json:"content"`
|
||||
} `json:"results"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
@@ -894,19 +920,25 @@ func researchContext(messages []design.Message) string {
|
||||
}
|
||||
for _, result := range report.Results {
|
||||
title := strings.TrimSpace(result.Title)
|
||||
url := strings.TrimSpace(result.URL)
|
||||
snippet := strings.TrimSpace(result.Snippet)
|
||||
if title == "" && snippet == "" {
|
||||
content := strings.TrimSpace(result.Content)
|
||||
if title == "" && snippet == "" && content == "" {
|
||||
continue
|
||||
}
|
||||
title = truncateRunes(title, 120)
|
||||
snippet = truncateRunes(snippet, 160)
|
||||
content = truncateRunes(content, 1800)
|
||||
line := "- " + title
|
||||
if url != "" {
|
||||
line += " (" + url + ")"
|
||||
}
|
||||
if snippet != "" {
|
||||
line += ": " + snippet
|
||||
}
|
||||
lines = append(lines, line)
|
||||
if len(lines) >= 8 {
|
||||
break
|
||||
if content != "" {
|
||||
lines = append(lines, " page content: "+content)
|
||||
}
|
||||
}
|
||||
if report.Error != "" {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package piagent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
@@ -37,3 +38,33 @@ func TestAgentMessageContentSummaryDoesNotLeakImageURLs(t *testing.T) {
|
||||
t.Fatalf("expected compact inline text summary, got %d runes", utf8.RuneCountInString(summary))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchContextIncludesPageContentForEveryResult(t *testing.T) {
|
||||
payload, err := json.Marshal(design.ResearchReport{
|
||||
Kind: "web_search",
|
||||
Source: "test",
|
||||
Query: "Japanese bakery brand identity",
|
||||
Results: []design.ResearchResult{
|
||||
{Title: "Bakery case study", URL: "https://example.com/one", Snippet: "Warm handmade identity.", Content: "Detailed first webpage content about packaging, signage, and illustration."},
|
||||
{Title: "Brand identity notes", URL: "https://example.com/two", Snippet: "Soft visual system.", Content: "Detailed second webpage content about colors, typography, and bakery tone."},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
context := researchContext([]design.Message{
|
||||
{Role: "tool", Type: "tool", Title: "Synthesize Skills", Content: string(payload)},
|
||||
})
|
||||
for _, want := range []string{
|
||||
"Query: Japanese bakery brand identity",
|
||||
"https://example.com/one",
|
||||
"Detailed first webpage content",
|
||||
"https://example.com/two",
|
||||
"Detailed second webpage content",
|
||||
} {
|
||||
if !strings.Contains(context, want) {
|
||||
t.Fatalf("expected research context to include %q, got %s", want, context)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,9 +402,9 @@ func withReferenceImageInstruction(prompt string, imageCount int) string {
|
||||
return prompt
|
||||
}
|
||||
if prefersEnglish(prompt) {
|
||||
return "Input reference image(s) are attached. Use them as the visual source of truth for product shape, fabric, color, structure, and identity; when extracting, continuing, or editing a product, do not invent a different item.\n" + prompt
|
||||
return "Input reference image(s) are attached. Use each image only for the role stated by the user, such as product/source, logo, style, color, composition, or edit target. If editing or preserving a source product, keep its identity; do not invent unstated roles from the image content.\n" + prompt
|
||||
}
|
||||
return "已随请求附带参考图;生成时必须以参考图中的商品外形、材质、颜色、结构细节和身份为准,拆出、延续或编辑商品时不要另造不同商品。\n" + prompt
|
||||
return "已随请求附带参考图;只按用户原话说明的用途使用每张图,例如商品/源图、Logo、风格、色彩、构图或编辑对象。需要编辑或延续源商品时保持其身份,不要仅凭图片内容臆测未说明的角色。\n" + prompt
|
||||
}
|
||||
|
||||
func imagePromptTooLarge(prompt string) bool {
|
||||
|
||||
Reference in New Issue
Block a user