feat(agent): clarify vague briefs before generating and harden prompts

Let the planner return a clarify/chat intent that asks 2-4 targeted
questions (imageCount 0) when essentials are missing, instead of forging
ahead. Tighten prompt guidance to preserve user wording and stated
reference-image roles, avoid inventing copy/specs, keep durable brand
constraints, and treat vague taste feedback as clarification. Also
fix the generation heartbeat goroutine to stop cleanly via defer, add
research-query fallbacks, and refresh image-only capability copy.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-08 10:56:55 +08:00
parent 1a1c0a6544
commit ad67fe1ebe
5 changed files with 318 additions and 44 deletions
+88 -20
View File
@@ -1377,12 +1377,17 @@ func (s *DesignService) planAgentTask(ctx context.Context, project design.Projec
}
func normalizeAgentTaskPlan(plan design.AgentTaskPlan, prompt string, mode string, messages []design.AgentChatMessage, webSearchEnabled bool, forceImage bool) design.AgentTaskPlan {
plan.Intent = normalizeAgentIntent(plan.Intent)
if forceImage || shouldGenerateForAgentPrompt(prompt, mode, messages) {
rawIntent := strings.TrimSpace(plan.Intent)
plan.Intent = normalizeAgentIntent(rawIntent)
if forceImage || shouldForceImageMode(mode) {
plan.Intent = "image"
}
if plan.Intent == "" {
plan.Intent = "chat"
if shouldGenerateForAgentPrompt(prompt, mode, messages) {
plan.Intent = "image"
} else {
plan.Intent = "chat"
}
}
if !webSearchEnabled {
plan.ShouldSearch = false
@@ -1434,9 +1439,9 @@ func normalizeAgentTaskPlan(plan design.AgentTaskPlan, prompt string, mode strin
func fallbackPlanningResponse(prompt string, intent string) string {
if normalizeAgentIntent(intent) == "image" {
if prefersEnglishText(prompt) {
return "I will break this into the right visual deliverables and generate them on the canvas."
return "I will turn this into concrete image deliverables and generate them on the canvas."
}
return "我会先把需求拆成合适的视觉交付项,再在画布中生成。"
return "我会先把需求拆成具体图片交付项,再在画布中生成。"
}
if prefersEnglishText(prompt) {
return "I will answer based on the current canvas context."
@@ -1573,9 +1578,9 @@ func (s *DesignService) updateAgentMessage(ctx context.Context, projectID string
func fallbackAgentChatResponse(prompt string, project design.Project) string {
if isCapabilityPrompt(strings.ToLower(strings.TrimSpace(prompt))) {
if prefersEnglishText(prompt) {
return "I mainly help with image and design-canvas work: generating visuals, editing images, shaping ecommerce pages, reading the current canvas, and suggesting the next concrete move. When you give me a clear visual task, I will enter the image-generation pipeline."
return "I focus on image generation and image editing: brand/site visuals, ecommerce product images, posters, banners, social graphics, reference-based generation, background removal, vectorization, mockup placement, and multi-round canvas iteration. Give me a clear visual task and I will generate or edit images on the canvas."
}
return "我主要负责图片与设计画布:生成视觉稿、编辑图片、做电商页面方向、拆解当前画布并给下一步建议。你给我明确的作图目标时,我进入生图链路。"
return "我专注图片生成和图片编辑:品牌/官网视觉、电商产品图、海报 Banner、社媒图、参考图生图、去背景、矢量化、Mockup 贴图,以及在无限画布里多轮迭代。你给我明确的图片任务后,我进入生图或修图链路。"
}
if strings.TrimSpace(project.Brief) != "" {
if prefersEnglishText(prompt) {
@@ -1594,7 +1599,9 @@ func normalizeAgentIntent(intent string) string {
switch intent {
case "image", "generate_image", "edit_image", "visual", "design":
return "image"
case "research":
case "":
return ""
case "research", "clarify", "brief", "briefing", "question", "questions", "chat", "conversation":
return "chat"
default:
return "chat"
@@ -2571,7 +2578,9 @@ func (s *DesignService) completeCreateProjectGeneration(ctx context.Context, pro
message.Type = "assistant"
}
if incremental {
project.Nodes = mergeCompletedIncrementalGeneratedNodes(project.Nodes, plan.GeneratedNodes, "")
project.Nodes = clearRemainingAgentGenerationPlaceholders(project.Nodes, "")
project.Messages = appendMissingIncrementalArtifactMessages(project.Messages, threadID, prompt, plan.GeneratedNodes, now.Add(-time.Millisecond))
} else {
project.Nodes = mergeGeneratedNodesIntoTarget(nonGeneratingNodes(project.Nodes), plan.GeneratedNodes, "")
project.Messages = append(project.Messages, toolArtifactsMessage(threadID, prompt, plan.GeneratedNodes, now.Add(-time.Millisecond)))
@@ -2940,7 +2949,11 @@ func (s *DesignService) completeFollowupGeneration(ctx context.Context, projectI
message.Type = "assistant"
}
if incremental {
project.Nodes = mergeCompletedIncrementalGeneratedNodes(project.Nodes, plan.GeneratedNodes, targetNodeID)
project.Nodes = clearRemainingAgentGenerationPlaceholders(project.Nodes, targetNodeID)
if !isolated {
project.Messages = appendMissingIncrementalArtifactMessages(project.Messages, threadID, prompt, plan.GeneratedNodes, now.Add(-time.Millisecond))
}
} else {
baseNodes := nonGeneratingNodes(project.Nodes)
if targetNodeID != "" {
@@ -2986,7 +2999,15 @@ func (s *DesignService) planWithIncrementalImages(ctx context.Context, projectID
done := make(chan struct{})
if appendArtifacts {
go s.emitGenerationHeartbeat(ctx, projectID, threadID, req.Prompt, done)
heartbeatStopped := make(chan struct{})
go func() {
defer close(heartbeatStopped)
s.emitGenerationHeartbeat(ctx, projectID, threadID, req.Prompt, done)
}()
defer func() {
close(done)
<-heartbeatStopped
}()
}
var mu sync.Mutex
persistedNodes := make([]design.Node, 0, plannedGenerationResultCapacity(req))
@@ -3004,9 +3025,6 @@ func (s *DesignService) planWithIncrementalImages(ctx context.Context, projectID
persistedNodes = append(persistedNodes, node)
return s.applyIncrementalGeneratedNode(ctx, projectID, threadID, prompt, targetNodeID, node, appendArtifacts)
})
if appendArtifacts {
close(done)
}
if len(persistedNodes) > 0 {
plan.GeneratedNodes = append([]design.Node(nil), persistedNodes...)
}
@@ -3039,6 +3057,32 @@ func completedGeneratedImageNodes(nodes []design.Node) []design.Node {
return completed
}
func mergeCompletedIncrementalGeneratedNodes(nodes []design.Node, generated []design.Node, targetNodeID string) []design.Node {
completed := completedGeneratedImageNodes(generated)
if len(completed) == 0 {
return nodes
}
if strings.TrimSpace(targetNodeID) != "" {
return mergeGeneratedNodesIntoTarget(nodes, completed, targetNodeID)
}
for _, node := range completed {
nodes = upsertGeneratedNode(nodes, node)
}
return nodes
}
func appendMissingIncrementalArtifactMessages(messages []design.Message, threadID string, prompt string, nodes []design.Node, createdAt time.Time) []design.Message {
existing := generatedImageArtifactsByID(messages)
for _, node := range completedGeneratedImageNodes(nodes) {
if _, ok := existing[node.ID]; ok {
continue
}
messages = append(messages, toolArtifactsMessage(threadID, prompt, []design.Node{node}, createdAt))
existing[node.ID] = design.GeneratorTaskArtifact{}
}
return messages
}
func plannedGenerationResultCapacity(req design.AgentRequest) int {
count := req.TaskPlan.ImageCount
if count <= 0 {
@@ -3237,10 +3281,7 @@ func (s *DesignService) maybeAddResearchStep(ctx context.Context, projectID stri
Messages: messages,
Memory: buildAgentMemory(project, prompt, mode, messages),
})
if !decision.ShouldSearch {
return nil
}
query := strings.TrimSpace(decision.Query)
query := enabledResearchQuery(prompt, mode, decision)
if query == "" {
return nil
}
@@ -3254,16 +3295,38 @@ func (s *DesignService) maybeAddPlannedResearchStep(ctx context.Context, project
if strings.TrimSpace(plan.Intent) == "" {
return s.maybeAddResearchStep(ctx, projectID, threadID, project, prompt, mode, messages, enabled)
}
if !plan.ShouldSearch {
return nil
}
query := strings.TrimSpace(plan.SearchQuery)
if query == "" && plan.ShouldSearch {
query = fallbackResearchQuery(prompt, mode)
}
if query == "" {
return nil
}
return s.addResearchStep(ctx, projectID, threadID, query)
}
func enabledResearchQuery(prompt string, mode string, decision design.AgentResearchDecision) string {
if strings.TrimSpace(decision.Query) != "" {
return strings.TrimSpace(decision.Query)
}
if !shouldSearchWhenEnabled(prompt, mode) {
return ""
}
return fallbackResearchQuery(prompt, mode)
}
func shouldSearchWhenEnabled(prompt string, mode string) bool {
text := strings.ToLower(strings.TrimSpace(prompt))
if text == "" || isConversationalPrompt(text) {
return false
}
modeText := strings.ToLower(strings.TrimSpace(mode))
if strings.Contains(modeText, "image-action") {
return shouldRunResearchFallback(prompt, mode)
}
return true
}
func (s *DesignService) decideResearch(ctx context.Context, req design.AgentResearchDecisionRequest) design.AgentResearchDecision {
if decider, ok := s.creativeAgent.(design.ResearchDecider); ok {
if decision, err := decider.DecideResearch(ctx, req); err == nil {
@@ -3366,6 +3429,11 @@ func shouldGenerateForAgentPrompt(prompt string, mode string, messages []design.
return false
}
func shouldForceImageMode(mode string) bool {
modeText := strings.ToLower(strings.TrimSpace(mode))
return strings.Contains(modeText, "image-action") || modeText == "image" || modeText == "pure-image"
}
func hasAgentImageInput(messages []design.AgentChatMessage) bool {
for _, message := range messages {
for _, content := range message.Contents {
@@ -5692,7 +5760,7 @@ func inspirationItems() []design.InspirationItem {
return []design.InspirationItem{
{ID: "luxury-spring", Title: "2026 春季系列设计趋势", Category: "Luxury Showcase", Description: "强对比摄影、低饱和色块与清晰购买动线。", Accent: "#111827"},
{ID: "natural-ui", Title: "极简自然主义 UI", Category: "Mobile App", Description: "玻璃拟态信息卡与植物识别流程。", Accent: "#6b8f71"},
{ID: "blue-geometry", Title: "蓝色 3D 几何广告", Category: "Campaign", Description: "冷感材质、漂浮图形和高亮 CTA。", Accent: "#2563eb"},
{ID: "blue-geometry", Title: "蓝色立体几何广告", Category: "Campaign", Description: "冷感材质、漂浮图形和高亮 CTA。", Accent: "#2563eb"},
{ID: "brand-kit", Title: "天然护肤品牌板", Category: "Brand Kit", Description: "字体、插画、包装和详情页模块统一沉淀。", Accent: "#c8a568"},
}
}
@@ -319,6 +319,37 @@ func (a incrementalCreateProjectAgent) Replay(ctx context.Context, project desig
return design.AgentReplay{}, nil
}
type staleOverwriteIncrementalCreateProjectAgent struct {
repo design.Repository
t *testing.T
}
func (a staleOverwriteIncrementalCreateProjectAgent) Plan(ctx context.Context, req design.AgentRequest) (design.AgentPlan, error) {
return design.AgentPlan{}, errors.New("expected incremental generation")
}
func (a staleOverwriteIncrementalCreateProjectAgent) PlanIncremental(ctx context.Context, req design.AgentRequest, onResult func(design.AgentImageGenerationResult) error) (design.AgentPlan, error) {
if len(req.Project.Nodes) == 0 {
a.t.Fatal("expected planned placeholder before image generation")
}
staleProject := req.Project
node := design.Node{ID: req.Project.Nodes[0].ID, Type: design.NodeTypeImage, Title: "盲盒IP视觉体系", Content: "https://example.com/blind-box-system.png", Status: "success", Width: 1024, Height: 1024}
if err := onResult(design.AgentImageGenerationResult{Index: 0, Node: node}); err != nil {
return design.AgentPlan{}, err
}
if err := a.repo.Save(ctx, staleProject); err != nil {
return design.AgentPlan{}, err
}
return design.AgentPlan{
Message: design.Message{Role: "assistant", Type: "assistant", Title: "图片生成完成", Content: "盲盒IP视觉体系已生成。"},
GeneratedNodes: []design.Node{node},
}, nil
}
func (a staleOverwriteIncrementalCreateProjectAgent) Replay(ctx context.Context, project design.Project) (design.AgentReplay, error) {
return design.AgentReplay{}, nil
}
type partialFailingIncrementalCreateProjectAgent struct{}
func (a partialFailingIncrementalCreateProjectAgent) Plan(ctx context.Context, req design.AgentRequest) (design.AgentPlan, error) {
@@ -541,6 +572,54 @@ func TestAgentChatCreatesPlaceholdersFromModelPlannedImageTasks(t *testing.T) {
}
}
func TestAgentChatAllowsPlannerToClarifyVisualBriefBeforeGenerating(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
queue := &fakeJobQueue{}
service := NewDesignService(repo, nil, nil, nil, nil, fakeCreativeChatAgent{
decision: design.AgentDecision{Intent: "image", Reason: "visual request"},
taskPlan: design.AgentTaskPlan{
Intent: "clarify",
UserFacingResponse: "开始生图前我需要确认:Logo 是否已有?主视觉偏大胆色彩还是极简?首页要优先展示哪些品类?",
ImageCount: 0,
},
response: "开始生图前我需要确认:Logo 是否已有?主视觉偏大胆色彩还是极简?首页要优先展示哪些品类?",
})
service.SetJobQueue(queue)
project := design.Project{
ID: "project-clarify-before-generate",
Title: "Clarify before generate",
Status: design.StatusReady,
UpdatedAt: time.Now(),
}
if err := repo.Save(ctx, project); err != nil {
t.Fatal(err)
}
thread, err := service.AgentChat(ctx, project.ID, design.AgentChatRequest{
Messages: []design.AgentChatMessage{
{
Role: "user",
Contents: []design.AgentContent{
{Type: "text", Text: "still day 品牌官网设计,主打 mood 配色和舒适针织。"},
},
},
},
})
if err != nil {
t.Fatal(err)
}
if len(thread.Project.Nodes) != 0 {
t.Fatalf("expected no image placeholders while clarifying, got %#v", thread.Project.Nodes)
}
if len(queue.jobs) != 1 || queue.jobs[0].Kind != design.JobAgentConversation {
t.Fatalf("expected clarification to enqueue chat job, got %#v", queue.jobs)
}
if queue.jobs[0].Conversation.TaskPlan.Intent != "chat" {
t.Fatalf("expected clarification plan normalized to chat, got %#v", queue.jobs[0].Conversation.TaskPlan)
}
}
func TestCreateProjectAsyncReturnsBeforeTaskPlanning(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
@@ -702,6 +781,62 @@ func TestCreateProjectGenerationStreamsEachImageAsItFinishes(t *testing.T) {
}
}
func TestCreateProjectGenerationFinalizesAfterStaleHeartbeatOverwrite(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
creative := fakeCreativeChatAgent{
decision: design.AgentDecision{Intent: "image"},
taskPlan: design.AgentTaskPlan{
Intent: "image",
UserFacingResponse: "我将生成一张盲盒IP视觉体系图。",
ImageCount: 1,
ImageTasks: []design.AgentImageTask{
{Title: "盲盒IP视觉体系", Brief: "角色、包装、展示台、贴纸和发售海报总览"},
},
},
}
service := NewDesignService(repo, staleOverwriteIncrementalCreateProjectAgent{repo: repo, t: t}, nil, nil, nil, creative)
service.SetJobQueue(&fakeJobQueue{})
project, err := service.CreateProjectAsync(ctx, "", "设计一张盲盒IP视觉体系。", "gpt-image-2", "", false)
if err != nil {
t.Fatal(err)
}
if err := service.ProcessJob(ctx, design.Job{
Kind: design.JobCreateProjectGeneration,
ProjectID: project.ID,
UserID: project.UserID,
ThreadID: project.LastThreadID,
Prompt: project.Brief,
}); err != nil {
t.Fatal(err)
}
saved, err := repo.Get(ctx, project.ID)
if err != nil {
t.Fatal(err)
}
if saved.Status != design.StatusReady {
t.Fatalf("expected project ready after stale heartbeat overwrite, got %s", saved.Status)
}
if len(saved.Nodes) != 1 || saved.Nodes[0].Status != "success" || saved.Nodes[0].Content != "https://example.com/blind-box-system.png" {
t.Fatalf("expected final generated node restored, got %#v", saved.Nodes)
}
foundToolSuccess := false
foundArtifact := false
for _, message := range saved.Messages {
if message.Title == "GPT Image 2" && message.ThreadID == project.LastThreadID && message.Status == "success" {
foundToolSuccess = true
}
if message.Name == "generate_media" && strings.Contains(message.Content, "blind-box-system.png") {
foundArtifact = true
}
}
if !foundToolSuccess || !foundArtifact {
t.Fatalf("expected final tool success and restored artifact message, got %#v", saved.Messages)
}
}
func TestCreateProjectGenerationPartialFailurePreservesSuccessfulImagesAndClosesTask(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
@@ -1147,7 +1282,7 @@ func TestGenerationStepHeartbeatUpdatesLatestToolMessage(t *testing.T) {
}
}
func TestWebSearchRunsOnlyWhenModelDecisionNeedsIt(t *testing.T) {
func TestWebSearchRunsWhenEnabledEvenIfModelSkips(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
project := design.Project{
@@ -1172,13 +1307,18 @@ func TestWebSearchRunsOnlyWhenModelDecisionNeedsIt(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if searcher.called {
t.Fatal("expected web search to stay off when model decision says no")
if !searcher.called {
t.Fatal("expected enabled web search to run even when model decision says no")
}
if searcher.query != fallbackResearchQuery("优化当前首页视觉", "design") {
t.Fatalf("expected fallback search query, got %q", searcher.query)
}
if agent.memory.ShortTerm == "" || agent.memory.LongTerm == "" {
t.Fatalf("expected short and long memory in research decision, got %#v", agent.memory)
}
searcher.called = false
searcher.query = ""
agent.decision = design.AgentResearchDecision{ShouldSearch: true, Query: "still day ecommerce homepage reference", Reason: "needs external references"}
err = service.maybeAddResearchStep(ctx, project.ID, "thread-1", project, "优化当前首页视觉", "design", nil, true)
if err != nil {
@@ -1262,7 +1402,7 @@ func TestAgentChatUsesPlannerSearchQueryInsteadOfRawPrompt(t *testing.T) {
}
}
func TestInitialWebSearchDoesNotFallbackToRawPromptWhenModelSkips(t *testing.T) {
func TestInitialWebSearchFallsBackToSearchQueryWhenModelSkips(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
threadID := "thread-initial-search"
@@ -1288,15 +1428,18 @@ func TestInitialWebSearchDoesNotFallbackToRawPromptWhenModelSkips(t *testing.T)
if err != nil {
t.Fatal(err)
}
if searcher.called {
t.Fatalf("expected no raw-prompt fallback search, got query=%q", searcher.query)
if !searcher.called {
t.Fatal("expected enabled initial web search to run")
}
if searcher.query != fallbackResearchQuery("做一个美漫风格分镜", "design") {
t.Fatalf("expected fallback search query, got %q", searcher.query)
}
saved, err := repo.Get(ctx, project.ID)
if err != nil {
t.Fatal(err)
}
if len(saved.Messages) != len(project.Messages) {
t.Fatalf("expected no research message, got %#v", saved.Messages)
if len(saved.Messages) == len(project.Messages) || saved.Messages[len(saved.Messages)-1].Title != "Synthesize Skills" {
t.Fatalf("expected research message, got %#v", saved.Messages)
}
}