root
44406b72db
Moteva-style AI design workbench replica. Home prompt creates a project; the project page provides an infinite canvas with node drag, zoom/pan, a design chat panel, history replay, image asset upload, and project save/regenerate. - Backend: go-zero, DDD layering, sqlc/pgx, optional PostgreSQL; memory or Redis cache; asynq job queue; MinIO/S3/R2/OSS object storage; sky-valley/pi agent runtime adapter. - Frontend: Next.js App Router + Vite artifact build, TypeScript, i18n, shadcn/ui components, auth (OTP/Turnstile/Google/WeChat). - Deploy: Docker Compose and k3s manifests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
982 lines
37 KiB
Go
982 lines
37 KiB
Go
package piagent
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"img_infinite_canvas/internal/domain/design"
|
|
)
|
|
|
|
type CreativeAgentOptions struct {
|
|
BaseURL string
|
|
APIKey string
|
|
Model string
|
|
TimeoutSeconds int
|
|
}
|
|
|
|
type CreativeAgent struct {
|
|
baseURL string
|
|
apiKey string
|
|
model string
|
|
client *http.Client
|
|
}
|
|
|
|
func NewCreativeAgent(opts CreativeAgentOptions) *CreativeAgent {
|
|
if strings.TrimSpace(opts.BaseURL) == "" || strings.TrimSpace(opts.APIKey) == "" {
|
|
return nil
|
|
}
|
|
timeout := time.Duration(opts.TimeoutSeconds) * time.Second
|
|
if timeout <= 0 {
|
|
timeout = 300 * time.Second
|
|
}
|
|
model := strings.TrimSpace(opts.Model)
|
|
if model == "" {
|
|
model = "gpt-5.5"
|
|
}
|
|
|
|
return &CreativeAgent{
|
|
baseURL: strings.TrimRight(opts.BaseURL, "/"),
|
|
apiKey: opts.APIKey,
|
|
model: model,
|
|
client: &http.Client{Timeout: timeout},
|
|
}
|
|
}
|
|
|
|
func (c *CreativeAgent) Decide(ctx context.Context, req design.AgentDecisionRequest) (design.AgentDecision, error) {
|
|
content, err := c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are the routing brain of a Moteva-style creative production agent.",
|
|
"Decide whether the user wants the agent to call the image generation/editing pipeline or just answer in chat.",
|
|
"Return strict JSON only: {\"intent\":\"image\"|\"chat\",\"reason\":\"short reason\"}.",
|
|
"Use image only for explicit visual production, image editing, ecommerce design output, page/banner/poster/product/mockup generation, or iterative visual changes.",
|
|
"Use chat for greetings, thanks, capability questions, vague conversation, or clarification without a concrete visual action.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildDecisionPrompt(req),
|
|
},
|
|
}, 220, 0.1, true)
|
|
if err != nil {
|
|
return design.AgentDecision{}, err
|
|
}
|
|
|
|
var parsed struct {
|
|
Intent string `json:"intent"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
if err := json.Unmarshal([]byte(extractJSONObject(content)), &parsed); err != nil {
|
|
return design.AgentDecision{}, err
|
|
}
|
|
return design.AgentDecision{Intent: parsed.Intent, Reason: parsed.Reason}, nil
|
|
}
|
|
|
|
func (c *CreativeAgent) Respond(ctx context.Context, req design.AgentConversationRequest) (string, error) {
|
|
return c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are a top-tier ecommerce art director and Moteva-style creative production agent.",
|
|
"Answer naturally in the user's language and locale, like a senior creative partner who understands image generation, ecommerce pages, product visuals, and canvas workflows.",
|
|
"Use short memory for the current turn and canvas state; use long memory for durable project preferences and prior decisions.",
|
|
"Do not use canned templates. Do not claim you generated an image unless an image tool actually ran.",
|
|
"Do not mention unsupported video, audio, or 3D features. This product currently supports image generation/editing and infinite-canvas design workflows.",
|
|
"For greetings, reply briefly and invite a concrete next step. For capability questions, explain useful image/design capabilities with current canvas context.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildConversationPrompt(req),
|
|
},
|
|
}, 900, 0.65, false)
|
|
}
|
|
|
|
func (c *CreativeAgent) StreamRespond(ctx context.Context, req design.AgentConversationRequest, onDelta func(delta string) error) (string, error) {
|
|
return c.chatStream(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are a top-tier ecommerce art director and Moteva-style creative production agent.",
|
|
"Answer naturally in the user's language and locale, like a senior creative partner who understands image generation, ecommerce pages, product visuals, and canvas workflows.",
|
|
"Use short memory for the current turn and canvas state; use long memory for durable project preferences and prior decisions.",
|
|
"Do not use canned templates. Do not claim you generated an image unless an image tool actually ran.",
|
|
"Do not mention unsupported video, audio, or 3D features. This product currently supports image generation/editing and infinite-canvas design workflows.",
|
|
"For greetings, reply briefly and invite a concrete next step. For capability questions, explain useful image/design capabilities with current canvas context.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildConversationPrompt(req),
|
|
},
|
|
}, 900, 0.65, onDelta)
|
|
}
|
|
|
|
func (c *CreativeAgent) StreamPlanNarration(ctx context.Context, req design.AgentTaskPlanRequest, onDelta func(delta string) error) (string, error) {
|
|
return c.chatStream(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You write the visible live thinking text for a Moteva-style creative agent.",
|
|
"This is not hidden chain-of-thought. Do not reveal private reasoning, step-by-step deliberation, policies, or tool internals.",
|
|
"Write one concise localized sentence that tells the user what the agent is preparing to do next.",
|
|
"If web research results are present, mention that references have been collected and how they will guide the visual direction.",
|
|
"Do not use markdown, numbering, or generic canned wording. Use the user's language and locale automatically.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": strings.Join([]string{
|
|
buildAgentTaskPlanPrompt(req),
|
|
"",
|
|
"Write only the live text shown below the “Thinking...” label.",
|
|
}, "\n"),
|
|
},
|
|
}, 260, 0.45, onDelta)
|
|
}
|
|
|
|
func (c *CreativeAgent) PlanAgentTask(ctx context.Context, req design.AgentTaskPlanRequest) (design.AgentTaskPlan, error) {
|
|
content, err := c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are the automatic planner for a Moteva-style creative agent.",
|
|
"Read the user's request, canvas, inline references, short memory, and long memory, then decide the next workflow.",
|
|
"Return strict JSON only: {\"intent\":\"image\"|\"chat\",\"reason\":\"short internal reason\",\"userFacingResponse\":\"one concise localized sentence explaining what you will do\",\"shouldSearch\":true|false,\"searchQuery\":\"query or empty\",\"searchReason\":\"short reason\",\"imageCount\":1-10,\"imageTasks\":[{\"title\":\"localized short title\",\"brief\":\"specific output brief\"}]}",
|
|
"Intent image means the user wants visual production, image editing, ecommerce/page/poster/product/character/illustration/output generation, or concrete canvas creation.",
|
|
"Intent chat means greetings, thanks, vague discussion, capability questions, or missing critical information.",
|
|
"If web search is disabled, shouldSearch must be false. If enabled, choose search only when external references will improve the work: public brand/product facts, current trends, competitor/style references, niche cultural or visual references, or explicit research/search requests.",
|
|
"When shouldSearch is true, searchQuery must be your analyzed search keywords, not the raw user prompt. Keep it focused on reference material such as design drafts, case studies, visual identity, design process, moodboards, and style references.",
|
|
"For logo tasks, avoid logo-maker/tool results: include terms like logo design case study, brand identity, visual identity, design process, moodboard, and negative terms such as -maker -generator -free -template when useful.",
|
|
"Image count must match explicit deliverables, not just numeric counts. Examples: front view + side view + details + color scheme = 4 images; three poster options = 3; one pure edit = 1.",
|
|
"Image tasks must be concrete, ordered, and aligned with the count. For character/design work, separate views, detail sheets, palette/style sheets, and final composite boards when requested.",
|
|
"Do not expose hidden chain-of-thought. The userFacingResponse is a short planning summary, not a long explanation.",
|
|
"Use the user's language and locale automatically.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildAgentTaskPlanPrompt(req),
|
|
},
|
|
}, 900, 0.2, true)
|
|
if err != nil {
|
|
return design.AgentTaskPlan{}, err
|
|
}
|
|
|
|
var parsed struct {
|
|
Intent string `json:"intent"`
|
|
Reason string `json:"reason"`
|
|
UserFacingResponse string `json:"userFacingResponse"`
|
|
ShouldSearch bool `json:"shouldSearch"`
|
|
SearchQuery string `json:"searchQuery"`
|
|
SearchReason string `json:"searchReason"`
|
|
ImageCount int `json:"imageCount"`
|
|
ImageTasks []struct {
|
|
Title string `json:"title"`
|
|
Brief string `json:"brief"`
|
|
} `json:"imageTasks"`
|
|
}
|
|
if err := json.Unmarshal([]byte(extractJSONObject(content)), &parsed); err != nil {
|
|
return design.AgentTaskPlan{}, err
|
|
}
|
|
tasks := make([]design.AgentImageTask, 0, len(parsed.ImageTasks))
|
|
for _, task := range parsed.ImageTasks {
|
|
title := strings.TrimSpace(task.Title)
|
|
brief := strings.TrimSpace(task.Brief)
|
|
if title == "" && brief == "" {
|
|
continue
|
|
}
|
|
tasks = append(tasks, design.AgentImageTask{Title: title, Brief: brief})
|
|
}
|
|
return design.AgentTaskPlan{
|
|
Intent: strings.TrimSpace(parsed.Intent),
|
|
Reason: strings.TrimSpace(parsed.Reason),
|
|
UserFacingResponse: strings.TrimSpace(parsed.UserFacingResponse),
|
|
ShouldSearch: parsed.ShouldSearch,
|
|
SearchQuery: strings.TrimSpace(parsed.SearchQuery),
|
|
SearchReason: strings.TrimSpace(parsed.SearchReason),
|
|
ImageCount: parsed.ImageCount,
|
|
ImageTasks: tasks,
|
|
}, nil
|
|
}
|
|
|
|
func (c *CreativeAgent) SummarizeImage(ctx context.Context, req design.AgentImageSummaryRequest) (string, error) {
|
|
return c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are a senior ecommerce creative director reviewing a just-generated image result.",
|
|
"Write a concise, non-generic response in the user's language and locale.",
|
|
"Never use a fixed version phrase like 'v5 出来了' unless the user explicitly asked for that version.",
|
|
"Do not invent exact visual details you cannot see. Tie the critique to the prompt, likely ecommerce/page/composition goals, and give 2-4 concrete next actions.",
|
|
"Sound like a professional art director, not a template.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildImageSummaryPrompt(req),
|
|
},
|
|
}, 900, 0.7, false)
|
|
}
|
|
|
|
func (c *CreativeAgent) BuildImagePrompt(ctx context.Context, req design.AgentImagePromptRequest) (design.AgentImagePrompt, error) {
|
|
content, err := c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are the harness planner for a Moteva-style ecommerce image agent.",
|
|
"First distill the user request, current canvas context, recent conversation, memory, web research, and reference images into a compact visual intent summary; then write the exact standalone image-generation brief for GPT Image 2.",
|
|
"Use short memory for the current working state and latest turn; use long memory for durable brand, style, product, and user preference decisions.",
|
|
"When short memory conflicts with long memory, the latest explicit user instruction wins.",
|
|
"Return strict JSON only: {\"title\":\"short localized artifact title\",\"prompt\":\"detailed image generation prompt\"}.",
|
|
"The title must be content-specific, not a model name. Use the user's language and locale. Keep it under 18 CJK characters or 48 Latin characters.",
|
|
"Use this proven prompt framework as a thinking guide, not a rigid template: task type, subject description, style definition, technical parameters, output specification.",
|
|
"Only format the final prompt with explicit labels such as 任务类型/主体描述/风格定义/技术参数/输出规格 when that improves clarity. Do not force empty or irrelevant fields.",
|
|
"Write directly in the user's language; Chinese prompts are preferred when the user writes Chinese. Do not translate Chinese requests into English by default.",
|
|
"If the image must contain visible text, include the exact text content in 主体描述.",
|
|
"Use 风格定义 to anchor references such as 参考 XX 风格 when the user gave a style reference.",
|
|
"The prompt must preserve only the relevant distilled context from previous canvas work, current brand/product/page direction, referenced images, and requested iteration.",
|
|
"If web research results are available, use them as evidence and cite their visual implications inside the prompt without fabricating facts.",
|
|
"Never copy raw conversation history, raw canvas dumps, image URLs, storage URLs, JSON, IDs, coordinates, or internal event text into the final prompt.",
|
|
"The final prompt sent to GPT Image 2 must be self-contained and compact; it should read like an art-director brief, not like application context.",
|
|
"Keep important subjects, products, furniture, people, text, logos, buttons, and decorative elements fully inside the visible frame with safe margins; avoid accidental edge cropping.",
|
|
"Include ratio/resolution such as 3:4, 9:16, or 4K only when the user explicitly requested it. Otherwise do not invent output specs.",
|
|
"Keep the final prompt generation-friendly: under 700 CJK characters or 1100 Latin characters, with one clear composition and no over-constrained checklist.",
|
|
"Be concrete about composition, ecommerce hierarchy, art direction, typography, product treatment, colors, and what to avoid. Do not mention internal tools.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildImagePromptPlannerInput(req),
|
|
},
|
|
}, 1300, 0.55, true)
|
|
if err != nil {
|
|
return design.AgentImagePrompt{}, err
|
|
}
|
|
|
|
var parsed struct {
|
|
Title string `json:"title"`
|
|
Prompt string `json:"prompt"`
|
|
}
|
|
if err := json.Unmarshal([]byte(extractJSONObject(content)), &parsed); err != nil {
|
|
return design.AgentImagePrompt{}, err
|
|
}
|
|
parsed.Title = strings.TrimSpace(parsed.Title)
|
|
parsed.Prompt = compactImageGenerationPrompt(parsed.Prompt, imageGenerationPromptMaxRunes)
|
|
if parsed.Prompt == "" {
|
|
return design.AgentImagePrompt{}, fmt.Errorf("creative agent returned empty image prompt")
|
|
}
|
|
return design.AgentImagePrompt{Title: parsed.Title, Prompt: parsed.Prompt}, nil
|
|
}
|
|
|
|
func (c *CreativeAgent) VectorizeImage(ctx context.Context, req design.ImageVectorizationRequest) (design.ImageVectorization, error) {
|
|
imageDataURL := strings.TrimSpace(req.ImageDataURL)
|
|
if imageDataURL == "" {
|
|
return design.ImageVectorization{}, fmt.Errorf("vectorize image is required")
|
|
}
|
|
content, err := c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are an expert SVG vectorization engine for a design canvas.",
|
|
"Look at the supplied image and recreate it as compact, clean SVG code.",
|
|
"Return strict JSON only: {\"svg\":\"<svg ...>...</svg>\",\"width\":1024,\"height\":1024}.",
|
|
"The SVG must be self-contained and scalable: use <path>, <rect>, <circle>, <ellipse>, <polygon>, <line>, <polyline>, <text>, gradients, masks, and groups when useful.",
|
|
"Do not embed raster images. Do not use <image>, base64 data, external URLs, scripts, foreignObject, CSS imports, or HTML.",
|
|
"Keep transparent backgrounds transparent unless the source has an intentional filled background.",
|
|
"Preserve the visible subject silhouette, logo/text layout, color blocks, strokes, and readable details. Favor crisp geometry over photorealistic texture.",
|
|
"Set xmlns, width, height, and viewBox. Use width/height matching the requested dimensions when provided.",
|
|
"Keep the SVG reasonably compact while still faithful enough for download and canvas display.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": []map[string]any{
|
|
{
|
|
"type": "text",
|
|
"text": buildVectorizePrompt(req),
|
|
},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": map[string]any{
|
|
"url": imageDataURL,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}, 16000, 0.1, true)
|
|
if err != nil {
|
|
return design.ImageVectorization{}, err
|
|
}
|
|
return parseVectorizeResponse(content, req)
|
|
}
|
|
|
|
func (c *CreativeAgent) AnalyzeMockupModel(ctx context.Context, req design.MockupModelAnalysisRequest) (design.MockupModelAnalysis, error) {
|
|
if strings.TrimSpace(req.ImageDataURL) == "" {
|
|
return design.MockupModelAnalysis{}, fmt.Errorf("mockup model image is required")
|
|
}
|
|
content, err := c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are a visual mockup geometry model generator.",
|
|
"Analyze the image and return the raw numeric model parameters that the canvas renderer will use to place a print/design onto a real surface.",
|
|
"Return strict JSON only. Do not return markdown. Do not invent blob URLs, dataId values, or binary files.",
|
|
"All of these fields must come from your visual estimate of the image: depthMap, shadeMap, segmentationMap, xOffsetMap, yOffsetMap, and cameraIntrinsics.",
|
|
"Use compact raw arrays so the response can be sent directly to the frontend: depthMap, shadeMap, xOffsetMap, yOffsetMap must be 16x16; segmentationMap must be 32x32.",
|
|
"Array lengths are strict: every 16x16 map must contain exactly 256 numeric values and segmentationMap must contain exactly 1024 numeric values. Never use ellipses, omitted ranges, comments, or shortened arrays.",
|
|
"depthMap and shadeMap data values are floats 0..1. segmentationMap data values are 0 outside and 255 on the printable surface, with soft edge values allowed. xOffsetMap and yOffsetMap values are floats -1..1.",
|
|
"Pick the usable printable/decal surface, not the existing artwork bounds: for apparel this is the main front print zone below neckline/drawstrings and above pocket/hem, large enough for portrait or landscape artwork. Do not choose only the current logo/text letters, the whole hoodie, sleeve, pocket, model body, or background. For bags/packages/screens choose the actual flat design panel, not the whole product silhouette.",
|
|
"placement, surface.polygon, segmentationMap, depthMap, shadeMap, xOffsetMap, and yOffsetMap must refer to the same normalized target-image coordinate system. cameraIntrinsics principal point cx/cy must be normalized 0..1, usually 0.5.",
|
|
"Return shape: {\"placement\":{\"x\":0,\"y\":0,\"width\":0,\"height\":0},\"surface\":{\"targetId\":\"\",\"polygon\":[{\"x\":0,\"y\":0}],\"mesh\":{\"columns\":5,\"rows\":5,\"points\":[{\"x\":0,\"y\":0}]}},\"model\":{\"version\":2,\"depthMap\":{\"width\":16,\"height\":16,\"data\":[...]},\"depthScaleMin\":0,\"depthScaleMax\":1,\"shadeMap\":{\"width\":16,\"height\":16,\"data\":[...]},\"segmentationMap\":{\"width\":32,\"height\":32,\"data\":[...]},\"xOffsetMap\":{\"width\":16,\"height\":16,\"data\":[...]},\"yOffsetMap\":{\"width\":16,\"height\":16,\"data\":[...]},\"cameraIntrinsics\":{\"matrix\":[fx,0,cx,0,fy,cy,0,0,1]},\"colorCorrection\":\"limited_brightness\",\"foregroundImage\":\"\"}}.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": []map[string]any{
|
|
{
|
|
"type": "text",
|
|
"text": buildMockupModelPrompt(req),
|
|
},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": map[string]any{
|
|
"url": req.ImageDataURL,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}, 16000, 0.15, true)
|
|
if err != nil {
|
|
return design.MockupModelAnalysis{}, err
|
|
}
|
|
|
|
var analysis design.MockupModelAnalysis
|
|
if err := json.Unmarshal([]byte(extractJSONObject(content)), &analysis); err != nil {
|
|
return design.MockupModelAnalysis{}, err
|
|
}
|
|
return analysis, nil
|
|
}
|
|
|
|
func (c *CreativeAgent) DecideResearch(ctx context.Context, req design.AgentResearchDecisionRequest) (design.AgentResearchDecision, error) {
|
|
content, err := c.chat(ctx, []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You decide whether a Moteva-style creative agent should use web search.",
|
|
"Web search is available because the user enabled web reference, but it is not mandatory.",
|
|
"Return strict JSON only: {\"shouldSearch\":true|false,\"query\":\"search query or empty\",\"reason\":\"short reason\"}.",
|
|
"Use search only when the task materially depends on current facts, public brand/product/site details, competitors, trends, market references, unfamiliar proper nouns, or an explicit request to research the web.",
|
|
"Do not search for greetings, capability questions, pure canvas operations, ordinary image edits, or style refinements that can be handled from the canvas and conversation context.",
|
|
"If searching, write one focused keyword query in the user's language when suitable; use English only when it will likely find stronger global results.",
|
|
"The query must be your analyzed retrieval keywords, not the raw user prompt. Aim for design references: drafts, case studies, visual identity, design process, moodboards, competitors, and style systems.",
|
|
"For logo tasks, avoid logo-maker/tool results: include logo design case study, brand identity, visual identity, design process, moodboard, and negative terms such as -maker -generator -free -template when useful.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": buildResearchDecisionPrompt(req),
|
|
},
|
|
}, 360, 0.15, true)
|
|
if err != nil {
|
|
return design.AgentResearchDecision{}, err
|
|
}
|
|
|
|
var parsed struct {
|
|
ShouldSearch bool `json:"shouldSearch"`
|
|
Query string `json:"query"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
if err := json.Unmarshal([]byte(extractJSONObject(content)), &parsed); err != nil {
|
|
return design.AgentResearchDecision{}, err
|
|
}
|
|
return design.AgentResearchDecision{
|
|
ShouldSearch: parsed.ShouldSearch,
|
|
Query: strings.TrimSpace(parsed.Query),
|
|
Reason: strings.TrimSpace(parsed.Reason),
|
|
}, nil
|
|
}
|
|
|
|
func (c *CreativeAgent) chat(ctx context.Context, messages []map[string]any, maxTokens int, temperature float64, jsonMode bool) (string, error) {
|
|
body := map[string]any{
|
|
"model": c.model,
|
|
"messages": messages,
|
|
"max_tokens": maxTokens,
|
|
"temperature": temperature,
|
|
}
|
|
if jsonMode {
|
|
body["response_format"] = map[string]any{"type": "json_object"}
|
|
}
|
|
|
|
rawBody, err := json.Marshal(body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/chat/completions", bytes.NewReader(rawBody))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respBody, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return "", fmt.Errorf("creative agent failed: %s: %s", resp.Status, trimErrorBody(respBody))
|
|
}
|
|
|
|
var payload struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
}
|
|
if err := json.Unmarshal(respBody, &payload); err != nil {
|
|
return "", err
|
|
}
|
|
if len(payload.Choices) == 0 {
|
|
return "", fmt.Errorf("creative agent returned no choices")
|
|
}
|
|
content := strings.TrimSpace(decodeChatContent(payload.Choices[0].Message.Content))
|
|
if content == "" {
|
|
return "", fmt.Errorf("creative agent returned empty content")
|
|
}
|
|
return content, nil
|
|
}
|
|
|
|
func (c *CreativeAgent) chatStream(ctx context.Context, messages []map[string]any, maxTokens int, temperature float64, onDelta func(delta string) error) (string, error) {
|
|
body := map[string]any{
|
|
"model": c.model,
|
|
"messages": messages,
|
|
"max_tokens": maxTokens,
|
|
"temperature": temperature,
|
|
"stream": true,
|
|
}
|
|
|
|
rawBody, err := json.Marshal(body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/chat/completions", bytes.NewReader(rawBody))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Accept", "text/event-stream")
|
|
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if readErr != nil {
|
|
return "", readErr
|
|
}
|
|
return "", fmt.Errorf("creative agent stream failed: %s: %s", resp.Status, trimErrorBody(respBody))
|
|
}
|
|
|
|
var builder strings.Builder
|
|
scanner := bufio.NewScanner(resp.Body)
|
|
scanner.Buffer(make([]byte, 0, 64*1024), 4<<20)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" || strings.HasPrefix(line, ":") {
|
|
continue
|
|
}
|
|
if !strings.HasPrefix(line, "data:") {
|
|
continue
|
|
}
|
|
data := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
|
|
if data == "[DONE]" {
|
|
break
|
|
}
|
|
delta := decodeChatStreamDelta([]byte(data))
|
|
if delta == "" {
|
|
continue
|
|
}
|
|
builder.WriteString(delta)
|
|
if onDelta != nil {
|
|
if err := onDelta(delta); err != nil {
|
|
return builder.String(), err
|
|
}
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return builder.String(), err
|
|
}
|
|
if strings.TrimSpace(builder.String()) == "" {
|
|
return "", fmt.Errorf("creative agent stream returned empty content")
|
|
}
|
|
return builder.String(), nil
|
|
}
|
|
|
|
func buildDecisionPrompt(req design.AgentDecisionRequest) string {
|
|
return strings.Join([]string{
|
|
"Project context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Harness memory:",
|
|
memoryContext(req.Memory),
|
|
"",
|
|
"Mode: " + req.Mode,
|
|
"User message:",
|
|
req.Prompt,
|
|
"",
|
|
"Input summary:",
|
|
agentMessageContentSummary(req.Messages),
|
|
}, "\n")
|
|
}
|
|
|
|
func buildResearchDecisionPrompt(req design.AgentResearchDecisionRequest) string {
|
|
return strings.Join([]string{
|
|
"Current canvas context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Harness memory:",
|
|
memoryContext(req.Memory),
|
|
"",
|
|
"Visible canvas assets:",
|
|
canvasAssetContext(req.Project.Nodes),
|
|
"",
|
|
"Recent conversation:",
|
|
recentConversationContext(req.Project.Messages),
|
|
"",
|
|
"User request and inline references:",
|
|
req.Prompt,
|
|
agentMessageContentSummary(req.Messages),
|
|
"",
|
|
"Mode: " + req.Mode,
|
|
}, "\n")
|
|
}
|
|
|
|
func buildAgentTaskPlanPrompt(req design.AgentTaskPlanRequest) string {
|
|
searchState := "disabled"
|
|
if req.WebSearchEnabled {
|
|
searchState = "enabled"
|
|
}
|
|
return strings.Join([]string{
|
|
"Current canvas context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Harness memory:",
|
|
memoryContext(req.Memory),
|
|
"",
|
|
"Visible canvas assets:",
|
|
canvasAssetContext(req.Project.Nodes),
|
|
"",
|
|
"Recent conversation:",
|
|
recentConversationContext(req.Project.Messages),
|
|
"",
|
|
"Inline references and user inputs:",
|
|
req.Prompt,
|
|
agentMessageContentSummary(req.Messages),
|
|
"",
|
|
"Mode: " + req.Mode,
|
|
"Web search: " + searchState,
|
|
"",
|
|
"Plan the workflow before any tool runs. Decide intent, whether research is useful, how many output images/artboards are actually needed, and the concrete image tasks.",
|
|
}, "\n")
|
|
}
|
|
|
|
func buildConversationPrompt(req design.AgentConversationRequest) string {
|
|
return strings.Join([]string{
|
|
"Current canvas context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Harness memory:",
|
|
memoryContext(req.Memory),
|
|
"",
|
|
"Recent conversation:",
|
|
recentConversationContext(req.Project.Messages),
|
|
"",
|
|
"Agent task plan:",
|
|
taskPlanContext(req.TaskPlan),
|
|
"",
|
|
"Inline references:",
|
|
agentMessageContentSummary(req.Messages),
|
|
"",
|
|
"User message:",
|
|
req.Prompt,
|
|
"",
|
|
"Respond as the creative production agent. If the user is asking what you can do, be specific to image/ecommerce/canvas work. If they are just greeting, keep it short.",
|
|
}, "\n")
|
|
}
|
|
|
|
func buildImageSummaryPrompt(req design.AgentImageSummaryRequest) string {
|
|
return strings.Join([]string{
|
|
"Current canvas context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Harness memory:",
|
|
memoryContext(req.Memory),
|
|
"",
|
|
"User image task:",
|
|
req.Prompt,
|
|
"",
|
|
"Generated image URL:",
|
|
req.ImageURL,
|
|
"",
|
|
"Write the message that should appear after GPT Image 2 finishes.",
|
|
}, "\n")
|
|
}
|
|
|
|
func buildImagePromptPlannerInput(req design.AgentImagePromptRequest) string {
|
|
return strings.Join([]string{
|
|
"Current canvas context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Harness memory:",
|
|
memoryContext(req.Memory),
|
|
"",
|
|
"Visible canvas assets:",
|
|
canvasAssetContext(req.Project.Nodes),
|
|
"",
|
|
"Recent conversation:",
|
|
recentConversationContext(req.Project.Messages),
|
|
"",
|
|
"Agent task plan:",
|
|
taskPlanContext(req.TaskPlan),
|
|
"",
|
|
"Web research context:",
|
|
researchContext(req.Project.Messages),
|
|
"",
|
|
"User request and inline references:",
|
|
req.Prompt,
|
|
agentMessageContentSummary(req.Messages),
|
|
"",
|
|
"Mode: " + req.Mode,
|
|
}, "\n")
|
|
}
|
|
|
|
func buildMockupModelPrompt(req design.MockupModelAnalysisRequest) string {
|
|
return strings.Join([]string{
|
|
"Current canvas context:",
|
|
projectBriefContext(req.Project),
|
|
"",
|
|
"Selected target node:",
|
|
fmt.Sprintf("- title: %s", strings.TrimSpace(req.Node.Title)),
|
|
fmt.Sprintf("- canvas size: %.0fx%.0f", req.Node.Width, req.Node.Height),
|
|
fmt.Sprintf("- source image pixels: %dx%d", req.ImageWidth, req.ImageHeight),
|
|
"",
|
|
"Generate a Moteva-style mockup model for this selected image.",
|
|
"The placement and polygon must identify the printable area. The raw map arrays must describe the same area and surface deformation.",
|
|
"Return only the JSON object requested by the system message.",
|
|
}, "\n")
|
|
}
|
|
|
|
func buildVectorizePrompt(req design.ImageVectorizationRequest) string {
|
|
width := req.Width
|
|
height := req.Height
|
|
if width <= 0 {
|
|
width = 1024
|
|
}
|
|
if height <= 0 {
|
|
height = 1024
|
|
}
|
|
lines := []string{
|
|
"Vectorize the supplied image into editable SVG markup.",
|
|
fmt.Sprintf("Target SVG dimensions: %dx%d.", width, height),
|
|
"Use the image itself as the visual source of truth.",
|
|
"Return only the JSON object requested by the system message.",
|
|
}
|
|
if title := strings.TrimSpace(req.Title); title != "" {
|
|
lines = append(lines, "Canvas node title: "+title)
|
|
}
|
|
if prompt := strings.TrimSpace(req.Prompt); prompt != "" {
|
|
lines = append(lines, "User instruction: "+prompt)
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func parseVectorizeResponse(content string, req design.ImageVectorizationRequest) (design.ImageVectorization, error) {
|
|
var parsed struct {
|
|
SVG string `json:"svg"`
|
|
Width int64 `json:"width"`
|
|
Height int64 `json:"height"`
|
|
}
|
|
source := content
|
|
if err := json.Unmarshal([]byte(extractJSONObject(content)), &parsed); err == nil && strings.TrimSpace(parsed.SVG) != "" {
|
|
source = parsed.SVG
|
|
}
|
|
svg := extractSVGMarkup(source)
|
|
if svg == "" {
|
|
return design.ImageVectorization{}, fmt.Errorf("creative agent returned no SVG markup")
|
|
}
|
|
width := parsed.Width
|
|
height := parsed.Height
|
|
if width <= 0 {
|
|
width = req.Width
|
|
}
|
|
if height <= 0 {
|
|
height = req.Height
|
|
}
|
|
return design.ImageVectorization{
|
|
SVG: svg,
|
|
Width: width,
|
|
Height: height,
|
|
}, nil
|
|
}
|
|
|
|
func extractSVGMarkup(content string) string {
|
|
content = strings.TrimSpace(content)
|
|
if strings.HasPrefix(content, "```") {
|
|
content = strings.Trim(content, "`")
|
|
content = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(content), "svg"))
|
|
}
|
|
lower := strings.ToLower(content)
|
|
start := strings.Index(lower, "<svg")
|
|
end := strings.LastIndex(lower, "</svg>")
|
|
if start < 0 || end < start {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(content[start : end+len("</svg>")])
|
|
}
|
|
|
|
func taskPlanContext(plan design.AgentTaskPlan) string {
|
|
if strings.TrimSpace(plan.Intent) == "" && len(plan.ImageTasks) == 0 && strings.TrimSpace(plan.UserFacingResponse) == "" {
|
|
return "none"
|
|
}
|
|
var lines []string
|
|
if strings.TrimSpace(plan.Intent) != "" {
|
|
lines = append(lines, "intent: "+strings.TrimSpace(plan.Intent))
|
|
}
|
|
if strings.TrimSpace(plan.UserFacingResponse) != "" {
|
|
lines = append(lines, "user response: "+strings.TrimSpace(plan.UserFacingResponse))
|
|
}
|
|
if plan.ImageCount > 0 {
|
|
lines = append(lines, fmt.Sprintf("image count: %d", plan.ImageCount))
|
|
}
|
|
if plan.ShouldSearch {
|
|
lines = append(lines, "web search: true")
|
|
if strings.TrimSpace(plan.SearchQuery) != "" {
|
|
lines = append(lines, "search query: "+strings.TrimSpace(plan.SearchQuery))
|
|
}
|
|
}
|
|
for index, task := range plan.ImageTasks {
|
|
title := strings.TrimSpace(task.Title)
|
|
brief := strings.TrimSpace(task.Brief)
|
|
if title == "" && brief == "" {
|
|
continue
|
|
}
|
|
lines = append(lines, fmt.Sprintf("task %d: %s - %s", index+1, title, brief))
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func projectBriefContext(project design.Project) string {
|
|
var builder strings.Builder
|
|
if strings.TrimSpace(project.Title) != "" {
|
|
builder.WriteString("Title: ")
|
|
builder.WriteString(project.Title)
|
|
builder.WriteString("\n")
|
|
}
|
|
if strings.TrimSpace(project.Brief) != "" {
|
|
builder.WriteString("Brief: ")
|
|
builder.WriteString(project.Brief)
|
|
builder.WriteString("\n")
|
|
}
|
|
builder.WriteString(fmt.Sprintf("Canvas nodes: %d", len(project.Nodes)))
|
|
return builder.String()
|
|
}
|
|
|
|
func memoryContext(memory design.AgentMemory) string {
|
|
if memory.IsZero() {
|
|
return "none"
|
|
}
|
|
var sections []string
|
|
if strings.TrimSpace(memory.ShortTerm) != "" {
|
|
sections = append(sections, "Short memory:\n"+strings.TrimSpace(memory.ShortTerm))
|
|
}
|
|
if strings.TrimSpace(memory.LongTerm) != "" {
|
|
sections = append(sections, "Long memory:\n"+strings.TrimSpace(memory.LongTerm))
|
|
}
|
|
return strings.Join(sections, "\n")
|
|
}
|
|
|
|
func canvasAssetContext(nodes []design.Node) string {
|
|
if len(nodes) == 0 {
|
|
return "none"
|
|
}
|
|
start := 0
|
|
if len(nodes) > 8 {
|
|
start = len(nodes) - 8
|
|
}
|
|
var lines []string
|
|
for _, node := range nodes[start:] {
|
|
if node.Status == "generating" || node.LayerRole == "pen-stroke" {
|
|
continue
|
|
}
|
|
content := strings.TrimSpace(node.Content)
|
|
if strings.HasPrefix(content, "http") || strings.HasPrefix(content, "/canvas-assets/") {
|
|
content = "asset reference"
|
|
} else {
|
|
content = truncateRunes(content, 120)
|
|
}
|
|
lines = append(lines, fmt.Sprintf("- %s | %s | %.0fx%.0f | %s", node.Type, node.Title, node.Width, node.Height, content))
|
|
}
|
|
if len(lines) == 0 {
|
|
return "none"
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func recentConversationContext(messages []design.Message) string {
|
|
if len(messages) == 0 {
|
|
return "none"
|
|
}
|
|
start := 0
|
|
if len(messages) > 6 {
|
|
start = len(messages) - 6
|
|
}
|
|
var lines []string
|
|
for _, message := range messages[start:] {
|
|
content := strings.TrimSpace(message.Content)
|
|
if content == "" || message.Role == "tool" || message.Role == "error" {
|
|
continue
|
|
}
|
|
content = truncateRunes(content, 220)
|
|
lines = append(lines, fmt.Sprintf("- %s: %s", message.Role, content))
|
|
}
|
|
if len(lines) == 0 {
|
|
return "none"
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func researchContext(messages []design.Message) string {
|
|
if len(messages) == 0 {
|
|
return "none"
|
|
}
|
|
start := 0
|
|
if len(messages) > 12 {
|
|
start = len(messages) - 12
|
|
}
|
|
var lines []string
|
|
for _, message := range messages[start:] {
|
|
if message.Role != "tool" || message.Title != "Synthesize Skills" {
|
|
continue
|
|
}
|
|
var report struct {
|
|
Query string `json:"query"`
|
|
Results []struct {
|
|
Title string `json:"title"`
|
|
URL string `json:"url"`
|
|
Snippet string `json:"snippet"`
|
|
} `json:"results"`
|
|
Error string `json:"error"`
|
|
}
|
|
if err := json.Unmarshal([]byte(message.Content), &report); err != nil {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(report.Query) != "" {
|
|
lines = append(lines, "Query: "+strings.TrimSpace(report.Query))
|
|
}
|
|
for _, result := range report.Results {
|
|
title := strings.TrimSpace(result.Title)
|
|
snippet := strings.TrimSpace(result.Snippet)
|
|
if title == "" && snippet == "" {
|
|
continue
|
|
}
|
|
title = truncateRunes(title, 120)
|
|
snippet = truncateRunes(snippet, 160)
|
|
line := "- " + title
|
|
if snippet != "" {
|
|
line += ": " + snippet
|
|
}
|
|
lines = append(lines, line)
|
|
if len(lines) >= 8 {
|
|
break
|
|
}
|
|
}
|
|
if report.Error != "" {
|
|
lines = append(lines, "Search error: "+report.Error)
|
|
}
|
|
}
|
|
if len(lines) == 0 {
|
|
return "none"
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func agentMessageContentSummary(messages []design.AgentChatMessage) string {
|
|
var items []string
|
|
for _, message := range messages {
|
|
for _, content := range message.Contents {
|
|
switch strings.ToLower(strings.TrimSpace(content.Type)) {
|
|
case "text":
|
|
if text := strings.TrimSpace(content.Text); text != "" {
|
|
items = append(items, "text: "+truncateRunes(text, 300))
|
|
}
|
|
case "image":
|
|
if strings.TrimSpace(content.ImageURL) != "" {
|
|
size := ""
|
|
if content.ImageWidth > 0 && content.ImageHeight > 0 {
|
|
size = fmt.Sprintf(" %.0fx%.0f", float64(content.ImageWidth), float64(content.ImageHeight))
|
|
}
|
|
items = append(items, "image reference selected"+size)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(items) == 0 {
|
|
return "none"
|
|
}
|
|
return strings.Join(items, "\n")
|
|
}
|
|
|
|
func decodeChatStreamDelta(data []byte) string {
|
|
var payload struct {
|
|
Choices []struct {
|
|
Delta struct {
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"delta"`
|
|
Message struct {
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
}
|
|
if err := json.Unmarshal(data, &payload); err != nil || len(payload.Choices) == 0 {
|
|
return ""
|
|
}
|
|
content := decodeChatContent(payload.Choices[0].Delta.Content)
|
|
if strings.TrimSpace(content) == "" {
|
|
content = decodeChatContent(payload.Choices[0].Message.Content)
|
|
}
|
|
return content
|
|
}
|
|
|
|
func extractJSONObject(content string) string {
|
|
content = strings.TrimSpace(content)
|
|
if strings.HasPrefix(content, "```") {
|
|
content = strings.Trim(content, "`")
|
|
content = strings.TrimSpace(strings.TrimPrefix(content, "json"))
|
|
}
|
|
start := strings.Index(content, "{")
|
|
end := strings.LastIndex(content, "}")
|
|
if start >= 0 && end >= start {
|
|
return content[start : end+1]
|
|
}
|
|
return content
|
|
}
|