root
2f5291a0f9
Server: forward canvas/reference images to the image model so extract, continue, and edit requests stay faithful to the source product. - Add ImageRequestOptions.Images and route requests with inputs to the /v1/images/edits endpoint, via multipart upload (default) or JSON image_url payload, selectable through Agent.Image.InputImageTransport. - Extract reference URLs from prompt directives, inline @image tokens, and chat message image/mask contents; fold them into the idempotency key and prepend a reference-fidelity instruction to the prompt. Frontend: - Shrink CanvasToast to a compact top pill. - Paste an internal transparent clipboard node instead of re-uploading when it matches the clipboard image. - Keep agent-project polling alive on stream error while streaming.
1003 lines
31 KiB
Go
1003 lines
31 KiB
Go
package piagent
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"unicode/utf8"
|
||
|
||
"img_infinite_canvas/internal/domain/design"
|
||
|
||
"github.com/google/uuid"
|
||
"github.com/sky-valley/pi/agent"
|
||
"github.com/sky-valley/pi/ai"
|
||
)
|
||
|
||
const (
|
||
imageGenerationPromptMaxRunes = 900
|
||
imageGenerationFallbackMaxRunes = 650
|
||
imageGenerationPromptTooLargeRunes = 1300
|
||
)
|
||
|
||
type DesignAgent struct {
|
||
base design.AgentRunner
|
||
imageAgent ImageGenerator
|
||
textAgent design.CreativeAgent
|
||
tools []agent.AgentTool
|
||
}
|
||
|
||
func NewDesignAgent(base design.AgentRunner, imageAgent ImageGenerator, textAgents ...design.CreativeAgent) *DesignAgent {
|
||
var textAgent design.CreativeAgent
|
||
if len(textAgents) > 0 {
|
||
textAgent = textAgents[0]
|
||
}
|
||
return &DesignAgent{
|
||
base: base,
|
||
imageAgent: imageAgent,
|
||
textAgent: textAgent,
|
||
tools: designTools(),
|
||
}
|
||
}
|
||
|
||
func (a *DesignAgent) Plan(ctx context.Context, req design.AgentRequest) (design.AgentPlan, error) {
|
||
plan, err := a.base.Plan(ctx, req)
|
||
if err != nil {
|
||
return design.AgentPlan{}, err
|
||
}
|
||
|
||
plan.Message.Title = "Pi Agent 设计任务完成"
|
||
|
||
if shouldGenerateImage(req) {
|
||
if a.imageAgent == nil {
|
||
return design.AgentPlan{}, fmt.Errorf("gpt image 2 is not configured")
|
||
}
|
||
generationReq := req
|
||
generationReq.ImageSize = resolveImageSizeForGeneration(req.ImageSize, req.Prompt)
|
||
imagePrompts := a.buildImagePrompts(ctx, generationReq)
|
||
imageURLs, err := a.generatePlannedImages(ctx, imagePrompts, generationReq)
|
||
if err != nil {
|
||
return design.AgentPlan{}, fmt.Errorf("gpt image 2 generation failed: %w", err)
|
||
}
|
||
applyPlannedGeneratedImages(&plan, generationReq, imageURLs, imagePrompts)
|
||
title := ""
|
||
if len(imagePrompts) > 0 {
|
||
title = imagePrompts[0].Title
|
||
}
|
||
plan.Message.Title = generatedMessageTitle(title, req.Prompt)
|
||
plan.Message.Content = a.imageGenerationSummary(ctx, req, imageURLs[0])
|
||
}
|
||
return plan, nil
|
||
}
|
||
|
||
func (a *DesignAgent) PlanIncremental(ctx context.Context, req design.AgentRequest, onResult func(design.AgentImageGenerationResult) error) (design.AgentPlan, error) {
|
||
plan, err := a.base.Plan(ctx, req)
|
||
if err != nil {
|
||
return design.AgentPlan{}, err
|
||
}
|
||
|
||
plan.Message.Title = "Pi Agent 设计任务完成"
|
||
|
||
if !shouldGenerateImage(req) {
|
||
return plan, nil
|
||
}
|
||
if a.imageAgent == nil {
|
||
return design.AgentPlan{}, fmt.Errorf("gpt image 2 is not configured")
|
||
}
|
||
|
||
generationReq := req
|
||
generationReq.ImageSize = resolveImageSizeForGeneration(req.ImageSize, req.Prompt)
|
||
imagePrompts := a.buildImagePrompts(ctx, generationReq)
|
||
nodes, err := a.generatePlannedImagesIncremental(ctx, imagePrompts, generationReq, plan.GeneratedNodes, onResult)
|
||
if err != nil {
|
||
if len(nodes) > 0 {
|
||
plan.GeneratedNodes = nodes
|
||
plan.Connections = nil
|
||
}
|
||
return plan, fmt.Errorf("gpt image 2 generation failed: %w", err)
|
||
}
|
||
|
||
plan.GeneratedNodes = nodes
|
||
plan.Connections = nil
|
||
title := ""
|
||
if len(imagePrompts) > 0 {
|
||
title = imagePrompts[0].Title
|
||
}
|
||
plan.Message.Title = generatedMessageTitle(title, req.Prompt)
|
||
if len(nodes) > 0 {
|
||
plan.Message.Content = a.imageGenerationSummary(ctx, req, nodes[0].Content)
|
||
}
|
||
return plan, nil
|
||
}
|
||
|
||
func (a *DesignAgent) buildImagePrompts(ctx context.Context, req design.AgentRequest) []design.AgentImagePrompt {
|
||
count := plannedGenerationCount(req)
|
||
prompts := make([]design.AgentImagePrompt, 0, count)
|
||
for index := 0; index < count; index++ {
|
||
taskReq := req
|
||
if index < len(req.TaskPlan.ImageTasks) {
|
||
task := req.TaskPlan.ImageTasks[index]
|
||
if strings.TrimSpace(task.Brief) != "" {
|
||
taskReq.Prompt = strings.TrimSpace(task.Brief)
|
||
}
|
||
taskReq.TaskPlan.ImageTasks = []design.AgentImageTask{task}
|
||
taskReq.TaskPlan.ImageCount = 1
|
||
}
|
||
imagePrompt := a.buildImagePrompt(ctx, taskReq)
|
||
imagePrompt.Prompt = prepareGPTImagePrompt(imagePrompt.Prompt, imageGenerationPromptMaxRunes)
|
||
if imagePrompt.Title == "" && index < len(req.TaskPlan.ImageTasks) {
|
||
imagePrompt.Title = strings.TrimSpace(req.TaskPlan.ImageTasks[index].Title)
|
||
}
|
||
prompts = append(prompts, imagePrompt)
|
||
}
|
||
return prompts
|
||
}
|
||
|
||
func plannedGenerationCount(req design.AgentRequest) int {
|
||
count := req.TaskPlan.ImageCount
|
||
if count <= 0 && len(req.TaskPlan.ImageTasks) > 0 {
|
||
count = len(req.TaskPlan.ImageTasks)
|
||
}
|
||
if count <= 0 {
|
||
count = 1
|
||
}
|
||
if count > 10 {
|
||
count = 10
|
||
}
|
||
return count
|
||
}
|
||
|
||
func (a *DesignAgent) generatePlannedImages(ctx context.Context, prompts []design.AgentImagePrompt, req design.AgentRequest) ([]string, error) {
|
||
if len(prompts) == 0 {
|
||
prompts = []design.AgentImagePrompt{{Title: normalizeGeneratedTitle("", req.Prompt), Prompt: prepareGPTImagePrompt(fallbackDirectImagePrompt(req), imageGenerationFallbackMaxRunes)}}
|
||
}
|
||
if len(prompts) == 1 {
|
||
image, err := a.generateImageWithFallback(ctx, req, prompts[0].Prompt, fallbackImagePrompt(req, 0), 0)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
urls := generatedImageURLs(image)
|
||
if len(urls) == 0 {
|
||
return nil, fmt.Errorf("image generation returned empty image")
|
||
}
|
||
return urls[:1], nil
|
||
}
|
||
|
||
ctx, cancel := context.WithCancel(ctx)
|
||
defer cancel()
|
||
urls := make([]string, len(prompts))
|
||
errs := make(chan error, len(prompts))
|
||
var wg sync.WaitGroup
|
||
for index := range prompts {
|
||
index := index
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
image, err := a.generateImageWithFallback(ctx, req, prompts[index].Prompt, fallbackImagePrompt(req, index), index)
|
||
if err != nil {
|
||
errs <- err
|
||
cancel()
|
||
return
|
||
}
|
||
imageURLs := generatedImageURLs(image)
|
||
if len(imageURLs) == 0 {
|
||
errs <- fmt.Errorf("image generation returned empty image")
|
||
cancel()
|
||
return
|
||
}
|
||
urls[index] = imageURLs[0]
|
||
}()
|
||
}
|
||
wg.Wait()
|
||
close(errs)
|
||
if err := <-errs; err != nil {
|
||
return nil, err
|
||
}
|
||
return urls, nil
|
||
}
|
||
|
||
type plannedImageResult struct {
|
||
index int
|
||
node design.Node
|
||
err error
|
||
}
|
||
|
||
func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prompts []design.AgentImagePrompt, req design.AgentRequest, fallback []design.Node, onResult func(design.AgentImageGenerationResult) error) ([]design.Node, error) {
|
||
if len(prompts) == 0 {
|
||
prompts = []design.AgentImagePrompt{{Title: normalizeGeneratedTitle("", req.Prompt), Prompt: prepareGPTImagePrompt(fallbackDirectImagePrompt(req), imageGenerationFallbackMaxRunes)}}
|
||
}
|
||
|
||
slots := generatedImageSlots(req, fallback, len(prompts))
|
||
results := make(chan plannedImageResult, len(prompts))
|
||
var wg sync.WaitGroup
|
||
for index := range prompts {
|
||
index := index
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
image, err := a.generateImageWithFallback(ctx, req, prompts[index].Prompt, fallbackImagePrompt(req, index), index)
|
||
if err != nil {
|
||
results <- plannedImageResult{index: index, err: err}
|
||
return
|
||
}
|
||
imageURLs := generatedImageURLs(image)
|
||
if len(imageURLs) == 0 {
|
||
results <- plannedImageResult{index: index, err: fmt.Errorf("image generation returned empty image")}
|
||
return
|
||
}
|
||
title := ""
|
||
if index < len(prompts) {
|
||
title = prompts[index].Title
|
||
}
|
||
results <- plannedImageResult{
|
||
index: index,
|
||
node: generatedImageNodeFromSlot(slots[index], req, imageURLs[0], title, index, len(prompts)),
|
||
}
|
||
}()
|
||
}
|
||
go func() {
|
||
wg.Wait()
|
||
close(results)
|
||
}()
|
||
|
||
successes := make([]plannedImageResult, 0, len(prompts))
|
||
var firstErr error
|
||
for result := range results {
|
||
if result.err != nil {
|
||
if firstErr == nil {
|
||
firstErr = result.err
|
||
}
|
||
continue
|
||
}
|
||
if onResult != nil {
|
||
if err := onResult(design.AgentImageGenerationResult{Index: result.index, Node: result.node}); err != nil && firstErr == nil {
|
||
firstErr = err
|
||
continue
|
||
}
|
||
}
|
||
successes = append(successes, result)
|
||
}
|
||
if len(successes) == 0 {
|
||
if firstErr != nil {
|
||
return nil, firstErr
|
||
}
|
||
return nil, fmt.Errorf("image generation returned empty image")
|
||
}
|
||
|
||
sort.SliceStable(successes, func(i, j int) bool {
|
||
return successes[i].index < successes[j].index
|
||
})
|
||
nodes := make([]design.Node, 0, len(successes))
|
||
for _, result := range successes {
|
||
nodes = append(nodes, result.node)
|
||
}
|
||
if firstErr != nil {
|
||
return nodes, fmt.Errorf("only %d of %d planned images returned: %w", len(successes), len(prompts), firstErr)
|
||
}
|
||
return nodes, nil
|
||
}
|
||
|
||
func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) {
|
||
referenceImages := referenceImageURLs(req)
|
||
prompt = withReferenceImageInstruction(prompt, len(referenceImages))
|
||
fallback = withReferenceImageInstruction(fallback, len(referenceImages))
|
||
prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes)
|
||
fallback = prepareGPTImagePrompt(fallback, imageGenerationFallbackMaxRunes)
|
||
if imagePromptTooLarge(prompt) && strings.TrimSpace(fallback) != "" {
|
||
prompt = fallback
|
||
fallback = ""
|
||
}
|
||
image, err := a.imageAgent.Generate(ctx, prompt, ImageRequestOptions{
|
||
Model: req.ImageModel,
|
||
Size: req.ImageSize,
|
||
Count: 1,
|
||
Images: referenceImages,
|
||
IdempotencyKey: imageGenerationIdempotencyKey(req, index, prompt, false),
|
||
})
|
||
if err == nil {
|
||
return image, nil
|
||
}
|
||
if ctx.Err() != nil || isImageGenerationTerminalError(err) {
|
||
return GeneratedImage{}, err
|
||
}
|
||
if strings.TrimSpace(fallback) == "" || strings.TrimSpace(fallback) == strings.TrimSpace(prompt) {
|
||
return GeneratedImage{}, err
|
||
}
|
||
fallbackImage, fallbackErr := a.imageAgent.Generate(ctx, fallback, ImageRequestOptions{
|
||
Model: req.ImageModel,
|
||
Size: req.ImageSize,
|
||
Count: 1,
|
||
Images: referenceImages,
|
||
IdempotencyKey: imageGenerationIdempotencyKey(req, index, fallback, true),
|
||
})
|
||
if fallbackErr == nil {
|
||
return fallbackImage, nil
|
||
}
|
||
return GeneratedImage{}, fmt.Errorf("%w; fallback direct prompt also failed: %v", err, fallbackErr)
|
||
}
|
||
|
||
func imageGenerationIdempotencyKey(req design.AgentRequest, index int, prompt string, fallback bool) string {
|
||
kind := "primary"
|
||
if fallback {
|
||
kind = "fallback"
|
||
}
|
||
seed := strings.Join([]string{
|
||
strings.TrimSpace(req.Project.ID),
|
||
strings.TrimSpace(req.Mode),
|
||
strings.TrimSpace(req.Prompt),
|
||
strings.TrimSpace(req.ImageModel),
|
||
strings.TrimSpace(req.ImageSize),
|
||
strconv.Itoa(index),
|
||
kind,
|
||
strings.Join(referenceImageURLs(req), "\n"),
|
||
strings.TrimSpace(prompt),
|
||
}, "\n")
|
||
sum := sha256.Sum256([]byte(seed))
|
||
return "img-gen-" + kind + "-" + hex.EncodeToString(sum[:16])
|
||
}
|
||
|
||
func referenceImageURLs(req design.AgentRequest) []string {
|
||
urls := make([]string, 0)
|
||
seen := make(map[string]struct{})
|
||
add := func(value string) {
|
||
value = strings.TrimSpace(strings.TrimRight(value, "，。,."))
|
||
if value == "" {
|
||
return
|
||
}
|
||
if _, ok := seen[value]; ok {
|
||
return
|
||
}
|
||
seen[value] = struct{}{}
|
||
urls = append(urls, value)
|
||
}
|
||
addFromText := func(text string) {
|
||
for _, match := range referenceImageDirectiveURLPattern.FindAllStringSubmatch(text, -1) {
|
||
if len(match) > 1 {
|
||
add(match[1])
|
||
}
|
||
}
|
||
for _, match := range promptImageTokenURLPattern.FindAllStringSubmatch(text, -1) {
|
||
if len(match) > 1 && strings.TrimSpace(match[1]) != "" {
|
||
add(match[1])
|
||
continue
|
||
}
|
||
if len(match) > 2 {
|
||
add(match[2])
|
||
}
|
||
}
|
||
}
|
||
|
||
addFromText(req.Prompt)
|
||
for _, message := range req.Messages {
|
||
for _, content := range message.Contents {
|
||
switch strings.ToLower(strings.TrimSpace(content.Type)) {
|
||
case "image", "mask":
|
||
add(content.ImageURL)
|
||
case "text":
|
||
addFromText(content.Text)
|
||
}
|
||
if len(urls) >= 16 {
|
||
return urls[:16]
|
||
}
|
||
}
|
||
}
|
||
if len(urls) > 16 {
|
||
return urls[:16]
|
||
}
|
||
return urls
|
||
}
|
||
|
||
func withReferenceImageInstruction(prompt string, imageCount int) string {
|
||
prompt = strings.TrimSpace(prompt)
|
||
if prompt == "" || imageCount <= 0 {
|
||
return prompt
|
||
}
|
||
lower := strings.ToLower(prompt)
|
||
if strings.Contains(prompt, "附带参考图") || strings.Contains(lower, "attached reference image") || strings.Contains(lower, "input reference image") {
|
||
return prompt
|
||
}
|
||
if prefersEnglish(prompt) {
|
||
return "Input reference image(s) are attached. Use them as the visual source of truth for product shape, fabric, color, structure, and identity; when extracting, continuing, or editing a product, do not invent a different item.\n" + prompt
|
||
}
|
||
return "已随请求附带参考图；生成时必须以参考图中的商品外形、材质、颜色、结构细节和身份为准，拆出、延续或编辑商品时不要另造不同商品。\n" + prompt
|
||
}
|
||
|
||
func imagePromptTooLarge(prompt string) bool {
|
||
return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes
|
||
}
|
||
|
||
func fallbackImagePrompt(req design.AgentRequest, index int) string {
|
||
if index >= 0 && index < len(req.TaskPlan.ImageTasks) {
|
||
task := req.TaskPlan.ImageTasks[index]
|
||
if brief := strings.TrimSpace(task.Brief); brief != "" {
|
||
if title := strings.TrimSpace(task.Title); title != "" && !strings.Contains(brief, title) {
|
||
return title + ": " + brief
|
||
}
|
||
return brief
|
||
}
|
||
}
|
||
return fallbackDirectImagePrompt(req)
|
||
}
|
||
|
||
func prepareGPTImagePrompt(prompt string, maxRunes int) string {
|
||
prompt = compactImageGenerationPrompt(prompt, maxRunes)
|
||
if prompt == "" {
|
||
return ""
|
||
}
|
||
return prompt
|
||
}
|
||
|
||
func compactImageGenerationPrompt(prompt string, maxRunes int) string {
|
||
prompt = strings.TrimSpace(prompt)
|
||
prompt = strings.ReplaceAll(prompt, fullFrameSafetyInstruction, "")
|
||
prompt = strings.ReplaceAll(prompt, "Composition safety:", "")
|
||
prompt = imagePromptURLPattern.ReplaceAllString(prompt, "[参考图]")
|
||
lines := strings.Split(prompt, "\n")
|
||
compacted := make([]string, 0, len(lines))
|
||
for _, line := range lines {
|
||
line = strings.Join(strings.Fields(strings.TrimSpace(line)), " ")
|
||
if line != "" {
|
||
compacted = append(compacted, line)
|
||
}
|
||
}
|
||
prompt = strings.Join(compacted, "\n")
|
||
return truncateRunes(prompt, maxRunes)
|
||
}
|
||
|
||
func truncateRunes(value string, maxRunes int) string {
|
||
value = strings.TrimSpace(value)
|
||
if maxRunes <= 0 || utf8.RuneCountInString(value) <= maxRunes {
|
||
return value
|
||
}
|
||
runes := []rune(value)
|
||
value = strings.TrimSpace(string(runes[:maxRunes]))
|
||
if cut := strings.LastIndexAny(value, " ,，.。;；:：\n\t"); cut > maxRunes*3/4 {
|
||
value = strings.TrimSpace(value[:cut])
|
||
}
|
||
return value
|
||
}
|
||
|
||
func variantImagePrompt(prompt string, index int, count int) string {
|
||
if count <= 1 {
|
||
return prompt
|
||
}
|
||
return fmt.Sprintf("%s\n\nCreate option %d of %d. Keep the same brief, but make this option visually distinct in composition, rhythm, and detail choices.", prompt, index, count)
|
||
}
|
||
|
||
const fullFrameSafetyInstruction = "Composition safety: keep important subjects, products, text, logos, buttons, and decorative elements fully inside the visible image with safe margins. Do not crop edges unless requested. Do not add pixel size, aspect ratio, or resolution unless specified."
|
||
|
||
var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×|\*)\s*([1-9]\d{2,4})`)
|
||
|
||
var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`)
|
||
|
||
var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`)
|
||
var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[（(][^）)\n]+[）)])?\s*[:：]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`)
|
||
var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`)
|
||
|
||
func fullFrameImagePrompt(prompt string) string {
|
||
prompt = strings.TrimSpace(prompt)
|
||
if prompt == "" {
|
||
return fullFrameSafetyInstruction
|
||
}
|
||
if strings.Contains(prompt, fullFrameSafetyInstruction) || strings.Contains(strings.ToLower(prompt), "composition safety:") {
|
||
return prompt
|
||
}
|
||
return prompt + "\n\n" + fullFrameSafetyInstruction
|
||
}
|
||
|
||
func resolveImageSizeForGeneration(requestedSize string, prompt string) string {
|
||
requestedSize = normalizeImageSize(requestedSize)
|
||
if requestedSize != "" {
|
||
return requestedSize
|
||
}
|
||
if explicitSize := explicitImageSizeFromPrompt(prompt); explicitSize != "" {
|
||
return explicitSize
|
||
}
|
||
return design.DefaultGeneratedImageSize
|
||
}
|
||
|
||
func explicitImageSizeFromPrompt(prompt string) string {
|
||
prompt = strings.TrimSpace(prompt)
|
||
if prompt == "" {
|
||
return ""
|
||
}
|
||
if match := explicitPixelSizePattern.FindStringSubmatch(prompt); len(match) == 3 {
|
||
width, widthErr := strconv.Atoi(match[1])
|
||
height, heightErr := strconv.Atoi(match[2])
|
||
if widthErr == nil && heightErr == nil && width >= 256 && height >= 256 {
|
||
return fmt.Sprintf("%dx%d", width, height)
|
||
}
|
||
}
|
||
if match := explicitAspectRatioPattern.FindStringSubmatch(prompt); len(match) >= 2 {
|
||
switch strings.ReplaceAll(match[1], " ", "") {
|
||
case "1:1":
|
||
return "1024x1024"
|
||
case "3:2":
|
||
return "1536x1024"
|
||
case "2:3":
|
||
return "1024x1536"
|
||
case "4:3":
|
||
return "1365x1024"
|
||
case "3:4":
|
||
return "1024x1365"
|
||
case "16:9":
|
||
return "2048x1152"
|
||
case "9:16":
|
||
return "1152x2048"
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func normalizeImageSize(size string) string {
|
||
width, height, ok := imageSizeDimensions(size)
|
||
if !ok {
|
||
return ""
|
||
}
|
||
return fmt.Sprintf("%.0fx%.0f", width, height)
|
||
}
|
||
|
||
func (a *DesignAgent) Replay(ctx context.Context, project design.Project) (design.AgentReplay, error) {
|
||
replay, err := a.base.Replay(ctx, project)
|
||
if err != nil {
|
||
return design.AgentReplay{}, err
|
||
}
|
||
return replay, nil
|
||
}
|
||
|
||
func designTools() []agent.AgentTool {
|
||
return []agent.AgentTool{
|
||
{
|
||
Name: "collect_visual_references",
|
||
Label: "Collect visual references",
|
||
Description: "Collect Moteva-style visual references for a design request without launching a local desktop agent.",
|
||
Parameters: ai.Object(
|
||
ai.Prop("prompt", ai.String("User design request")),
|
||
ai.Prop("category", ai.String("Design category such as ecommerce, ui, brand, campaign")),
|
||
),
|
||
PromptGuidelines: []string{
|
||
"Use web and inspiration references to ground the visual direction.",
|
||
"Do not call video generation tools.",
|
||
},
|
||
},
|
||
{
|
||
Name: "compose_canvas_plan",
|
||
Label: "Compose canvas plan",
|
||
Description: "Turn a design request into visual canvas artboards and assistant narration.",
|
||
Parameters: ai.Object(
|
||
ai.Prop("prompt", ai.String("User design request")),
|
||
ai.Prop("project_id", ai.String("Current canvas project id")),
|
||
),
|
||
PromptGuidelines: []string{
|
||
"Return actionable nodes that can be replayed from chat history.",
|
||
"Keep handlers thin; persist canvas state through the project service.",
|
||
},
|
||
},
|
||
{
|
||
Name: "generate_image_with_gpt_image_2",
|
||
Label: "Generate image with GPT Image 2",
|
||
Description: "Generate an image through the configured OpenAI-compatible GPT Image 2 endpoint and place it on the infinite canvas.",
|
||
Parameters: ai.Object(
|
||
ai.Prop("prompt", ai.String("Image generation prompt")),
|
||
ai.Prop("size", ai.String("Requested output size")),
|
||
),
|
||
PromptGuidelines: []string{
|
||
"Use this tool for image generation requests.",
|
||
"Return the generated image as a canvas image node.",
|
||
},
|
||
},
|
||
{
|
||
Name: "prepare_image_asset_upload",
|
||
Label: "Prepare image asset upload",
|
||
Description: "Prepare an object-storage upload target for generated or referenced images.",
|
||
Parameters: ai.Object(
|
||
ai.Prop("file_name", ai.String("Original file name")),
|
||
ai.Prop("content_type", ai.String("Image MIME type")),
|
||
),
|
||
PromptGuidelines: []string{
|
||
"Use configured OSS/R2/AWS/MinIO storage for image assets.",
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
func shouldGenerateImage(req design.AgentRequest) bool {
|
||
intent := strings.ToLower(strings.TrimSpace(req.TaskPlan.Intent))
|
||
switch intent {
|
||
case "image", "generate_image", "edit_image", "visual", "design":
|
||
return true
|
||
case "chat", "conversation", "research":
|
||
return false
|
||
}
|
||
if req.TaskPlan.ImageCount > 0 || len(req.TaskPlan.ImageTasks) > 0 {
|
||
return true
|
||
}
|
||
|
||
mode := strings.ToLower(strings.TrimSpace(req.Mode))
|
||
if strings.Contains(mode, "image") || strings.Contains(mode, "design") {
|
||
return true
|
||
}
|
||
|
||
prompt := strings.ToLower(req.Prompt)
|
||
keywords := []string{"图", "图片", "海报", "视觉", "官网", "页面", "image", "poster", "website", "visual"}
|
||
for _, keyword := range keywords {
|
||
if strings.Contains(prompt, keyword) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func applyGeneratedImage(plan *design.AgentPlan, req design.AgentRequest, imageURL string, title string) {
|
||
applyGeneratedImages(plan, req, []string{imageURL}, title, 1)
|
||
}
|
||
|
||
func applyGeneratedImages(plan *design.AgentPlan, req design.AgentRequest, imageURLs []string, title string, requestedCount int) {
|
||
if requestedCount < 1 {
|
||
requestedCount = 1
|
||
}
|
||
if len(imageURLs) < requestedCount {
|
||
requestedCount = len(imageURLs)
|
||
}
|
||
slots := generatedImageSlots(req, plan.GeneratedNodes, requestedCount)
|
||
nodes := make([]design.Node, 0, requestedCount)
|
||
for index := 0; index < requestedCount; index++ {
|
||
node := slots[index]
|
||
node.Type = design.NodeTypeImage
|
||
node.Title = generatedNodeTitle(title, req.Prompt, index, requestedCount)
|
||
node.Content = imageURLs[index]
|
||
node.Status = "success"
|
||
if node.ID == "" {
|
||
node.ID = uuid.NewString()
|
||
}
|
||
if width, height, ok := imageSizeDimensions(req.ImageSize); ok {
|
||
node.Width = width
|
||
node.Height = height
|
||
}
|
||
if node.Width <= 0 {
|
||
node.Width = design.DefaultGeneratedImageWidth
|
||
}
|
||
if node.Height <= 0 {
|
||
node.Height = design.DefaultGeneratedImageHeight
|
||
}
|
||
if node.Tone == "" {
|
||
node.Tone = "visual"
|
||
}
|
||
nodes = append(nodes, node)
|
||
}
|
||
plan.GeneratedNodes = nodes
|
||
plan.Connections = nil
|
||
}
|
||
|
||
func applyPlannedGeneratedImages(plan *design.AgentPlan, req design.AgentRequest, imageURLs []string, prompts []design.AgentImagePrompt) {
|
||
requestedCount := len(imageURLs)
|
||
if requestedCount < 1 {
|
||
requestedCount = 1
|
||
}
|
||
slots := generatedImageSlots(req, plan.GeneratedNodes, requestedCount)
|
||
nodes := make([]design.Node, 0, requestedCount)
|
||
for index := 0; index < requestedCount; index++ {
|
||
title := ""
|
||
if index < len(prompts) {
|
||
title = prompts[index].Title
|
||
}
|
||
nodes = append(nodes, generatedImageNodeFromSlot(slots[index], req, imageURLs[index], title, index, requestedCount))
|
||
}
|
||
plan.GeneratedNodes = nodes
|
||
plan.Connections = nil
|
||
}
|
||
|
||
func generatedImageNodeFromSlot(slot design.Node, req design.AgentRequest, imageURL string, title string, index int, requestedCount int) design.Node {
|
||
node := slot
|
||
node.Type = design.NodeTypeImage
|
||
node.Title = generatedNodeTitle(title, req.Prompt, index, requestedCount)
|
||
node.Content = imageURL
|
||
node.Status = "success"
|
||
if node.ID == "" {
|
||
node.ID = uuid.NewString()
|
||
}
|
||
if width, height, ok := imageSizeDimensions(req.ImageSize); ok {
|
||
node.Width = width
|
||
node.Height = height
|
||
}
|
||
if node.Width <= 0 {
|
||
node.Width = design.DefaultGeneratedImageWidth
|
||
}
|
||
if node.Height <= 0 {
|
||
node.Height = design.DefaultGeneratedImageHeight
|
||
}
|
||
if node.Tone == "" {
|
||
node.Tone = "visual"
|
||
}
|
||
return node
|
||
}
|
||
|
||
func generatedImageURLs(image GeneratedImage) []string {
|
||
if len(image.URLs) > 0 {
|
||
return image.URLs
|
||
}
|
||
if strings.TrimSpace(image.URL) == "" {
|
||
return nil
|
||
}
|
||
return []string{image.URL}
|
||
}
|
||
|
||
func generatedNodeTitle(title string, prompt string, index int, count int) string {
|
||
title = normalizeGeneratedTitle(title, prompt)
|
||
if count <= 1 {
|
||
return title
|
||
}
|
||
if prefersEnglish(prompt + title) {
|
||
return fmt.Sprintf("%s Option %d", title, index+1)
|
||
}
|
||
return fmt.Sprintf("%s 方案 %d", title, index+1)
|
||
}
|
||
|
||
func (a *DesignAgent) buildImagePrompt(ctx context.Context, req design.AgentRequest) design.AgentImagePrompt {
|
||
if a.textAgent != nil {
|
||
prompt, err := a.textAgent.BuildImagePrompt(ctx, design.AgentImagePromptRequest{
|
||
Project: req.Project,
|
||
Prompt: req.Prompt,
|
||
Mode: req.Mode,
|
||
Messages: req.Messages,
|
||
Memory: req.Memory,
|
||
TaskPlan: req.TaskPlan,
|
||
})
|
||
if err == nil && strings.TrimSpace(prompt.Prompt) != "" {
|
||
prompt.Title = normalizeGeneratedTitle(prompt.Title, req.Prompt)
|
||
return prompt
|
||
}
|
||
}
|
||
return design.AgentImagePrompt{
|
||
Title: normalizeGeneratedTitle("", req.Prompt),
|
||
Prompt: fallbackDirectImagePrompt(req),
|
||
}
|
||
}
|
||
|
||
func normalizeGeneratedTitle(title string, prompt string) string {
|
||
title = strings.TrimSpace(title)
|
||
if title == "" || strings.EqualFold(title, "GPT Image 2") || strings.Contains(strings.ToLower(title), "gpt image") {
|
||
title = inferGeneratedTitle(prompt)
|
||
}
|
||
title = strings.Trim(title, "「」[]【】 ")
|
||
if len([]rune(title)) > 24 {
|
||
runes := []rune(title)
|
||
title = string(runes[:24])
|
||
}
|
||
return title
|
||
}
|
||
|
||
func generatedMessageTitle(title string, prompt string) string {
|
||
title = normalizeGeneratedTitle(title, prompt)
|
||
if title == "" {
|
||
if prefersEnglish(prompt) {
|
||
return "Image generated"
|
||
}
|
||
return "图片生成完成"
|
||
}
|
||
if prefersEnglish(prompt + title) {
|
||
return title + " generated"
|
||
}
|
||
return title + " 已生成"
|
||
}
|
||
|
||
func inferGeneratedTitle(prompt string) string {
|
||
lower := strings.ToLower(prompt)
|
||
if prefersEnglish(prompt) {
|
||
switch {
|
||
case strings.Contains(lower, "pdp") || strings.Contains(lower, "detail page"):
|
||
return "Product detail page"
|
||
case strings.Contains(lower, "landing") || strings.Contains(lower, "homepage") || strings.Contains(lower, "website"):
|
||
return "Homepage visual"
|
||
case strings.Contains(lower, "poster"):
|
||
return "Poster visual"
|
||
case strings.Contains(lower, "main image") || strings.Contains(lower, "product image"):
|
||
return "Product hero image"
|
||
case strings.Contains(lower, "banner"):
|
||
return "Banner visual"
|
||
case strings.Contains(lower, "logo"):
|
||
return "Logo visual"
|
||
default:
|
||
return "Generated image"
|
||
}
|
||
}
|
||
switch {
|
||
case strings.Contains(prompt, "详情页") || strings.Contains(lower, "pdp"):
|
||
return "电商详情页"
|
||
case strings.Contains(prompt, "首页") || strings.Contains(prompt, "官网") || strings.Contains(lower, "landing"):
|
||
return "官网首页视觉"
|
||
case strings.Contains(prompt, "海报") || strings.Contains(lower, "poster"):
|
||
return "海报视觉"
|
||
case strings.Contains(prompt, "主图") || strings.Contains(prompt, "产品图"):
|
||
return "产品主图"
|
||
case strings.Contains(prompt, "banner"):
|
||
return "Banner 视觉"
|
||
case strings.Contains(prompt, "logo"):
|
||
return "Logo 视觉"
|
||
default:
|
||
return "生成图片"
|
||
}
|
||
}
|
||
|
||
func fallbackDirectImagePrompt(req design.AgentRequest) string {
|
||
var lines []string
|
||
lines = append(lines, "Create one polished ecommerce/design image from this concise brief.")
|
||
if title := strings.TrimSpace(req.Project.Title); title != "" {
|
||
lines = append(lines, "Project: "+truncateRunes(title, 80))
|
||
}
|
||
if brief := strings.TrimSpace(req.Project.Brief); brief != "" {
|
||
lines = append(lines, "Project brief: "+truncateRunes(brief, 180))
|
||
}
|
||
if memory := compactFallbackMemory(req.Memory); memory != "" {
|
||
lines = append(lines, "Relevant memory: "+memory)
|
||
}
|
||
if task := compactFallbackTaskPlan(req.TaskPlan); task != "" {
|
||
lines = append(lines, "Task: "+task)
|
||
}
|
||
if prompt := strings.TrimSpace(req.Prompt); prompt != "" {
|
||
lines = append(lines, "User request: "+truncateRunes(prompt, 360))
|
||
}
|
||
if notes := compactFallbackInlineNotes(req.Messages); notes != "" {
|
||
lines = append(lines, "Inline notes: "+notes)
|
||
}
|
||
lines = append(lines, "Use strong commercial composition, clear hierarchy, refined typography when text is needed, coherent color and lighting, realistic safe margins, and no cropped important subjects.")
|
||
return compactImageGenerationPrompt(strings.Join(lines, "\n"), imageGenerationFallbackMaxRunes)
|
||
}
|
||
|
||
func compactFallbackMemory(memory design.AgentMemory) string {
|
||
if memory.IsZero() {
|
||
return ""
|
||
}
|
||
var parts []string
|
||
if value := strings.TrimSpace(memory.ShortTerm); value != "" {
|
||
parts = append(parts, truncateRunes(value, 180))
|
||
}
|
||
if value := strings.TrimSpace(memory.LongTerm); value != "" {
|
||
parts = append(parts, truncateRunes(value, 180))
|
||
}
|
||
return strings.Join(parts, " ")
|
||
}
|
||
|
||
func compactFallbackTaskPlan(plan design.AgentTaskPlan) string {
|
||
if len(plan.ImageTasks) == 0 {
|
||
return truncateRunes(strings.TrimSpace(plan.UserFacingResponse), 180)
|
||
}
|
||
var parts []string
|
||
for _, task := range plan.ImageTasks {
|
||
title := strings.TrimSpace(task.Title)
|
||
brief := strings.TrimSpace(task.Brief)
|
||
if title == "" && brief == "" {
|
||
continue
|
||
}
|
||
item := strings.TrimSpace(title + " " + brief)
|
||
parts = append(parts, truncateRunes(item, 160))
|
||
if len(parts) >= 3 {
|
||
break
|
||
}
|
||
}
|
||
return strings.Join(parts, "; ")
|
||
}
|
||
|
||
func compactFallbackInlineNotes(messages []design.AgentChatMessage) string {
|
||
var parts []string
|
||
for _, message := range messages {
|
||
for _, content := range message.Contents {
|
||
switch strings.ToLower(strings.TrimSpace(content.Type)) {
|
||
case "text":
|
||
if text := strings.TrimSpace(content.Text); text != "" {
|
||
parts = append(parts, truncateRunes(text, 160))
|
||
}
|
||
case "image":
|
||
if strings.TrimSpace(content.ImageURL) != "" {
|
||
parts = append(parts, "selected reference image")
|
||
}
|
||
}
|
||
if len(parts) >= 4 {
|
||
return strings.Join(parts, "; ")
|
||
}
|
||
}
|
||
}
|
||
return strings.Join(parts, "; ")
|
||
}
|
||
|
||
func generatedImageSlot(req design.AgentRequest, fallback []design.Node) design.Node {
|
||
slots := generatedImageSlots(req, fallback, 1)
|
||
if len(slots) > 0 {
|
||
return slots[0]
|
||
}
|
||
return design.Node{
|
||
ID: uuid.NewString(),
|
||
Type: design.NodeTypeImage,
|
||
Width: design.DefaultGeneratedImageWidth,
|
||
Height: design.DefaultGeneratedImageHeight,
|
||
}
|
||
}
|
||
|
||
func generatedImageSlots(req design.AgentRequest, fallback []design.Node, count int) []design.Node {
|
||
if count < 1 {
|
||
count = 1
|
||
}
|
||
slots := make([]design.Node, 0, count)
|
||
for index := len(req.Project.Nodes) - 1; index >= 0; index-- {
|
||
node := req.Project.Nodes[index]
|
||
if isImageGenerationSlot(node) {
|
||
slots = append(slots, node)
|
||
}
|
||
}
|
||
reverseNodes(slots)
|
||
if len(slots) >= count {
|
||
return slots[:count]
|
||
}
|
||
for _, node := range fallback {
|
||
if node.Type == design.NodeTypeImage || node.Type == design.NodeTypeFrame {
|
||
node.Width = design.DefaultGeneratedImageWidth
|
||
node.Height = design.DefaultGeneratedImageHeight
|
||
slots = append(slots, node)
|
||
if len(slots) >= count {
|
||
return slots[:count]
|
||
}
|
||
}
|
||
}
|
||
for len(slots) < count {
|
||
slots = append(slots, design.Node{
|
||
ID: uuid.NewString(),
|
||
Type: design.NodeTypeImage,
|
||
Width: design.DefaultGeneratedImageWidth,
|
||
Height: design.DefaultGeneratedImageHeight,
|
||
})
|
||
}
|
||
return slots
|
||
}
|
||
|
||
func isImageGenerationSlot(node design.Node) bool {
|
||
if node.Type != design.NodeTypeImage || node.Status != "generating" {
|
||
return false
|
||
}
|
||
role := strings.TrimSpace(node.LayerRole)
|
||
return role == "" || role == "image-generator"
|
||
}
|
||
|
||
func reverseNodes(nodes []design.Node) {
|
||
for left, right := 0, len(nodes)-1; left < right; left, right = left+1, right-1 {
|
||
nodes[left], nodes[right] = nodes[right], nodes[left]
|
||
}
|
||
}
|
||
|
||
func imageSizeDimensions(size string) (float64, float64, bool) {
|
||
return design.ParseImageSize(size)
|
||
}
|
||
|
||
func (a *DesignAgent) imageGenerationSummary(ctx context.Context, req design.AgentRequest, imageURL string) string {
|
||
if a.textAgent != nil {
|
||
content, err := a.textAgent.SummarizeImage(ctx, design.AgentImageSummaryRequest{
|
||
Project: req.Project,
|
||
Prompt: req.Prompt,
|
||
Mode: req.Mode,
|
||
ImageURL: imageURL,
|
||
Memory: req.Memory,
|
||
})
|
||
if err == nil && strings.TrimSpace(content) != "" {
|
||
return content
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func prefersEnglish(value string) bool {
|
||
value = strings.TrimSpace(value)
|
||
if value == "" {
|
||
return false
|
||
}
|
||
for _, r := range value {
|
||
if r >= 0x4E00 && r <= 0x9FFF {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|