From 039d1669725bd789304b49f6d8102ea3695e266a Mon Sep 17 00:00:00 2001 From: liangxu Date: Thu, 9 Jul 2026 12:26:43 +0800 Subject: [PATCH] feat(server): add layer-separation generator task Introduce the `layer-separation` generator (edit-elements node action) that splits a source image into a clean background layer, foreground image artifacts with bounding boxes, and editable text render data. - New layer_separation.go builds the separated result and artifacts - SubmitGeneratorTask routes layer-separation and move-object via a switch; standalone task completion + polling support - Extend artifact metadata with BBox and GeneratorName across the domain, API types, mapper, .api schema, and API.md - edit-elements gets a transparent manual-frame placeholder holding the separated foreground images and text layers beside the original Co-Authored-By: Claude Opus 4.8 (1M context) --- server/API.md | 19 + server/img_infinite_canvas.api | 19 +- server/internal/application/design_service.go | 33 +- server/internal/application/generator_task.go | 132 ++- .../internal/application/layer_separation.go | 892 ++++++++++++++++++ .../application/layer_separation_test.go | 410 ++++++++ server/internal/domain/design/agent.go | 18 +- server/internal/logic/mapper.go | 18 +- server/internal/types/types.go | 18 +- 9 files changed, 1523 insertions(+), 36 deletions(-) create mode 100644 server/internal/application/layer_separation.go create mode 100644 server/internal/application/layer_separation_test.go diff --git a/server/API.md b/server/API.md index d9c5869..9840d03 100644 --- a/server/API.md +++ b/server/API.md @@ -238,6 +238,25 @@ Returns the updated project, the assistant message, and generated nodes. The generation path uses the configured design agent runner. The default is the `sky-valley/pi` Go agent adapter, with local desktop/MCP agent behavior disabled. +## Generator Tasks + +`POST /api/generator/tasks` + +```json +{ + "cid": "client-request-id", + "project_id": "project-id", + "generator_name": "layer-separation", + "input_args": { + "image_url": "https://cdn.example.com/source.png" + } +} +``` + +`layer-separation` returns a task id that can be polled with `GET /api/generator/tasks?task_id=&project_id=`. Completed tasks return a clean background image, foreground image artifacts with `bbox`, and a `text_render_data` artifact for editable text reconstruction. + +The canvas `edit-elements` node action uses the same separation path but writes the separated result beside the original image. The original image remains unchanged; the clean background is a bottom image layer, and a transparent frame above it holds the separated foreground images and editable text layers. + ## Chat History Replay `GET /api/projects/:id/history` diff --git a/server/img_infinite_canvas.api b/server/img_infinite_canvas.api index 06f3a28..116f919 100644 --- a/server/img_infinite_canvas.api +++ b/server/img_infinite_canvas.api @@ -670,14 +670,16 @@ type GeneratorTaskQueueInfo { } type GeneratorTaskArtifactMetadata { - ArtifactId string `json:"artifact_id,optional"` - Format string `json:"format,optional"` - Height int64 `json:"height,optional"` - Label string `json:"label,optional"` - NodeId string `json:"node_id,optional"` - NodeName string `json:"node_name,optional"` - SizeBytes int64 `json:"size_bytes,optional"` - Width int64 `json:"width,optional"` + ArtifactId string `json:"artifact_id,optional"` + BBox []int64 `json:"bbox,optional"` + Format string `json:"format,optional"` + GeneratorName string `json:"generator_name,optional"` + Height int64 `json:"height,optional"` + Label string `json:"label,optional"` + NodeId string `json:"node_id,optional"` + NodeName string `json:"node_name,optional"` + SizeBytes int64 `json:"size_bytes,optional"` + Width int64 `json:"width,optional"` } type GeneratorTaskArtifact { @@ -1013,3 +1015,4 @@ service img_infinite_canvas-api { @handler deleteAsset delete /assets (DeleteAssetRequest) returns (DeleteAssetResponse) } + diff --git a/server/internal/application/design_service.go b/server/internal/application/design_service.go index 9a2b3f3..adca3a9 100644 --- a/server/internal/application/design_service.go +++ b/server/internal/application/design_service.go @@ -892,12 +892,14 @@ func (s *DesignService) RunNodeActionAsync(ctx context.Context, projectID string now := s.now() threadID := newID() target := project.Nodes[nodeIndex] - if req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" { + if req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements" { resultNode := newBackgroundRemovalPlaceholder(target) if req.Action == "vectorize" { resultNode = newVectorizePlaceholder(target) } else if req.Action == "move-object" { resultNode = newMoveObjectPlaceholder(target) + } else if req.Action == "edit-elements" { + resultNode = newLayerSeparationPlaceholder(target) } if req.Options == nil { req.Options = make(map[string]string) @@ -1845,7 +1847,10 @@ func (s *DesignService) completeNodeActionGeneration(ctx context.Context, projec if req.Action == "vectorize" { return s.completeVectorizeGeneration(ctx, projectID, threadID, target, req) } - if req.Action == "edit-elements" || req.Action == "edit-text" { + if req.Action == "edit-elements" { + return s.completeLayerSeparationGeneration(ctx, projectID, threadID, target, req) + } + if req.Action == "edit-text" { return s.completeEditableLayerGeneration(ctx, projectID, threadID, target, req) } editToolContent := "正在根据选中图片执行模型编辑，完成后会替换画布中的图片节点。" @@ -4709,8 +4714,30 @@ func newMoveObjectPlaceholder(target design.Node) design.Node { return result } +func newLayerSeparationPlaceholder(target design.Node) design.Node { + result := target + result.ID = newID() + result.Type = design.NodeTypeFrame + result.Title = nodeActionWorkingTitle("edit-elements") + result.X = target.X + target.Width + 32 + result.Y = target.Y + result.Content = target.Content + result.Tone = "frame" + result.Status = "generating" + result.ParentID = "" + result.LayerRole = "manual-frame" + result.FillColor = "transparent" + result.StrokeColor = "#C7C7C7" + result.StrokeWidth = 1 + result.StrokeStyle = "solid" + result.MockupSurface = design.MockupSurface{} + result.MockupModel = design.MockupModel{} + result.MockupSourceContent = "" + return result +} + func nodeActionResultNodeID(target design.Node, req design.NodeActionRequest) string { - if (req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object") && len(req.Options) > 0 { + if (req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements") && len(req.Options) > 0 { if id := strings.TrimSpace(req.Options[nodeActionResultNodeIDOption]); id != "" { return id } diff --git a/server/internal/application/generator_task.go b/server/internal/application/generator_task.go index 97e8347..32a8036 100644 --- a/server/internal/application/generator_task.go +++ b/server/internal/application/generator_task.go @@ -21,7 +21,9 @@ type generatorTaskMetadata struct { const ( removeBackgroundGeneratorName = "removeBG" imageVectorizerGeneratorName = "gpt-5.5-vectorizer" + layerSeparationGeneratorName = "layer-separation" moveObjectGeneratorName = "move-object" + layerSeparationMessageName = "layer_separation" ) func (s *DesignService) GeneratorTask(ctx context.Context, projectID string, taskID string) (design.GeneratorTask, error) { @@ -72,10 +74,17 @@ func (s *DesignService) SubmitGeneratorTask(ctx context.Context, req design.Gene if generatorName == "" { generatorName = moveObjectGeneratorName } - if normalizeGeneratorTaskName(generatorName) != moveObjectGeneratorName { + switch normalizeGeneratorTaskName(generatorName) { + case moveObjectGeneratorName: + return s.submitMoveObjectTask(ctx, req) + case layerSeparationGeneratorName: + return s.submitLayerSeparationTask(ctx, req) + default: return design.GeneratorTask{}, fmt.Errorf("%w: unsupported generator_name %s", design.ErrInvalidInput, req.GeneratorName) } +} +func (s *DesignService) submitMoveObjectTask(ctx context.Context, req design.GeneratorTaskSubmitRequest) (design.GeneratorTask, error) { input := normalizeMoveObjectInput(req.InputArgs) if strings.TrimSpace(input.ImageURL) == "" { return design.GeneratorTask{}, fmt.Errorf("%w: input_args.image_url is required", design.ErrInvalidInput) @@ -101,6 +110,53 @@ func (s *DesignService) SubmitGeneratorTask(ctx context.Context, req design.Gene return task, nil } +func (s *DesignService) submitLayerSeparationTask(ctx context.Context, req design.GeneratorTaskSubmitRequest) (design.GeneratorTask, error) { + input := normalizeLayerSeparationInput(req.InputArgs) + if strings.TrimSpace(input.ImageURL) == "" { + return design.GeneratorTask{}, fmt.Errorf("%w: input_args.image_url is required", design.ErrInvalidInput) + } + + taskID := newID() + price := generatorTaskPrice(input, layerSeparationGeneratorName) + task := design.GeneratorTask{ + GeneratorTaskID: taskID, + GeneratorName: layerSeparationGeneratorName, + Status: "submitted", + OriginalInputArgs: input, + QueueInfo: generatorTaskQueueInfo(layerSeparationGeneratorName, input, price), + } + s.storeStandaloneGeneratorTask(task) + go s.completeStandaloneLayerSeparationTask(taskID, strings.TrimSpace(req.ProjectID), input) + return task, nil +} + +func (s *DesignService) completeStandaloneLayerSeparationTask(taskID string, projectID string, input design.GeneratorTaskInputArgs) { + ctx, cancel := context.WithTimeout(context.Background(), generationBudget) + defer cancel() + + imageURL := strings.TrimSpace(input.ImageURL) + target := design.Node{ + ID: "source", + Type: design.NodeTypeImage, + Title: "Source image", + Content: imageURL, + Status: "success", + } + result, err := s.createLayerSeparationResult(ctx, standaloneArtifactProjectID(projectID, taskID), target, design.NodeActionRequest{ + Action: "edit-elements", + Prompt: strings.TrimSpace(input.Prompt), + }) + if err != nil { + s.markStandaloneGeneratorTaskFailed(taskID, err) + return + } + if len(result.Artifacts) == 0 { + s.markStandaloneGeneratorTaskFailed(taskID, fmt.Errorf("layer separation returned no artifacts")) + return + } + s.completeStandaloneGeneratorTask(taskID, result.Artifacts) +} + func (s *DesignService) completeStandaloneMoveObjectTask(taskID string, projectID string, input design.GeneratorTaskInputArgs) { ctx, cancel := context.WithTimeout(context.Background(), generationBudget) defer cancel() @@ -218,6 +274,17 @@ func normalizeMoveObjectInput(input design.GeneratorTaskInputArgs) design.Genera return input } +func normalizeLayerSeparationInput(input design.GeneratorTaskInputArgs) design.GeneratorTaskInputArgs { + if strings.TrimSpace(input.ImageURL) == "" && len(input.Image) > 0 { + input.ImageURL = strings.TrimSpace(input.Image[0]) + } + input.ImageURL = strings.TrimSpace(input.ImageURL) + input.Prompt = strings.TrimSpace(input.Prompt) + input.Resolution = strings.TrimSpace(input.Resolution) + input.AspectRatio = strings.TrimSpace(input.AspectRatio) + return input +} + func normalizeBox(box design.NormalizedBox) design.NormalizedBox { width := clampUnit(box.Width) height := clampUnit(box.Height) @@ -427,6 +494,7 @@ func generatorTaskMessageCompleted(message design.Message) bool { } return name == "canvas_action" || name == "text_extraction" || + name == layerSeparationMessageName || name == "mockup_model" || strings.TrimSpace(message.Role) == "assistant" } @@ -470,6 +538,8 @@ func generatorTaskGeneratorNameFromMessages(messages []design.Message, taskID st return "gpt-image-2" case strings.Contains(strings.ToLower(message.Title), "rmbg"): return removeBackgroundGeneratorName + case message.Name == layerSeparationMessageName: + return layerSeparationGeneratorName case message.Name == "mockup_model": return "mockup/model" case message.Name == "text_extraction": @@ -498,6 +568,10 @@ func generatorTaskArtifacts(messages []design.Message, taskID string) []design.G if message.ThreadID != taskID || !isToolArtifactsMessage(message) { continue } + if message.Name == layerSeparationMessageName || message.ToolHint == layerSeparationMessageName { + artifacts = append(artifacts, parseLayerSeparationArtifacts(message.Content)...) + continue + } artifacts = append(artifacts, parseGeneratorArtifacts(message.Content)...) } return artifacts @@ -549,6 +623,55 @@ func parseGeneratorArtifacts(content string) []design.GeneratorTaskArtifact { return artifacts } +func parseLayerSeparationArtifacts(content string) []design.GeneratorTaskArtifact { + var raw []struct { + Type string `json:"type"` + Content string `json:"content"` + Metadata struct { + ArtifactID string `json:"artifact_id"` + BBox []int64 `json:"bbox"` + Format string `json:"format"` + GeneratorName string `json:"generator_name"` + Height int64 `json:"height"` + Label string `json:"label"` + NodeID string `json:"node_id"` + NodeName string `json:"node_name"` + SizeBytes int64 `json:"size_bytes"` + Width int64 `json:"width"` + } `json:"metadata"` + } + if err := json.Unmarshal([]byte(content), &raw); err != nil { + return nil + } + artifacts := make([]design.GeneratorTaskArtifact, 0, len(raw)) + for _, item := range raw { + artifactType := strings.TrimSpace(item.Type) + if artifactType == "" { + continue + } + if strings.TrimSpace(item.Content) == "" && artifactType != "text" { + continue + } + artifacts = append(artifacts, design.GeneratorTaskArtifact{ + Type: artifactType, + Content: item.Content, + Metadata: design.GeneratorTaskArtifactMetadata{ + ArtifactID: item.Metadata.ArtifactID, + BBox: append([]int64(nil), item.Metadata.BBox...), + Format: item.Metadata.Format, + GeneratorName: item.Metadata.GeneratorName, + Height: item.Metadata.Height, + Label: item.Metadata.Label, + NodeID: item.Metadata.NodeID, + NodeName: item.Metadata.NodeName, + SizeBytes: item.Metadata.SizeBytes, + Width: item.Metadata.Width, + }, + }) + } + return artifacts +} + func generatorArtifactFormat(format string, imageURL string) string { format = strings.Trim(strings.ToLower(strings.TrimSpace(format)), ".") if format != "" { @@ -605,6 +728,8 @@ func generatorTaskPrice(input design.GeneratorTaskInputArgs, generatorName strin return 4 case imageVectorizerGeneratorName: return 14 + case layerSeparationGeneratorName: + return 8 case moveObjectGeneratorName: return 5 } @@ -668,6 +793,8 @@ func generatorTaskNameForNodeAction(req design.NodeActionRequest) string { switch normalizeNodeAction(req.Action) { case "remove-background": return removeBackgroundGeneratorName + case "edit-elements": + return layerSeparationGeneratorName case "edit-text": return "gpt-5.4-mini" case "move-object": @@ -749,6 +876,9 @@ func normalizeGeneratorTaskName(generatorName string) string { if normalized == "move-object" || normalized == "moveobject" { return moveObjectGeneratorName } + if normalized == "layer-separation" || normalized == "layer_separation" || normalized == "edit-elements" || strings.HasSuffix(normalized, "/layer-separation") { + return layerSeparationGeneratorName + } return name } diff --git a/server/internal/application/layer_separation.go b/server/internal/application/layer_separation.go new file mode 100644 index 0000000..3f41d86 --- /dev/null +++ b/server/internal/application/layer_separation.go @@ -0,0 +1,892 @@ +package application + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "image" + "image/color" + "image/draw" + "image/png" + "math" + "sort" + "strconv" + "strings" + "time" + + "img_infinite_canvas/internal/domain/design" +) + +const ( + layerSeparationForegroundLabel = "fg_image" + layerSeparationBackgroundLabel = "bg_image" + layerSeparationTextLabel = "text_render_data" +) + +type layerSeparationResult struct { + CanvasWidth int64 + CanvasHeight int64 + BackgroundID string + BackgroundURL string + Foregrounds []layerSeparationForeground + TextExtraction design.TextExtraction + Artifacts []design.GeneratorTaskArtifact +} + +type layerSeparationForeground struct { + ID string + URL string + BBox []int64 + Width int64 + Height int64 + Mask *image.Alpha + Rect image.Rectangle +} + +type layerSeparationForegroundCandidate struct { + BBox []int64 + Width int64 + Height int64 + Mask *image.Alpha + Rect image.Rectangle + Image image.Image +} + +func (s *DesignService) completeLayerSeparationGeneration(ctx context.Context, projectID string, threadID string, target design.Node, req design.NodeActionRequest) error { + resultNodeID := nodeActionResultNodeID(target, req) + if err := s.addGenerationStep(ctx, projectID, threadID, "tool", "图层分离", "正在拆分底图、前景元素和可编辑文字层。"); err != nil { + return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err) + } + + resultTarget := target + resultTarget.ID = resultNodeID + result, err := s.createLayerSeparationResult(ctx, projectID, resultTarget, req) + if err != nil { + return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err) + } + + project, err := s.repo.Get(ctx, projectID) + if err != nil { + return err + } + nodeIndex := findNodeIndex(project.Nodes, resultNodeID) + if nodeIndex < 0 { + project.Nodes = append(project.Nodes, newLayerSeparationPlaceholder(target)) + nodeIndex = len(project.Nodes) - 1 + project.Nodes[nodeIndex].ID = resultNodeID + } + + frameNode := project.Nodes[nodeIndex] + if frameNode.Width <= 0 { + frameNode.Width = float64(result.CanvasWidth) + } + if frameNode.Height <= 0 { + frameNode.Height = float64(result.CanvasHeight) + } + frameNode.Type = design.NodeTypeFrame + frameNode.Title = "Frame" + frameNode.Content = "" + frameNode.Tone = "frame" + frameNode.Status = "success" + frameNode.ParentID = "" + frameNode.LayerRole = "manual-frame" + frameNode.FillColor = "transparent" + frameNode.StrokeColor = "#C7C7C7" + frameNode.StrokeWidth = 1 + frameNode.StrokeStyle = "solid" + frameNode.MockupSurface = design.MockupSurface{} + frameNode.MockupModel = design.MockupModel{} + frameNode.MockupSourceContent = "" + + now := s.now() + backgroundNode := newLayerSeparationBackgroundNode(frameNode, result) + foregroundNodes := newForegroundImageLayers(frameNode, result.Foregrounds, result.CanvasWidth, result.CanvasHeight) + editableTextNodes := newEditableTextLayers(frameNode, result.TextExtraction) + replacementNodes := make([]design.Node, 0, len(foregroundNodes)+len(editableTextNodes)+2) + replacementNodes = append(replacementNodes, backgroundNode, frameNode) + replacementNodes = append(replacementNodes, foregroundNodes...) + replacementNodes = append(replacementNodes, editableTextNodes...) + project.Nodes = append(project.Nodes[:nodeIndex], append(replacementNodes, project.Nodes[nodeIndex+1:]...)...) + project.Messages = append(project.Messages, layerSeparationArtifactsMessage(threadID, result.Artifacts, now.Add(-time.Millisecond))) + project.Messages = append(project.Messages, design.Message{ + ID: newID(), + Role: "assistant", + Type: "assistant", + Title: nodeActionDoneTitle(req.Action), + Content: nodeActionDoneContent(req.Action, req.Prompt), + ThreadID: threadID, + ActionID: threadID, + StepID: newID(), + Name: "canvas_action", + ToolHint: "canvas_action", + Status: "success", + CreatedAt: now, + }) + project.Connections = nil + project.Status = projectStatusAfterCanvasTask(project.Nodes, design.StatusReady) + project.UpdatedAt = now + project = s.refreshCanvasSnapshot(project, now) + + return s.repo.Save(ctx, project) +} + +func (s *DesignService) createLayerSeparationResult(ctx context.Context, projectID string, target design.Node, req design.NodeActionRequest) (layerSeparationResult, error) { + sourceData, sourceContentType, err := loadImageContent(ctx, target.Content) + if err != nil { + return layerSeparationResult{}, err + } + source, _, err := image.Decode(bytes.NewReader(sourceData)) + if err != nil { + return layerSeparationResult{}, err + } + bounds := source.Bounds() + canvasWidth := int64(bounds.Dx()) + canvasHeight := int64(bounds.Dy()) + canvas := image.NewRGBA(bounds) + draw.Draw(canvas, bounds, source, bounds.Min, draw.Src) + + extraction, err := s.extractLayerSeparationText(ctx, target, req) + if err != nil { + return layerSeparationResult{}, err + } + for _, layer := range extraction.Layers { + removeTextLayerFromImage(canvas, layer) + } + + foregrounds := make([]layerSeparationForeground, 0, 1) + if fullForeground, elementForegrounds, ok := s.createLayerSeparationForegroundCandidates(ctx, target, sourceData, sourceContentType, bounds, extraction); ok { + if len(extraction.Layers) == 0 { + if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, fullForeground.Mask, fullForeground.Rect); ok { + if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok { + foregrounds = append(foregrounds, foreground) + } + canvas = plainBackground + } + } + if len(foregrounds) == 0 { + for _, candidate := range elementForegrounds { + foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, candidate) + if !ok { + continue + } + removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect) + foregrounds = append(foregrounds, foreground) + } + } + if len(foregrounds) == 0 && len(extraction.Layers) == 0 { + if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok { + removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect) + foregrounds = append(foregrounds, foreground) + } + } + if len(extraction.Layers) == 0 && len(foregrounds) == 1 { + if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, foregrounds[0].Mask, foregrounds[0].Rect); ok { + canvas = plainBackground + } + } + } + + backgroundID := newID() + backgroundURL, backgroundSize, err := s.persistLayerSeparationImage(ctx, projectID, backgroundID, "background", canvas) + if err != nil { + return layerSeparationResult{}, err + } + + textRenderData, err := layerSeparationTextRenderData(extraction, canvasWidth, canvasHeight) + if err != nil { + return layerSeparationResult{}, err + } + + artifacts := make([]design.GeneratorTaskArtifact, 0, len(foregrounds)+2) + for _, foreground := range foregrounds { + artifacts = append(artifacts, design.GeneratorTaskArtifact{ + Type: "image", + Content: foreground.URL, + Metadata: design.GeneratorTaskArtifactMetadata{ + ArtifactID: foreground.ID, + BBox: append([]int64(nil), foreground.BBox...), + Format: generatorArtifactFormat("", foreground.URL), + GeneratorName: layerSeparationGeneratorName, + Height: foreground.Height, + Label: layerSeparationForegroundLabel, + NodeID: foreground.ID, + NodeName: "Foreground Layer", + SizeBytes: 0, + Width: foreground.Width, + }, + }) + } + artifacts = append(artifacts, design.GeneratorTaskArtifact{ + Type: "image", + Content: backgroundURL, + Metadata: design.GeneratorTaskArtifactMetadata{ + ArtifactID: backgroundID, + Format: generatorArtifactFormat("", backgroundURL), + GeneratorName: layerSeparationGeneratorName, + Height: canvasHeight, + Label: layerSeparationBackgroundLabel, + NodeID: backgroundID, + NodeName: "Image", + SizeBytes: backgroundSize, + Width: canvasWidth, + }, + }, design.GeneratorTaskArtifact{ + Type: "text", + Content: textRenderData, + Metadata: design.GeneratorTaskArtifactMetadata{ + ArtifactID: newID(), + GeneratorName: layerSeparationGeneratorName, + Height: canvasHeight, + Label: layerSeparationTextLabel, + Width: canvasWidth, + }, + }) + + return layerSeparationResult{ + CanvasWidth: canvasWidth, + CanvasHeight: canvasHeight, + BackgroundID: backgroundID, + BackgroundURL: backgroundURL, + Foregrounds: foregrounds, + TextExtraction: extraction, + Artifacts: artifacts, + }, nil +} + +func (s *DesignService) extractLayerSeparationText(ctx context.Context, target design.Node, req design.NodeActionRequest) (design.TextExtraction, error) { + if extraction, ok, err := textExtractionFromOptions(req.Options); ok || err != nil { + return extraction, err + } + if s.textExtractor == nil { + return design.TextExtraction{}, nil + } + return s.extractTextExtraction(ctx, target, req) +} + +func (s *DesignService) createLayerSeparationForegroundCandidates(ctx context.Context, target design.Node, sourceData []byte, sourceContentType string, bounds image.Rectangle, extraction design.TextExtraction) (layerSeparationForegroundCandidate, []layerSeparationForegroundCandidate, bool) { + if s.backgroundRemover == nil { + return layerSeparationForegroundCandidate{}, nil, false + } + removed, err := s.backgroundRemover.RemoveBackground(ctx, design.BackgroundRemovalRequest{ + Image: sourceData, + ContentType: sourceContentType, + Width: bounds.Dx(), + Height: bounds.Dy(), + }) + if err != nil || len(removed.Image) == 0 { + return layerSeparationForegroundCandidate{}, nil, false + } + foregroundImage, _, err := image.Decode(bytes.NewReader(removed.Image)) + if err != nil { + return layerSeparationForegroundCandidate{}, nil, false + } + foreground := image.NewNRGBA(foregroundImage.Bounds()) + draw.Draw(foreground, foreground.Bounds(), foregroundImage, foregroundImage.Bounds().Min, draw.Src) + eraseTextLayersFromForeground(foreground, extraction) + foregroundBounds := foreground.Bounds() + rect, ok := alphaContentBounds(foreground, foregroundBounds, 12) + if !ok { + return layerSeparationForegroundCandidate{}, nil, false + } + mask, count := alphaMask(foreground, foregroundBounds, 12) + if count == 0 { + return layerSeparationForegroundCandidate{}, nil, false + } + full := newLayerSeparationForegroundCandidate(foreground, mask, rect, foregroundBounds) + return full, layerSeparationElementCandidates(foreground, foregroundBounds), true +} + +func (s *DesignService) persistLayerSeparationForeground(ctx context.Context, projectID string, candidate layerSeparationForegroundCandidate) (layerSeparationForeground, bool) { + if candidate.Image == nil || candidate.Mask == nil || candidate.Rect.Empty() || len(candidate.BBox) < 4 { + return layerSeparationForeground{}, false + } + foregroundID := newID() + foregroundURL, _, err := s.persistLayerSeparationImage(ctx, projectID, foregroundID, "foreground", candidate.Image) + if err != nil { + return layerSeparationForeground{}, false + } + return layerSeparationForeground{ + ID: foregroundID, + URL: foregroundURL, + BBox: append([]int64(nil), candidate.BBox...), + Width: candidate.Width, + Height: candidate.Height, + Mask: candidate.Mask, + Rect: candidate.Rect, + }, true +} + +func newLayerSeparationForegroundCandidate(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle, bounds image.Rectangle) layerSeparationForegroundCandidate { + return layerSeparationForegroundCandidate{ + BBox: []int64{ + int64(rect.Min.X - bounds.Min.X), + int64(rect.Min.Y - bounds.Min.Y), + int64(rect.Max.X - bounds.Min.X), + int64(rect.Max.Y - bounds.Min.Y), + }, + Width: int64(rect.Dx()), + Height: int64(rect.Dy()), + Mask: mask, + Rect: rect, + Image: foregroundImageFromMask(foreground, mask, rect), + } +} + +func eraseTextLayersFromForeground(foreground *image.NRGBA, extraction design.TextExtraction) { + if foreground == nil || len(extraction.Layers) == 0 { + return + } + bounds := foreground.Bounds() + if bounds.Dx()*bounds.Dy() < 4096 { + return + } + for _, layer := range extraction.Layers { + rect := normalizedTextMaskRect(bounds, layer) + if rect.Empty() { + continue + } + padding := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.14)), 2, 18) + rect = image.Rect(rect.Min.X-padding, rect.Min.Y-padding, rect.Max.X+padding, rect.Max.Y+padding).Intersect(bounds) + for y := rect.Min.Y; y < rect.Max.Y; y++ { + for x := rect.Min.X; x < rect.Max.X; x++ { + offset := foreground.PixOffset(x, y) + foreground.Pix[offset+3] = 0 + } + } + } +} + +func layerSeparationElementCandidates(foreground *image.NRGBA, bounds image.Rectangle) []layerSeparationForegroundCandidate { + if foreground == nil || bounds.Empty() { + return nil + } + width := bounds.Dx() + height := bounds.Dy() + visited := make([]bool, width*height) + components := make([]layerSeparationForegroundCandidate, 0, 4) + indexOf := func(x int, y int) int { + return (y-bounds.Min.Y)*width + (x - bounds.Min.X) + } + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + index := indexOf(x, y) + if visited[index] || foreground.NRGBAAt(x, y).A <= 12 { + visited[index] = true + continue + } + mask, rect, count := layerSeparationComponentMask(foreground, bounds, visited, x, y, indexOf) + if !keepLayerSeparationElement(rect, count, bounds) { + continue + } + components = append(components, newLayerSeparationForegroundCandidate(foreground, mask, rect, bounds)) + } + } + sort.Slice(components, func(i int, j int) bool { + leftArea := components[i].Rect.Dx() * components[i].Rect.Dy() + rightArea := components[j].Rect.Dx() * components[j].Rect.Dy() + return leftArea > rightArea + }) + if len(components) > 12 { + components = components[:12] + } + return components +} + +func layerSeparationComponentMask(foreground *image.NRGBA, bounds image.Rectangle, visited []bool, startX int, startY int, indexOf func(int, int) int) (*image.Alpha, image.Rectangle, int) { + mask := image.NewAlpha(bounds) + stack := []image.Point{{X: startX, Y: startY}} + minX, minY := startX, startY + maxX, maxY := startX+1, startY+1 + count := 0 + for len(stack) > 0 { + point := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if point.X < bounds.Min.X || point.X >= bounds.Max.X || point.Y < bounds.Min.Y || point.Y >= bounds.Max.Y { + continue + } + index := indexOf(point.X, point.Y) + if visited[index] { + continue + } + visited[index] = true + if foreground.NRGBAAt(point.X, point.Y).A <= 12 { + continue + } + mask.SetAlpha(point.X, point.Y, color.Alpha{A: 255}) + count++ + if point.X < minX { + minX = point.X + } + if point.Y < minY { + minY = point.Y + } + if point.X+1 > maxX { + maxX = point.X + 1 + } + if point.Y+1 > maxY { + maxY = point.Y + 1 + } + stack = append(stack, + image.Point{X: point.X + 1, Y: point.Y}, + image.Point{X: point.X - 1, Y: point.Y}, + image.Point{X: point.X, Y: point.Y + 1}, + image.Point{X: point.X, Y: point.Y - 1}, + ) + } + return mask, image.Rect(minX, minY, maxX, maxY), count +} + +func keepLayerSeparationElement(rect image.Rectangle, count int, bounds image.Rectangle) bool { + if rect.Empty() || count <= 0 { + return false + } + totalArea := bounds.Dx() * bounds.Dy() + if totalArea <= 0 { + return false + } + if totalArea < 4096 { + return true + } + bboxArea := rect.Dx() * rect.Dy() + minArea := maxInt(96, totalArea/6000) + if count < minArea || bboxArea < minArea { + return false + } + if float64(bboxArea)/float64(totalArea) > 0.18 { + return false + } + if float64(rect.Dx())/float64(bounds.Dx()) > 0.62 || float64(rect.Dy())/float64(bounds.Dy()) > 0.62 { + return false + } + return true +} + +func foregroundImageFromMask(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle) image.Image { + cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy())) + for y := rect.Min.Y; y < rect.Max.Y; y++ { + for x := rect.Min.X; x < rect.Max.X; x++ { + if mask.AlphaAt(x, y).A == 0 { + continue + } + cropped.SetNRGBA(x-rect.Min.X, y-rect.Min.Y, foreground.NRGBAAt(x, y)) + } + } + return cropped +} + +func (s *DesignService) persistLayerSeparationImage(ctx context.Context, projectID string, nodeID string, suffix string, img image.Image) (string, int64, error) { + var output bytes.Buffer + if err := png.Encode(&output, img); err != nil { + return "", 0, err + } + if s.assets == nil { + data := output.Bytes() + return "data:image/png;base64," + base64.StdEncoding.EncodeToString(data), int64(len(data)), nil + } + data, contentType, err := encodeRasterAssetAsWebP(output.Bytes(), "image/png") + if err != nil { + return "", 0, err + } + object, err := s.assets.PutObject(ctx, design.AssetObjectRequest{ + FileName: fmt.Sprintf("%s-%s-%s.png", projectID, nodeID, suffix), + ContentType: contentType, + Data: data, + }) + if err != nil { + return "", 0, err + } + if strings.TrimSpace(object.PublicURL) == "" { + return "", 0, fmt.Errorf("layer separation upload returned empty public URL") + } + return object.PublicURL, int64(len(data)), nil +} + +func newLayerSeparationBackgroundNode(frame design.Node, result layerSeparationResult) design.Node { + return design.Node{ + ID: result.BackgroundID, + Type: design.NodeTypeImage, + Title: "Image", + X: frame.X, + Y: frame.Y, + Width: frame.Width, + Height: frame.Height, + Content: result.BackgroundURL, + Tone: "visual", + Status: "success", + LayerRole: "clean-image", + } +} + +func newForegroundImageLayers(target design.Node, foregrounds []layerSeparationForeground, canvasWidth int64, canvasHeight int64) []design.Node { + if len(foregrounds) == 0 || canvasWidth <= 0 || canvasHeight <= 0 { + return nil + } + nodes := make([]design.Node, 0, len(foregrounds)) + for index, foreground := range foregrounds { + if len(foreground.BBox) < 4 || strings.TrimSpace(foreground.URL) == "" { + continue + } + x1 := clampFloat(float64(foreground.BBox[0])/float64(canvasWidth), 0, 1) + y1 := clampFloat(float64(foreground.BBox[1])/float64(canvasHeight), 0, 1) + x2 := clampFloat(float64(foreground.BBox[2])/float64(canvasWidth), x1, 1) + y2 := clampFloat(float64(foreground.BBox[3])/float64(canvasHeight), y1, 1) + width := target.Width * (x2 - x1) + height := target.Height * (y2 - y1) + if width <= 1 || height <= 1 { + continue + } + nodes = append(nodes, design.Node{ + ID: foreground.ID, + Type: design.NodeTypeImage, + Title: fmt.Sprintf("Foreground Layer %d", index+1), + X: target.X + target.Width*x1, + Y: target.Y + target.Height*y1, + Width: width, + Height: height, + Content: foreground.URL, + Tone: "separated-element", + Status: "success", + ParentID: target.ID, + LayerRole: "foreground-image", + }) + } + return nodes +} + +func layerSeparationArtifactsMessage(threadID string, artifacts []design.GeneratorTaskArtifact, createdAt time.Time) design.Message { + content, _ := json.Marshal(toLayerSeparationArtifactWire(artifacts)) + return design.Message{ + ID: newID(), + Role: "tool", + Type: "tool_use", + Title: layerSeparationMessageName, + Content: string(content), + ThreadID: threadID, + ActionID: threadID, + StepID: newID(), + ToolCallID: "call_" + strings.ReplaceAll(newID(), "-", ""), + Name: layerSeparationMessageName, + ToolHint: layerSeparationMessageName, + Status: "success", + CreatedAt: createdAt, + } +} + +func toLayerSeparationArtifactWire(artifacts []design.GeneratorTaskArtifact) []map[string]any { + items := make([]map[string]any, 0, len(artifacts)) + for _, artifact := range artifacts { + items = append(items, map[string]any{ + "type": artifact.Type, + "content": artifact.Content, + "metadata": map[string]any{ + "artifact_id": artifact.Metadata.ArtifactID, + "bbox": artifact.Metadata.BBox, + "format": artifact.Metadata.Format, + "generator_name": artifact.Metadata.GeneratorName, + "height": artifact.Metadata.Height, + "label": artifact.Metadata.Label, + "node_id": artifact.Metadata.NodeID, + "node_name": artifact.Metadata.NodeName, + "size_bytes": artifact.Metadata.SizeBytes, + "width": artifact.Metadata.Width, + }, + }) + } + return items +} + +func removeForegroundMaskFromImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) { + if mask == nil || rect.Empty() { + return + } + bounds := img.Bounds() + rect = rect.Intersect(bounds) + if rect.Empty() { + return + } + radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.025)), 2, 10) + mask, count := dilateTextMask(mask, rect, radius) + if count == 0 { + return + } + source := image.NewRGBA(bounds) + draw.Draw(source, bounds, img, bounds.Min, draw.Src) + for y := rect.Min.Y; y < rect.Max.Y; y++ { + for x := rect.Min.X; x < rect.Max.X; x++ { + if mask.AlphaAt(x, y).A == 0 { + continue + } + img.SetRGBA(x, y, estimatedRectBackgroundColor(source, rect, x, y)) + } + } + softenMaskedPixels(img, mask, rect) +} + +func plainLayerSeparationBackgroundImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) (*image.RGBA, bool) { + if img == nil || mask == nil || rect.Empty() { + return nil, false + } + bounds := img.Bounds() + if bounds.Empty() { + return nil, false + } + radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.02)), 3, 18) + expandedMask, _ := dilateTextMask(mask, bounds, radius) + stats, ok := unmaskedColorStats(img, expandedMask, bounds) + if !ok || stats.StdDev > 24 { + return nil, false + } + bg := image.NewRGBA(bounds) + fill := colorRGBA(int(math.Round(stats.R)), int(math.Round(stats.G)), int(math.Round(stats.B)), int(math.Round(stats.A))) + draw.Draw(bg, bounds, &image.Uniform{C: fill}, image.Point{}, draw.Src) + return bg, true +} + +type layerSeparationColorStats struct { + Count int64 + R float64 + G float64 + B float64 + A float64 + StdDev float64 +} + +func unmaskedColorStats(img *image.RGBA, mask *image.Alpha, bounds image.Rectangle) (layerSeparationColorStats, bool) { + stride := clampInt(int(math.Ceil(float64(maxInt(bounds.Dx(), bounds.Dy()))/640)), 1, 12) + var count int64 + var r, g, b, a float64 + var rr, gg, bb float64 + for y := bounds.Min.Y; y < bounds.Max.Y; y += stride { + for x := bounds.Min.X; x < bounds.Max.X; x += stride { + if mask.AlphaAt(x, y).A != 0 { + continue + } + c := img.RGBAAt(x, y) + cfR := float64(c.R) + cfG := float64(c.G) + cfB := float64(c.B) + cfA := float64(c.A) + r += cfR + g += cfG + b += cfB + a += cfA + rr += cfR * cfR + gg += cfG * cfG + bb += cfB * cfB + count++ + } + } + if count < 32 { + return layerSeparationColorStats{}, false + } + divisor := float64(count) + meanR := r / divisor + meanG := g / divisor + meanB := b / divisor + varR := math.Max(0, rr/divisor-meanR*meanR) + varG := math.Max(0, gg/divisor-meanG*meanG) + varB := math.Max(0, bb/divisor-meanB*meanB) + return layerSeparationColorStats{ + Count: count, + R: meanR, + G: meanG, + B: meanB, + A: a / divisor, + StdDev: math.Sqrt((varR + varG + varB) / 3), + }, true +} + +func alphaContentBounds(img image.Image, bounds image.Rectangle, threshold uint8) (image.Rectangle, bool) { + minX, minY := bounds.Max.X, bounds.Max.Y + maxX, maxY := bounds.Min.X, bounds.Min.Y + found := false + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + if alphaAt(img, x, y) <= threshold { + continue + } + if x < minX { + minX = x + } + if y < minY { + minY = y + } + if x+1 > maxX { + maxX = x + 1 + } + if y+1 > maxY { + maxY = y + 1 + } + found = true + } + } + if !found { + return image.Rectangle{}, false + } + return image.Rect(minX, minY, maxX, maxY), true +} + +func alphaMask(img image.Image, bounds image.Rectangle, threshold uint8) (*image.Alpha, int) { + mask := image.NewAlpha(bounds) + count := 0 + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + if alphaAt(img, x, y) <= threshold { + continue + } + mask.SetAlpha(x, y, color.Alpha{A: 255}) + count++ + } + } + return mask, count +} + +func alphaAt(img image.Image, x int, y int) uint8 { + _, _, _, a := img.At(x, y).RGBA() + return uint8(a >> 8) +} + +func cropImage(img image.Image, rect image.Rectangle) image.Image { + cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy())) + draw.Draw(cropped, cropped.Bounds(), img, rect.Min, draw.Src) + return cropped +} + +type layerSeparationTextDocument struct { + Layers []layerSeparationTextLayer `json:"layers"` + CanvasSize layerSeparationCanvasSize `json:"canvas_size"` +} + +type layerSeparationCanvasSize struct { + Width int64 `json:"width"` + Height int64 `json:"height"` +} + +type layerSeparationTextLayer struct { + TextInfo layerSeparationTextInfo `json:"text_info"` + X float64 `json:"x"` + Y float64 `json:"y"` + W float64 `json:"w"` + H float64 `json:"h"` +} + +type layerSeparationTextInfo struct { + Text string `json:"text"` + FontSizePx int64 `json:"font_size_px"` + FontFamily string `json:"font_family"` + FontWeight any `json:"font_weight"` + FontStyle string `json:"font_style"` + ColorCSS string `json:"color_css"` + TextAlign string `json:"text_align"` + Leading float64 `json:"leading"` + Tracking float64 `json:"tracking"` + Rotation float64 `json:"rotation"` + LeadingTrim string `json:"leading_trim"` +} + +func layerSeparationTextRenderData(extraction design.TextExtraction, canvasWidth int64, canvasHeight int64) (string, error) { + doc := layerSeparationTextDocument{ + Layers: make([]layerSeparationTextLayer, 0, len(extraction.Layers)), + CanvasSize: layerSeparationCanvasSize{ + Width: canvasWidth, + Height: canvasHeight, + }, + } + for _, layer := range extraction.Layers { + text := strings.TrimSpace(layer.Text) + if text == "" { + continue + } + x := clamp01(layer.X) + y := clamp01(layer.Y) + width := clamp01(layer.Width) + height := clamp01(layer.Height) + if x+width > 1 { + width = 1 - x + } + if y+height > 1 { + height = 1 - y + } + if width <= 0 || height <= 0 { + continue + } + fontSize := layer.FontSize + if fontSize <= 0 { + fontSize = math.Max(10, float64(canvasHeight)*height*0.72) + } + lineHeight := layer.LineHeight + if lineHeight <= 0 { + lineHeight = 1.2 + } + fontFamily := strings.TrimSpace(layer.FontFamily) + if fontFamily == "" { + fontFamily = "Inter" + } + doc.Layers = append(doc.Layers, layerSeparationTextLayer{ + TextInfo: layerSeparationTextInfo{ + Text: text, + FontSizePx: int64(math.Round(fontSize)), + FontFamily: fontFamily, + FontWeight: layerSeparationFontWeight(layer.FontWeight), + FontStyle: layerSeparationFontStyle(layer.FontWeight), + ColorCSS: layerSeparationColorCSS(layer.Color), + TextAlign: layerSeparationTextAlign(layer.TextAlign), + Leading: lineHeight, + Tracking: layer.LetterSpacing, + Rotation: layer.Rotation, + LeadingTrim: "CAP_HEIGHT", + }, + X: x * float64(canvasWidth), + Y: y * float64(canvasHeight), + W: width * float64(canvasWidth), + H: height * float64(canvasHeight), + }) + } + data, err := json.Marshal(doc) + if err != nil { + return "", err + } + return string(data), nil +} + +func layerSeparationFontWeight(value string) any { + value = strings.TrimSpace(value) + if value == "" { + return 400 + } + if weight, err := strconv.Atoi(value); err == nil { + return weight + } + return value +} + +func layerSeparationFontStyle(value string) string { + if strings.Contains(strings.ToLower(strings.TrimSpace(value)), "italic") { + return "italic" + } + return "normal" +} + +func layerSeparationTextAlign(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "left": + return "start" + case "right": + return "end" + case "center": + return "center" + default: + return "start" + } +} + +func layerSeparationColorCSS(value string) string { + parsed, ok := parseCanvasColor(value) + if !ok { + return "rgba(25, 25, 25, 1.0)" + } + alpha := float64(parsed.A) / 255 + return fmt.Sprintf("rgba(%d, %d, %d, %.3g)", parsed.R, parsed.G, parsed.B, alpha) +} diff --git a/server/internal/application/layer_separation_test.go b/server/internal/application/layer_separation_test.go new file mode 100644 index 0000000..6a7bca6 --- /dev/null +++ b/server/internal/application/layer_separation_test.go @@ -0,0 +1,410 @@ +package application + +import ( + "bytes" + "context" + "image" + "image/color" + "image/png" + "testing" + "time" + + "img_infinite_canvas/internal/domain/design" + "img_infinite_canvas/internal/infrastructure/memory" +) + +func TestRunEditElementsSeparatesBackgroundForegroundAndTextLayers(t *testing.T) { + ctx := context.Background() + repo := memory.NewProjectRepository() + assets := newFakeAssetStorage() + queue := &fakeJobQueue{} + extractor := &countingTextExtractor{ + result: design.TextExtraction{ + Layers: []design.ExtractedTextLayer{ + {Text: "Hi", X: 0.5, Y: 0.05, Width: 0.2, Height: 0.1, FontSize: 18, FontFamily: "Inter", FontWeight: "700", Color: "#111111"}, + }, + }, + } + remover := &fakeBackgroundRemover{result: design.BackgroundRemoval{ + Image: testLayerSeparationForegroundPNG(t), + ContentType: "image/png", + Width: 4, + Height: 4, + }} + service := NewDesignService(repo, nil, assets, nil, extractor) + service.SetJobQueue(queue) + service.SetBackgroundRemover(remover) + + source := testLayerSeparationSourceDataURL() + project := design.Project{ + ID: "project-layer-separation", + Title: "Layer separation", + Status: design.StatusReady, + UpdatedAt: time.Now(), + Nodes: []design.Node{{ + ID: "image-1", + Type: design.NodeTypeImage, + Title: "Source", + X: 100, + Y: 120, + Width: 400, + Height: 400, + Content: source, + Status: "success", + }}, + } + if err := repo.Save(ctx, project); err != nil { + t.Fatal(err) + } + + started, err := service.RunNodeActionAsync(ctx, project.ID, "image-1", design.NodeActionRequest{Action: "edit-elements"}) + if err != nil { + t.Fatal(err) + } + if len(queue.jobs) != 1 { + t.Fatalf("expected one queued job, got %#v", queue.jobs) + } + if len(started.Nodes) != 2 { + t.Fatalf("expected source plus separated placeholder, got %#v", started.Nodes) + } + if started.Nodes[0].Status != "success" || started.Nodes[0].Content != source { + t.Fatalf("expected source node to remain unchanged, got %#v", started.Nodes[0]) + } + if started.Nodes[1].Type != design.NodeTypeFrame || started.Nodes[1].Status != "generating" || started.Nodes[1].LayerRole != "manual-frame" || started.Nodes[1].ParentID != "" || started.Nodes[1].FillColor != "transparent" { + t.Fatalf("expected separated placeholder beside source, got %#v", started.Nodes[1]) + } + if started.Nodes[1].Content != source { + t.Fatalf("expected separated placeholder to preview source image, got %#v", started.Nodes[1]) + } + task, err := service.GeneratorTask(ctx, project.ID, started.LastThreadID) + if err != nil { + t.Fatal(err) + } + if task.GeneratorName != layerSeparationGeneratorName || task.Status != "submitted" { + t.Fatalf("expected submitted layer separation task, got %#v", task) + } + + if err := service.ProcessJob(ctx, queue.jobs[0]); err != nil { + t.Fatal(err) + } + saved, err := repo.Get(ctx, project.ID) + if err != nil { + t.Fatal(err) + } + if saved.Status != design.StatusReady { + t.Fatalf("expected ready project, got %s", saved.Status) + } + if len(saved.Nodes) != 5 { + t.Fatalf("expected original, background, frame, foreground, and text nodes, got %#v", saved.Nodes) + } + if saved.Nodes[0].Status != "success" || saved.Nodes[0].Content != source { + t.Fatalf("expected original source to remain unchanged, got %#v", saved.Nodes[0]) + } + background := saved.Nodes[1] + if background.Type != design.NodeTypeImage || background.LayerRole != "clean-image" || background.Status != "success" { + t.Fatalf("expected clean background node, got %#v", background) + } + if background.X != 532 || background.Y != 120 { + t.Fatalf("expected clean background at separated result origin, got %#v", background) + } + frame := saved.Nodes[2] + if frame.Type != design.NodeTypeFrame || frame.LayerRole != "manual-frame" || frame.Status != "success" { + t.Fatalf("expected separated frame beside source, got %#v", frame) + } + if frame.X != 532 || frame.Y != 120 { + t.Fatalf("expected frame beside source, got %#v", frame) + } + if frame.ParentID != "" || frame.FillColor != "transparent" { + t.Fatalf("expected top-level transparent separated frame, got %#v", frame) + } + if background.ParentID != "" { + t.Fatalf("expected clean background to be a bottom image layer outside frame, got %#v", background) + } + foreground := saved.Nodes[3] + if foreground.LayerRole != "foreground-image" || foreground.ParentID != frame.ID { + t.Fatalf("expected separated foreground node, got %#v", foreground) + } + if foreground.X != 632 || foreground.Y != 220 || foreground.Width != 200 || foreground.Height != 200 { + t.Fatalf("unexpected foreground geometry: %#v", foreground) + } + text := saved.Nodes[4] + if text.LayerRole != "editable-text" || text.ParentID != frame.ID || text.Content != "Hi" { + t.Fatalf("expected editable text node, got %#v", text) + } + + task, err = service.GeneratorTask(ctx, project.ID, started.LastThreadID) + if err != nil { + t.Fatal(err) + } + if task.Status != "completed" { + t.Fatalf("expected completed task, got %s", task.Status) + } + if len(task.Artifacts) != 3 { + t.Fatalf("expected foreground, background, and text artifacts, got %#v", task.Artifacts) + } + if task.Artifacts[0].Metadata.Label != layerSeparationForegroundLabel || task.Artifacts[1].Metadata.Label != layerSeparationBackgroundLabel || task.Artifacts[2].Metadata.Label != layerSeparationTextLabel { + t.Fatalf("unexpected artifact labels: %#v", task.Artifacts) + } + if got := task.Artifacts[0].Metadata.BBox; len(got) != 4 || got[0] != 1 || got[1] != 1 || got[2] != 3 || got[3] != 3 { + t.Fatalf("unexpected foreground bbox: %#v", got) + } + if task.Artifacts[2].Type != "text" || !bytes.Contains([]byte(task.Artifacts[2].Content), []byte(`"text":"Hi"`)) { + t.Fatalf("expected text render data artifact, got %#v", task.Artifacts[2]) + } +} + +func TestSubmitLayerSeparationTaskCompletesStandaloneArtifacts(t *testing.T) { + ctx := context.Background() + assets := newFakeAssetStorage() + extractor := &countingTextExtractor{ + result: design.TextExtraction{ + Layers: []design.ExtractedTextLayer{{Text: "Title", X: 0.1, Y: 0.1, Width: 0.3, Height: 0.1}}, + }, + } + remover := &fakeBackgroundRemover{result: design.BackgroundRemoval{ + Image: testLayerSeparationForegroundPNG(t), + ContentType: "image/png", + Width: 4, + Height: 4, + }} + service := NewDesignService(nil, nil, assets, nil, extractor) + service.SetBackgroundRemover(remover) + + task, err := service.SubmitGeneratorTask(ctx, design.GeneratorTaskSubmitRequest{ + ProjectID: "project-standalone-layer-separation", + GeneratorName: layerSeparationGeneratorName, + InputArgs: design.GeneratorTaskInputArgs{ + ImageURL: testLayerSeparationSourceDataURL(), + }, + }) + if err != nil { + t.Fatal(err) + } + completed := waitForStandaloneGeneratorTask(t, service, task.GeneratorTaskID) + if completed.Status != "completed" { + t.Fatalf("expected completed standalone task, got %#v", completed) + } + if completed.GeneratorName != layerSeparationGeneratorName || len(completed.Artifacts) != 3 { + t.Fatalf("unexpected standalone task: %#v", completed) + } + if completed.Artifacts[0].Metadata.Label != layerSeparationForegroundLabel || completed.Artifacts[2].Metadata.Label != layerSeparationTextLabel { + t.Fatalf("unexpected standalone artifacts: %#v", completed.Artifacts) + } +} + +func TestPlainLayerSeparationBackgroundUsesSolidColorWhenBackdropIsPlain(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 24, 24)) + for y := 0; y < 24; y++ { + for x := 0; x < 24; x++ { + img.SetRGBA(x, y, color.RGBA{R: 188, G: 185, B: 178, A: 255}) + } + } + mask := image.NewAlpha(img.Bounds()) + rect := image.Rect(8, 4, 16, 22) + for y := rect.Min.Y; y < rect.Max.Y; y++ { + for x := rect.Min.X; x < rect.Max.X; x++ { + img.SetRGBA(x, y, color.RGBA{R: 32, G: 26, B: 22, A: 255}) + mask.SetAlpha(x, y, color.Alpha{A: 255}) + } + } + + bg, ok := plainLayerSeparationBackgroundImage(img, mask, rect) + if !ok { + t.Fatal("expected plain background branch") + } + center := bg.RGBAAt(12, 12) + if center.R != 188 || center.G != 185 || center.B != 178 || center.A != 255 { + t.Fatalf("expected solid background color, got %#v", center) + } +} + +func TestPlainLayerSeparationBackgroundSkipsComplexBackdrop(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 24, 24)) + for y := 0; y < 24; y++ { + for x := 0; x < 24; x++ { + img.SetRGBA(x, y, color.RGBA{R: uint8(20 + x*8), G: uint8(40 + y*7), B: uint8(80 + (x+y)*3), A: 255}) + } + } + mask := image.NewAlpha(img.Bounds()) + rect := image.Rect(9, 6, 15, 20) + for y := rect.Min.Y; y < rect.Max.Y; y++ { + for x := rect.Min.X; x < rect.Max.X; x++ { + mask.SetAlpha(x, y, color.Alpha{A: 255}) + } + } + + if _, ok := plainLayerSeparationBackgroundImage(img, mask, rect); ok { + t.Fatal("expected complex background to use repair branch") + } +} + +func TestLayerSeparationPosterKeepsComplexBackgroundAndExtractsSmallElements(t *testing.T) { + ctx := context.Background() + extractor := &countingTextExtractor{ + result: design.TextExtraction{ + Layers: []design.ExtractedTextLayer{{ + Text: "SALE", + X: 0.1, + Y: 0.1, + Width: 0.35, + Height: 0.125, + FontSize: 12, + FontFamily: "Inter", + FontWeight: "700", + Color: "#111111", + }}, + }, + } + remover := &fakeBackgroundRemover{result: design.BackgroundRemoval{ + Image: testPosterForegroundPNG(t), + ContentType: "image/png", + Width: 80, + Height: 80, + }} + service := NewDesignService(nil, nil, nil, nil, extractor) + service.SetBackgroundRemover(remover) + + result, err := service.createLayerSeparationResult(ctx, "project-poster-layer-separation", design.Node{ + ID: "poster", + Type: design.NodeTypeImage, + Title: "Poster", + Content: testPosterSourceDataURL(), + Status: "success", + }, design.NodeActionRequest{Action: "edit-elements"}) + if err != nil { + t.Fatal(err) + } + if len(result.TextExtraction.Layers) != 1 { + t.Fatalf("expected editable text extraction to remain, got %#v", result.TextExtraction.Layers) + } + if len(result.Foregrounds) != 1 { + t.Fatalf("expected only the small decorative foreground to be extracted, got %#v", result.Foregrounds) + } + if got := result.Foregrounds[0].BBox; len(got) != 4 || got[0] != 54 || got[1] != 44 || got[2] != 66 || got[3] != 56 { + t.Fatalf("unexpected decorative foreground bbox: %#v", got) + } + + backgroundData, _, err := loadImageContent(ctx, result.BackgroundURL) + if err != nil { + t.Fatal(err) + } + background, _, err := image.Decode(bytes.NewReader(backgroundData)) + if err != nil { + t.Fatal(err) + } + topLeft := color.NRGBAModel.Convert(background.At(2, 2)).(color.NRGBA) + bottomRight := color.NRGBAModel.Convert(background.At(76, 76)).(color.NRGBA) + if topLeft == bottomRight { + t.Fatalf("expected complex poster background to be preserved, got solid-looking pixels %#v", topLeft) + } + textPixel := color.NRGBAModel.Convert(background.At(14, 12)).(color.NRGBA) + if textPixel.R < 32 && textPixel.G < 32 && textPixel.B < 32 { + t.Fatalf("expected text pixels to be removed from background, got %#v", textPixel) + } + ornamentPixel := color.NRGBAModel.Convert(background.At(59, 49)).(color.NRGBA) + if ornamentPixel.R > 200 && ornamentPixel.G < 80 && ornamentPixel.B < 80 { + t.Fatalf("expected extracted ornament to be removed from background, got %#v", ornamentPixel) + } + scenePixel := color.NRGBAModel.Convert(background.At(20, 48)).(color.NRGBA) + if scenePixel.B < 170 { + t.Fatalf("expected large poster scenery to stay on the background, got %#v", scenePixel) + } +} + +func waitForStandaloneGeneratorTask(t *testing.T, service *DesignService, taskID string) design.GeneratorTask { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + task, err := service.GeneratorTask(context.Background(), "", taskID) + if err != nil { + t.Fatal(err) + } + if task.Status == "completed" || task.Status == "failed" { + return task + } + time.Sleep(10 * time.Millisecond) + } + task, _ := service.GeneratorTask(context.Background(), "", taskID) + t.Fatalf("timed out waiting for standalone task: %#v", task) + return design.GeneratorTask{} +} + +func testLayerSeparationSourceDataURL() string { + img := image.NewNRGBA(image.Rect(0, 0, 4, 4)) + for y := 0; y < 4; y++ { + for x := 0; x < 4; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 240, G: 240, B: 240, A: 255}) + } + } + for y := 1; y < 3; y++ { + for x := 1; x < 3; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 200, G: 20, B: 30, A: 255}) + } + } + return testPNGDataURLFromImage(img) +} + +func testLayerSeparationForegroundPNG(t *testing.T) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, 4, 4)) + for y := 1; y < 3; y++ { + for x := 1; x < 3; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 200, G: 20, B: 30, A: 255}) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func testPosterSourceDataURL() string { + img := image.NewNRGBA(image.Rect(0, 0, 80, 80)) + for y := 0; y < 80; y++ { + for x := 0; x < 80; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(190 + x/3), G: uint8(186 + y/4), B: uint8(174 + (x+y)/8), A: 255}) + } + } + for y := 38; y < 70; y++ { + for x := 8; x < 50; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 64, G: 106, B: 204, A: 255}) + } + } + for y := 8; y < 18; y++ { + for x := 8; x < 36; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 17, G: 17, B: 17, A: 255}) + } + } + for y := 44; y < 56; y++ { + for x := 54; x < 66; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 225, G: 32, B: 44, A: 255}) + } + } + return testPNGDataURLFromImage(img) +} + +func testPosterForegroundPNG(t *testing.T) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, 80, 80)) + for y := 8; y < 18; y++ { + for x := 8; x < 36; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 17, G: 17, B: 17, A: 255}) + } + } + for y := 38; y < 70; y++ { + for x := 8; x < 50; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 64, G: 106, B: 204, A: 255}) + } + } + for y := 44; y < 56; y++ { + for x := 54; x < 66; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 225, G: 32, B: 44, A: 255}) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} diff --git a/server/internal/domain/design/agent.go b/server/internal/domain/design/agent.go index f05f975..d8afe62 100644 --- a/server/internal/domain/design/agent.go +++ b/server/internal/domain/design/agent.go @@ -107,14 +107,16 @@ type GeneratorTaskQueueInfo struct { } type GeneratorTaskArtifactMetadata struct { - ArtifactID string - Format string - Height int64 - Label string - NodeID string - NodeName string - SizeBytes int64 - Width int64 + ArtifactID string + BBox []int64 + Format string + GeneratorName string + Height int64 + Label string + NodeID string + NodeName string + SizeBytes int64 + Width int64 } type GeneratorTaskArtifact struct { diff --git a/server/internal/logic/mapper.go b/server/internal/logic/mapper.go index e731e44..caabbc6 100644 --- a/server/internal/logic/mapper.go +++ b/server/internal/logic/mapper.go @@ -133,14 +133,16 @@ func toAPIGeneratorTaskArtifacts(items []design.GeneratorTaskArtifact) []types.G Type: item.Type, Content: item.Content, Metadata: types.GeneratorTaskArtifactMetadata{ - ArtifactId: item.Metadata.ArtifactID, - Format: item.Metadata.Format, - Height: item.Metadata.Height, - Label: item.Metadata.Label, - NodeId: item.Metadata.NodeID, - NodeName: item.Metadata.NodeName, - SizeBytes: item.Metadata.SizeBytes, - Width: item.Metadata.Width, + ArtifactId: item.Metadata.ArtifactID, + BBox: append([]int64(nil), item.Metadata.BBox...), + Format: item.Metadata.Format, + GeneratorName: item.Metadata.GeneratorName, + Height: item.Metadata.Height, + Label: item.Metadata.Label, + NodeId: item.Metadata.NodeID, + NodeName: item.Metadata.NodeName, + SizeBytes: item.Metadata.SizeBytes, + Width: item.Metadata.Width, }, }) } diff --git a/server/internal/types/types.go b/server/internal/types/types.go index d5368de..fd053e5 100644 --- a/server/internal/types/types.go +++ b/server/internal/types/types.go @@ -417,14 +417,16 @@ type GeneratorTaskArtifact struct { } type GeneratorTaskArtifactMetadata struct { - ArtifactId string `json:"artifact_id,optional"` - Format string `json:"format,optional"` - Height int64 `json:"height,optional"` - Label string `json:"label,optional"` - NodeId string `json:"node_id,optional"` - NodeName string `json:"node_name,optional"` - SizeBytes int64 `json:"size_bytes,optional"` - Width int64 `json:"width,optional"` + ArtifactId string `json:"artifact_id,optional"` + BBox []int64 `json:"bbox,optional"` + Format string `json:"format,optional"` + GeneratorName string `json:"generator_name,optional"` + Height int64 `json:"height,optional"` + Label string `json:"label,optional"` + NodeId string `json:"node_id,optional"` + NodeName string `json:"node_name,optional"` + SizeBytes int64 `json:"size_bytes,optional"` + Width int64 `json:"width,optional"` } type GeneratorTaskData struct {