feat(server): add layer-separation generator task

Introduce the `layer-separation` generator (edit-elements node action)
that splits a source image into a clean background layer, foreground
image artifacts with bounding boxes, and editable text render data.

- New layer_separation.go builds the separated result and artifacts
- SubmitGeneratorTask routes layer-separation and move-object via a
  switch; standalone task completion + polling support
- Extend artifact metadata with BBox and GeneratorName across the
  domain, API types, mapper, .api schema, and API.md
- edit-elements gets a transparent manual-frame placeholder holding the
  separated foreground images and text layers beside the original

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-09 12:26:43 +08:00
parent 676e1c4c67
commit 039d166972
9 changed files with 1523 additions and 36 deletions
+19
View File
@@ -238,6 +238,25 @@ Returns the updated project, the assistant message, and generated nodes.
The generation path uses the configured design agent runner. The default is the `sky-valley/pi` Go agent adapter, with local desktop/MCP agent behavior disabled.
## Generator Tasks
`POST /api/generator/tasks`
```json
{
"cid": "client-request-id",
"project_id": "project-id",
"generator_name": "layer-separation",
"input_args": {
"image_url": "https://cdn.example.com/source.png"
}
}
```
`layer-separation` returns a task id that can be polled with `GET /api/generator/tasks?task_id=<id>&project_id=<project-id>`. Completed tasks return a clean background image, foreground image artifacts with `bbox`, and a `text_render_data` artifact for editable text reconstruction.
The canvas `edit-elements` node action uses the same separation path but writes the separated result beside the original image. The original image remains unchanged; the clean background is a bottom image layer, and a transparent frame above it holds the separated foreground images and editable text layers.
## Chat History Replay
`GET /api/projects/:id/history`
+11 -8
View File
@@ -670,14 +670,16 @@ type GeneratorTaskQueueInfo {
}
type GeneratorTaskArtifactMetadata {
ArtifactId string `json:"artifact_id,optional"`
Format string `json:"format,optional"`
Height int64 `json:"height,optional"`
Label string `json:"label,optional"`
NodeId string `json:"node_id,optional"`
NodeName string `json:"node_name,optional"`
SizeBytes int64 `json:"size_bytes,optional"`
Width int64 `json:"width,optional"`
ArtifactId string `json:"artifact_id,optional"`
BBox []int64 `json:"bbox,optional"`
Format string `json:"format,optional"`
GeneratorName string `json:"generator_name,optional"`
Height int64 `json:"height,optional"`
Label string `json:"label,optional"`
NodeId string `json:"node_id,optional"`
NodeName string `json:"node_name,optional"`
SizeBytes int64 `json:"size_bytes,optional"`
Width int64 `json:"width,optional"`
}
type GeneratorTaskArtifact {
@@ -1013,3 +1015,4 @@ service img_infinite_canvas-api {
@handler deleteAsset
delete /assets (DeleteAssetRequest) returns (DeleteAssetResponse)
}
+30 -3
View File
@@ -892,12 +892,14 @@ func (s *DesignService) RunNodeActionAsync(ctx context.Context, projectID string
now := s.now()
threadID := newID()
target := project.Nodes[nodeIndex]
if req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" {
if req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements" {
resultNode := newBackgroundRemovalPlaceholder(target)
if req.Action == "vectorize" {
resultNode = newVectorizePlaceholder(target)
} else if req.Action == "move-object" {
resultNode = newMoveObjectPlaceholder(target)
} else if req.Action == "edit-elements" {
resultNode = newLayerSeparationPlaceholder(target)
}
if req.Options == nil {
req.Options = make(map[string]string)
@@ -1845,7 +1847,10 @@ func (s *DesignService) completeNodeActionGeneration(ctx context.Context, projec
if req.Action == "vectorize" {
return s.completeVectorizeGeneration(ctx, projectID, threadID, target, req)
}
if req.Action == "edit-elements" || req.Action == "edit-text" {
if req.Action == "edit-elements" {
return s.completeLayerSeparationGeneration(ctx, projectID, threadID, target, req)
}
if req.Action == "edit-text" {
return s.completeEditableLayerGeneration(ctx, projectID, threadID, target, req)
}
editToolContent := "正在根据选中图片执行模型编辑,完成后会替换画布中的图片节点。"
@@ -4709,8 +4714,30 @@ func newMoveObjectPlaceholder(target design.Node) design.Node {
return result
}
func newLayerSeparationPlaceholder(target design.Node) design.Node {
result := target
result.ID = newID()
result.Type = design.NodeTypeFrame
result.Title = nodeActionWorkingTitle("edit-elements")
result.X = target.X + target.Width + 32
result.Y = target.Y
result.Content = target.Content
result.Tone = "frame"
result.Status = "generating"
result.ParentID = ""
result.LayerRole = "manual-frame"
result.FillColor = "transparent"
result.StrokeColor = "#C7C7C7"
result.StrokeWidth = 1
result.StrokeStyle = "solid"
result.MockupSurface = design.MockupSurface{}
result.MockupModel = design.MockupModel{}
result.MockupSourceContent = ""
return result
}
func nodeActionResultNodeID(target design.Node, req design.NodeActionRequest) string {
if (req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object") && len(req.Options) > 0 {
if (req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements") && len(req.Options) > 0 {
if id := strings.TrimSpace(req.Options[nodeActionResultNodeIDOption]); id != "" {
return id
}
+131 -1
View File
@@ -21,7 +21,9 @@ type generatorTaskMetadata struct {
const (
removeBackgroundGeneratorName = "removeBG"
imageVectorizerGeneratorName = "gpt-5.5-vectorizer"
layerSeparationGeneratorName = "layer-separation"
moveObjectGeneratorName = "move-object"
layerSeparationMessageName = "layer_separation"
)
func (s *DesignService) GeneratorTask(ctx context.Context, projectID string, taskID string) (design.GeneratorTask, error) {
@@ -72,10 +74,17 @@ func (s *DesignService) SubmitGeneratorTask(ctx context.Context, req design.Gene
if generatorName == "" {
generatorName = moveObjectGeneratorName
}
if normalizeGeneratorTaskName(generatorName) != moveObjectGeneratorName {
switch normalizeGeneratorTaskName(generatorName) {
case moveObjectGeneratorName:
return s.submitMoveObjectTask(ctx, req)
case layerSeparationGeneratorName:
return s.submitLayerSeparationTask(ctx, req)
default:
return design.GeneratorTask{}, fmt.Errorf("%w: unsupported generator_name %s", design.ErrInvalidInput, req.GeneratorName)
}
}
func (s *DesignService) submitMoveObjectTask(ctx context.Context, req design.GeneratorTaskSubmitRequest) (design.GeneratorTask, error) {
input := normalizeMoveObjectInput(req.InputArgs)
if strings.TrimSpace(input.ImageURL) == "" {
return design.GeneratorTask{}, fmt.Errorf("%w: input_args.image_url is required", design.ErrInvalidInput)
@@ -101,6 +110,53 @@ func (s *DesignService) SubmitGeneratorTask(ctx context.Context, req design.Gene
return task, nil
}
func (s *DesignService) submitLayerSeparationTask(ctx context.Context, req design.GeneratorTaskSubmitRequest) (design.GeneratorTask, error) {
input := normalizeLayerSeparationInput(req.InputArgs)
if strings.TrimSpace(input.ImageURL) == "" {
return design.GeneratorTask{}, fmt.Errorf("%w: input_args.image_url is required", design.ErrInvalidInput)
}
taskID := newID()
price := generatorTaskPrice(input, layerSeparationGeneratorName)
task := design.GeneratorTask{
GeneratorTaskID: taskID,
GeneratorName: layerSeparationGeneratorName,
Status: "submitted",
OriginalInputArgs: input,
QueueInfo: generatorTaskQueueInfo(layerSeparationGeneratorName, input, price),
}
s.storeStandaloneGeneratorTask(task)
go s.completeStandaloneLayerSeparationTask(taskID, strings.TrimSpace(req.ProjectID), input)
return task, nil
}
func (s *DesignService) completeStandaloneLayerSeparationTask(taskID string, projectID string, input design.GeneratorTaskInputArgs) {
ctx, cancel := context.WithTimeout(context.Background(), generationBudget)
defer cancel()
imageURL := strings.TrimSpace(input.ImageURL)
target := design.Node{
ID: "source",
Type: design.NodeTypeImage,
Title: "Source image",
Content: imageURL,
Status: "success",
}
result, err := s.createLayerSeparationResult(ctx, standaloneArtifactProjectID(projectID, taskID), target, design.NodeActionRequest{
Action: "edit-elements",
Prompt: strings.TrimSpace(input.Prompt),
})
if err != nil {
s.markStandaloneGeneratorTaskFailed(taskID, err)
return
}
if len(result.Artifacts) == 0 {
s.markStandaloneGeneratorTaskFailed(taskID, fmt.Errorf("layer separation returned no artifacts"))
return
}
s.completeStandaloneGeneratorTask(taskID, result.Artifacts)
}
func (s *DesignService) completeStandaloneMoveObjectTask(taskID string, projectID string, input design.GeneratorTaskInputArgs) {
ctx, cancel := context.WithTimeout(context.Background(), generationBudget)
defer cancel()
@@ -218,6 +274,17 @@ func normalizeMoveObjectInput(input design.GeneratorTaskInputArgs) design.Genera
return input
}
func normalizeLayerSeparationInput(input design.GeneratorTaskInputArgs) design.GeneratorTaskInputArgs {
if strings.TrimSpace(input.ImageURL) == "" && len(input.Image) > 0 {
input.ImageURL = strings.TrimSpace(input.Image[0])
}
input.ImageURL = strings.TrimSpace(input.ImageURL)
input.Prompt = strings.TrimSpace(input.Prompt)
input.Resolution = strings.TrimSpace(input.Resolution)
input.AspectRatio = strings.TrimSpace(input.AspectRatio)
return input
}
func normalizeBox(box design.NormalizedBox) design.NormalizedBox {
width := clampUnit(box.Width)
height := clampUnit(box.Height)
@@ -427,6 +494,7 @@ func generatorTaskMessageCompleted(message design.Message) bool {
}
return name == "canvas_action" ||
name == "text_extraction" ||
name == layerSeparationMessageName ||
name == "mockup_model" ||
strings.TrimSpace(message.Role) == "assistant"
}
@@ -470,6 +538,8 @@ func generatorTaskGeneratorNameFromMessages(messages []design.Message, taskID st
return "gpt-image-2"
case strings.Contains(strings.ToLower(message.Title), "rmbg"):
return removeBackgroundGeneratorName
case message.Name == layerSeparationMessageName:
return layerSeparationGeneratorName
case message.Name == "mockup_model":
return "mockup/model"
case message.Name == "text_extraction":
@@ -498,6 +568,10 @@ func generatorTaskArtifacts(messages []design.Message, taskID string) []design.G
if message.ThreadID != taskID || !isToolArtifactsMessage(message) {
continue
}
if message.Name == layerSeparationMessageName || message.ToolHint == layerSeparationMessageName {
artifacts = append(artifacts, parseLayerSeparationArtifacts(message.Content)...)
continue
}
artifacts = append(artifacts, parseGeneratorArtifacts(message.Content)...)
}
return artifacts
@@ -549,6 +623,55 @@ func parseGeneratorArtifacts(content string) []design.GeneratorTaskArtifact {
return artifacts
}
func parseLayerSeparationArtifacts(content string) []design.GeneratorTaskArtifact {
var raw []struct {
Type string `json:"type"`
Content string `json:"content"`
Metadata struct {
ArtifactID string `json:"artifact_id"`
BBox []int64 `json:"bbox"`
Format string `json:"format"`
GeneratorName string `json:"generator_name"`
Height int64 `json:"height"`
Label string `json:"label"`
NodeID string `json:"node_id"`
NodeName string `json:"node_name"`
SizeBytes int64 `json:"size_bytes"`
Width int64 `json:"width"`
} `json:"metadata"`
}
if err := json.Unmarshal([]byte(content), &raw); err != nil {
return nil
}
artifacts := make([]design.GeneratorTaskArtifact, 0, len(raw))
for _, item := range raw {
artifactType := strings.TrimSpace(item.Type)
if artifactType == "" {
continue
}
if strings.TrimSpace(item.Content) == "" && artifactType != "text" {
continue
}
artifacts = append(artifacts, design.GeneratorTaskArtifact{
Type: artifactType,
Content: item.Content,
Metadata: design.GeneratorTaskArtifactMetadata{
ArtifactID: item.Metadata.ArtifactID,
BBox: append([]int64(nil), item.Metadata.BBox...),
Format: item.Metadata.Format,
GeneratorName: item.Metadata.GeneratorName,
Height: item.Metadata.Height,
Label: item.Metadata.Label,
NodeID: item.Metadata.NodeID,
NodeName: item.Metadata.NodeName,
SizeBytes: item.Metadata.SizeBytes,
Width: item.Metadata.Width,
},
})
}
return artifacts
}
func generatorArtifactFormat(format string, imageURL string) string {
format = strings.Trim(strings.ToLower(strings.TrimSpace(format)), ".")
if format != "" {
@@ -605,6 +728,8 @@ func generatorTaskPrice(input design.GeneratorTaskInputArgs, generatorName strin
return 4
case imageVectorizerGeneratorName:
return 14
case layerSeparationGeneratorName:
return 8
case moveObjectGeneratorName:
return 5
}
@@ -668,6 +793,8 @@ func generatorTaskNameForNodeAction(req design.NodeActionRequest) string {
switch normalizeNodeAction(req.Action) {
case "remove-background":
return removeBackgroundGeneratorName
case "edit-elements":
return layerSeparationGeneratorName
case "edit-text":
return "gpt-5.4-mini"
case "move-object":
@@ -749,6 +876,9 @@ func normalizeGeneratorTaskName(generatorName string) string {
if normalized == "move-object" || normalized == "moveobject" {
return moveObjectGeneratorName
}
if normalized == "layer-separation" || normalized == "layer_separation" || normalized == "edit-elements" || strings.HasSuffix(normalized, "/layer-separation") {
return layerSeparationGeneratorName
}
return name
}
@@ -0,0 +1,892 @@
package application
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"image"
"image/color"
"image/draw"
"image/png"
"math"
"sort"
"strconv"
"strings"
"time"
"img_infinite_canvas/internal/domain/design"
)
const (
layerSeparationForegroundLabel = "fg_image"
layerSeparationBackgroundLabel = "bg_image"
layerSeparationTextLabel = "text_render_data"
)
type layerSeparationResult struct {
CanvasWidth int64
CanvasHeight int64
BackgroundID string
BackgroundURL string
Foregrounds []layerSeparationForeground
TextExtraction design.TextExtraction
Artifacts []design.GeneratorTaskArtifact
}
type layerSeparationForeground struct {
ID string
URL string
BBox []int64
Width int64
Height int64
Mask *image.Alpha
Rect image.Rectangle
}
type layerSeparationForegroundCandidate struct {
BBox []int64
Width int64
Height int64
Mask *image.Alpha
Rect image.Rectangle
Image image.Image
}
func (s *DesignService) completeLayerSeparationGeneration(ctx context.Context, projectID string, threadID string, target design.Node, req design.NodeActionRequest) error {
resultNodeID := nodeActionResultNodeID(target, req)
if err := s.addGenerationStep(ctx, projectID, threadID, "tool", "图层分离", "正在拆分底图、前景元素和可编辑文字层。"); err != nil {
return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err)
}
resultTarget := target
resultTarget.ID = resultNodeID
result, err := s.createLayerSeparationResult(ctx, projectID, resultTarget, req)
if err != nil {
return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err)
}
project, err := s.repo.Get(ctx, projectID)
if err != nil {
return err
}
nodeIndex := findNodeIndex(project.Nodes, resultNodeID)
if nodeIndex < 0 {
project.Nodes = append(project.Nodes, newLayerSeparationPlaceholder(target))
nodeIndex = len(project.Nodes) - 1
project.Nodes[nodeIndex].ID = resultNodeID
}
frameNode := project.Nodes[nodeIndex]
if frameNode.Width <= 0 {
frameNode.Width = float64(result.CanvasWidth)
}
if frameNode.Height <= 0 {
frameNode.Height = float64(result.CanvasHeight)
}
frameNode.Type = design.NodeTypeFrame
frameNode.Title = "Frame"
frameNode.Content = ""
frameNode.Tone = "frame"
frameNode.Status = "success"
frameNode.ParentID = ""
frameNode.LayerRole = "manual-frame"
frameNode.FillColor = "transparent"
frameNode.StrokeColor = "#C7C7C7"
frameNode.StrokeWidth = 1
frameNode.StrokeStyle = "solid"
frameNode.MockupSurface = design.MockupSurface{}
frameNode.MockupModel = design.MockupModel{}
frameNode.MockupSourceContent = ""
now := s.now()
backgroundNode := newLayerSeparationBackgroundNode(frameNode, result)
foregroundNodes := newForegroundImageLayers(frameNode, result.Foregrounds, result.CanvasWidth, result.CanvasHeight)
editableTextNodes := newEditableTextLayers(frameNode, result.TextExtraction)
replacementNodes := make([]design.Node, 0, len(foregroundNodes)+len(editableTextNodes)+2)
replacementNodes = append(replacementNodes, backgroundNode, frameNode)
replacementNodes = append(replacementNodes, foregroundNodes...)
replacementNodes = append(replacementNodes, editableTextNodes...)
project.Nodes = append(project.Nodes[:nodeIndex], append(replacementNodes, project.Nodes[nodeIndex+1:]...)...)
project.Messages = append(project.Messages, layerSeparationArtifactsMessage(threadID, result.Artifacts, now.Add(-time.Millisecond)))
project.Messages = append(project.Messages, design.Message{
ID: newID(),
Role: "assistant",
Type: "assistant",
Title: nodeActionDoneTitle(req.Action),
Content: nodeActionDoneContent(req.Action, req.Prompt),
ThreadID: threadID,
ActionID: threadID,
StepID: newID(),
Name: "canvas_action",
ToolHint: "canvas_action",
Status: "success",
CreatedAt: now,
})
project.Connections = nil
project.Status = projectStatusAfterCanvasTask(project.Nodes, design.StatusReady)
project.UpdatedAt = now
project = s.refreshCanvasSnapshot(project, now)
return s.repo.Save(ctx, project)
}
func (s *DesignService) createLayerSeparationResult(ctx context.Context, projectID string, target design.Node, req design.NodeActionRequest) (layerSeparationResult, error) {
sourceData, sourceContentType, err := loadImageContent(ctx, target.Content)
if err != nil {
return layerSeparationResult{}, err
}
source, _, err := image.Decode(bytes.NewReader(sourceData))
if err != nil {
return layerSeparationResult{}, err
}
bounds := source.Bounds()
canvasWidth := int64(bounds.Dx())
canvasHeight := int64(bounds.Dy())
canvas := image.NewRGBA(bounds)
draw.Draw(canvas, bounds, source, bounds.Min, draw.Src)
extraction, err := s.extractLayerSeparationText(ctx, target, req)
if err != nil {
return layerSeparationResult{}, err
}
for _, layer := range extraction.Layers {
removeTextLayerFromImage(canvas, layer)
}
foregrounds := make([]layerSeparationForeground, 0, 1)
if fullForeground, elementForegrounds, ok := s.createLayerSeparationForegroundCandidates(ctx, target, sourceData, sourceContentType, bounds, extraction); ok {
if len(extraction.Layers) == 0 {
if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, fullForeground.Mask, fullForeground.Rect); ok {
if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok {
foregrounds = append(foregrounds, foreground)
}
canvas = plainBackground
}
}
if len(foregrounds) == 0 {
for _, candidate := range elementForegrounds {
foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, candidate)
if !ok {
continue
}
removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect)
foregrounds = append(foregrounds, foreground)
}
}
if len(foregrounds) == 0 && len(extraction.Layers) == 0 {
if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok {
removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect)
foregrounds = append(foregrounds, foreground)
}
}
if len(extraction.Layers) == 0 && len(foregrounds) == 1 {
if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, foregrounds[0].Mask, foregrounds[0].Rect); ok {
canvas = plainBackground
}
}
}
backgroundID := newID()
backgroundURL, backgroundSize, err := s.persistLayerSeparationImage(ctx, projectID, backgroundID, "background", canvas)
if err != nil {
return layerSeparationResult{}, err
}
textRenderData, err := layerSeparationTextRenderData(extraction, canvasWidth, canvasHeight)
if err != nil {
return layerSeparationResult{}, err
}
artifacts := make([]design.GeneratorTaskArtifact, 0, len(foregrounds)+2)
for _, foreground := range foregrounds {
artifacts = append(artifacts, design.GeneratorTaskArtifact{
Type: "image",
Content: foreground.URL,
Metadata: design.GeneratorTaskArtifactMetadata{
ArtifactID: foreground.ID,
BBox: append([]int64(nil), foreground.BBox...),
Format: generatorArtifactFormat("", foreground.URL),
GeneratorName: layerSeparationGeneratorName,
Height: foreground.Height,
Label: layerSeparationForegroundLabel,
NodeID: foreground.ID,
NodeName: "Foreground Layer",
SizeBytes: 0,
Width: foreground.Width,
},
})
}
artifacts = append(artifacts, design.GeneratorTaskArtifact{
Type: "image",
Content: backgroundURL,
Metadata: design.GeneratorTaskArtifactMetadata{
ArtifactID: backgroundID,
Format: generatorArtifactFormat("", backgroundURL),
GeneratorName: layerSeparationGeneratorName,
Height: canvasHeight,
Label: layerSeparationBackgroundLabel,
NodeID: backgroundID,
NodeName: "Image",
SizeBytes: backgroundSize,
Width: canvasWidth,
},
}, design.GeneratorTaskArtifact{
Type: "text",
Content: textRenderData,
Metadata: design.GeneratorTaskArtifactMetadata{
ArtifactID: newID(),
GeneratorName: layerSeparationGeneratorName,
Height: canvasHeight,
Label: layerSeparationTextLabel,
Width: canvasWidth,
},
})
return layerSeparationResult{
CanvasWidth: canvasWidth,
CanvasHeight: canvasHeight,
BackgroundID: backgroundID,
BackgroundURL: backgroundURL,
Foregrounds: foregrounds,
TextExtraction: extraction,
Artifacts: artifacts,
}, nil
}
func (s *DesignService) extractLayerSeparationText(ctx context.Context, target design.Node, req design.NodeActionRequest) (design.TextExtraction, error) {
if extraction, ok, err := textExtractionFromOptions(req.Options); ok || err != nil {
return extraction, err
}
if s.textExtractor == nil {
return design.TextExtraction{}, nil
}
return s.extractTextExtraction(ctx, target, req)
}
func (s *DesignService) createLayerSeparationForegroundCandidates(ctx context.Context, target design.Node, sourceData []byte, sourceContentType string, bounds image.Rectangle, extraction design.TextExtraction) (layerSeparationForegroundCandidate, []layerSeparationForegroundCandidate, bool) {
if s.backgroundRemover == nil {
return layerSeparationForegroundCandidate{}, nil, false
}
removed, err := s.backgroundRemover.RemoveBackground(ctx, design.BackgroundRemovalRequest{
Image: sourceData,
ContentType: sourceContentType,
Width: bounds.Dx(),
Height: bounds.Dy(),
})
if err != nil || len(removed.Image) == 0 {
return layerSeparationForegroundCandidate{}, nil, false
}
foregroundImage, _, err := image.Decode(bytes.NewReader(removed.Image))
if err != nil {
return layerSeparationForegroundCandidate{}, nil, false
}
foreground := image.NewNRGBA(foregroundImage.Bounds())
draw.Draw(foreground, foreground.Bounds(), foregroundImage, foregroundImage.Bounds().Min, draw.Src)
eraseTextLayersFromForeground(foreground, extraction)
foregroundBounds := foreground.Bounds()
rect, ok := alphaContentBounds(foreground, foregroundBounds, 12)
if !ok {
return layerSeparationForegroundCandidate{}, nil, false
}
mask, count := alphaMask(foreground, foregroundBounds, 12)
if count == 0 {
return layerSeparationForegroundCandidate{}, nil, false
}
full := newLayerSeparationForegroundCandidate(foreground, mask, rect, foregroundBounds)
return full, layerSeparationElementCandidates(foreground, foregroundBounds), true
}
func (s *DesignService) persistLayerSeparationForeground(ctx context.Context, projectID string, candidate layerSeparationForegroundCandidate) (layerSeparationForeground, bool) {
if candidate.Image == nil || candidate.Mask == nil || candidate.Rect.Empty() || len(candidate.BBox) < 4 {
return layerSeparationForeground{}, false
}
foregroundID := newID()
foregroundURL, _, err := s.persistLayerSeparationImage(ctx, projectID, foregroundID, "foreground", candidate.Image)
if err != nil {
return layerSeparationForeground{}, false
}
return layerSeparationForeground{
ID: foregroundID,
URL: foregroundURL,
BBox: append([]int64(nil), candidate.BBox...),
Width: candidate.Width,
Height: candidate.Height,
Mask: candidate.Mask,
Rect: candidate.Rect,
}, true
}
func newLayerSeparationForegroundCandidate(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle, bounds image.Rectangle) layerSeparationForegroundCandidate {
return layerSeparationForegroundCandidate{
BBox: []int64{
int64(rect.Min.X - bounds.Min.X),
int64(rect.Min.Y - bounds.Min.Y),
int64(rect.Max.X - bounds.Min.X),
int64(rect.Max.Y - bounds.Min.Y),
},
Width: int64(rect.Dx()),
Height: int64(rect.Dy()),
Mask: mask,
Rect: rect,
Image: foregroundImageFromMask(foreground, mask, rect),
}
}
func eraseTextLayersFromForeground(foreground *image.NRGBA, extraction design.TextExtraction) {
if foreground == nil || len(extraction.Layers) == 0 {
return
}
bounds := foreground.Bounds()
if bounds.Dx()*bounds.Dy() < 4096 {
return
}
for _, layer := range extraction.Layers {
rect := normalizedTextMaskRect(bounds, layer)
if rect.Empty() {
continue
}
padding := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.14)), 2, 18)
rect = image.Rect(rect.Min.X-padding, rect.Min.Y-padding, rect.Max.X+padding, rect.Max.Y+padding).Intersect(bounds)
for y := rect.Min.Y; y < rect.Max.Y; y++ {
for x := rect.Min.X; x < rect.Max.X; x++ {
offset := foreground.PixOffset(x, y)
foreground.Pix[offset+3] = 0
}
}
}
}
func layerSeparationElementCandidates(foreground *image.NRGBA, bounds image.Rectangle) []layerSeparationForegroundCandidate {
if foreground == nil || bounds.Empty() {
return nil
}
width := bounds.Dx()
height := bounds.Dy()
visited := make([]bool, width*height)
components := make([]layerSeparationForegroundCandidate, 0, 4)
indexOf := func(x int, y int) int {
return (y-bounds.Min.Y)*width + (x - bounds.Min.X)
}
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
index := indexOf(x, y)
if visited[index] || foreground.NRGBAAt(x, y).A <= 12 {
visited[index] = true
continue
}
mask, rect, count := layerSeparationComponentMask(foreground, bounds, visited, x, y, indexOf)
if !keepLayerSeparationElement(rect, count, bounds) {
continue
}
components = append(components, newLayerSeparationForegroundCandidate(foreground, mask, rect, bounds))
}
}
sort.Slice(components, func(i int, j int) bool {
leftArea := components[i].Rect.Dx() * components[i].Rect.Dy()
rightArea := components[j].Rect.Dx() * components[j].Rect.Dy()
return leftArea > rightArea
})
if len(components) > 12 {
components = components[:12]
}
return components
}
func layerSeparationComponentMask(foreground *image.NRGBA, bounds image.Rectangle, visited []bool, startX int, startY int, indexOf func(int, int) int) (*image.Alpha, image.Rectangle, int) {
mask := image.NewAlpha(bounds)
stack := []image.Point{{X: startX, Y: startY}}
minX, minY := startX, startY
maxX, maxY := startX+1, startY+1
count := 0
for len(stack) > 0 {
point := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if point.X < bounds.Min.X || point.X >= bounds.Max.X || point.Y < bounds.Min.Y || point.Y >= bounds.Max.Y {
continue
}
index := indexOf(point.X, point.Y)
if visited[index] {
continue
}
visited[index] = true
if foreground.NRGBAAt(point.X, point.Y).A <= 12 {
continue
}
mask.SetAlpha(point.X, point.Y, color.Alpha{A: 255})
count++
if point.X < minX {
minX = point.X
}
if point.Y < minY {
minY = point.Y
}
if point.X+1 > maxX {
maxX = point.X + 1
}
if point.Y+1 > maxY {
maxY = point.Y + 1
}
stack = append(stack,
image.Point{X: point.X + 1, Y: point.Y},
image.Point{X: point.X - 1, Y: point.Y},
image.Point{X: point.X, Y: point.Y + 1},
image.Point{X: point.X, Y: point.Y - 1},
)
}
return mask, image.Rect(minX, minY, maxX, maxY), count
}
func keepLayerSeparationElement(rect image.Rectangle, count int, bounds image.Rectangle) bool {
if rect.Empty() || count <= 0 {
return false
}
totalArea := bounds.Dx() * bounds.Dy()
if totalArea <= 0 {
return false
}
if totalArea < 4096 {
return true
}
bboxArea := rect.Dx() * rect.Dy()
minArea := maxInt(96, totalArea/6000)
if count < minArea || bboxArea < minArea {
return false
}
if float64(bboxArea)/float64(totalArea) > 0.18 {
return false
}
if float64(rect.Dx())/float64(bounds.Dx()) > 0.62 || float64(rect.Dy())/float64(bounds.Dy()) > 0.62 {
return false
}
return true
}
func foregroundImageFromMask(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle) image.Image {
cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy()))
for y := rect.Min.Y; y < rect.Max.Y; y++ {
for x := rect.Min.X; x < rect.Max.X; x++ {
if mask.AlphaAt(x, y).A == 0 {
continue
}
cropped.SetNRGBA(x-rect.Min.X, y-rect.Min.Y, foreground.NRGBAAt(x, y))
}
}
return cropped
}
func (s *DesignService) persistLayerSeparationImage(ctx context.Context, projectID string, nodeID string, suffix string, img image.Image) (string, int64, error) {
var output bytes.Buffer
if err := png.Encode(&output, img); err != nil {
return "", 0, err
}
if s.assets == nil {
data := output.Bytes()
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(data), int64(len(data)), nil
}
data, contentType, err := encodeRasterAssetAsWebP(output.Bytes(), "image/png")
if err != nil {
return "", 0, err
}
object, err := s.assets.PutObject(ctx, design.AssetObjectRequest{
FileName: fmt.Sprintf("%s-%s-%s.png", projectID, nodeID, suffix),
ContentType: contentType,
Data: data,
})
if err != nil {
return "", 0, err
}
if strings.TrimSpace(object.PublicURL) == "" {
return "", 0, fmt.Errorf("layer separation upload returned empty public URL")
}
return object.PublicURL, int64(len(data)), nil
}
func newLayerSeparationBackgroundNode(frame design.Node, result layerSeparationResult) design.Node {
return design.Node{
ID: result.BackgroundID,
Type: design.NodeTypeImage,
Title: "Image",
X: frame.X,
Y: frame.Y,
Width: frame.Width,
Height: frame.Height,
Content: result.BackgroundURL,
Tone: "visual",
Status: "success",
LayerRole: "clean-image",
}
}
func newForegroundImageLayers(target design.Node, foregrounds []layerSeparationForeground, canvasWidth int64, canvasHeight int64) []design.Node {
if len(foregrounds) == 0 || canvasWidth <= 0 || canvasHeight <= 0 {
return nil
}
nodes := make([]design.Node, 0, len(foregrounds))
for index, foreground := range foregrounds {
if len(foreground.BBox) < 4 || strings.TrimSpace(foreground.URL) == "" {
continue
}
x1 := clampFloat(float64(foreground.BBox[0])/float64(canvasWidth), 0, 1)
y1 := clampFloat(float64(foreground.BBox[1])/float64(canvasHeight), 0, 1)
x2 := clampFloat(float64(foreground.BBox[2])/float64(canvasWidth), x1, 1)
y2 := clampFloat(float64(foreground.BBox[3])/float64(canvasHeight), y1, 1)
width := target.Width * (x2 - x1)
height := target.Height * (y2 - y1)
if width <= 1 || height <= 1 {
continue
}
nodes = append(nodes, design.Node{
ID: foreground.ID,
Type: design.NodeTypeImage,
Title: fmt.Sprintf("Foreground Layer %d", index+1),
X: target.X + target.Width*x1,
Y: target.Y + target.Height*y1,
Width: width,
Height: height,
Content: foreground.URL,
Tone: "separated-element",
Status: "success",
ParentID: target.ID,
LayerRole: "foreground-image",
})
}
return nodes
}
func layerSeparationArtifactsMessage(threadID string, artifacts []design.GeneratorTaskArtifact, createdAt time.Time) design.Message {
content, _ := json.Marshal(toLayerSeparationArtifactWire(artifacts))
return design.Message{
ID: newID(),
Role: "tool",
Type: "tool_use",
Title: layerSeparationMessageName,
Content: string(content),
ThreadID: threadID,
ActionID: threadID,
StepID: newID(),
ToolCallID: "call_" + strings.ReplaceAll(newID(), "-", ""),
Name: layerSeparationMessageName,
ToolHint: layerSeparationMessageName,
Status: "success",
CreatedAt: createdAt,
}
}
func toLayerSeparationArtifactWire(artifacts []design.GeneratorTaskArtifact) []map[string]any {
items := make([]map[string]any, 0, len(artifacts))
for _, artifact := range artifacts {
items = append(items, map[string]any{
"type": artifact.Type,
"content": artifact.Content,
"metadata": map[string]any{
"artifact_id": artifact.Metadata.ArtifactID,
"bbox": artifact.Metadata.BBox,
"format": artifact.Metadata.Format,
"generator_name": artifact.Metadata.GeneratorName,
"height": artifact.Metadata.Height,
"label": artifact.Metadata.Label,
"node_id": artifact.Metadata.NodeID,
"node_name": artifact.Metadata.NodeName,
"size_bytes": artifact.Metadata.SizeBytes,
"width": artifact.Metadata.Width,
},
})
}
return items
}
func removeForegroundMaskFromImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) {
if mask == nil || rect.Empty() {
return
}
bounds := img.Bounds()
rect = rect.Intersect(bounds)
if rect.Empty() {
return
}
radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.025)), 2, 10)
mask, count := dilateTextMask(mask, rect, radius)
if count == 0 {
return
}
source := image.NewRGBA(bounds)
draw.Draw(source, bounds, img, bounds.Min, draw.Src)
for y := rect.Min.Y; y < rect.Max.Y; y++ {
for x := rect.Min.X; x < rect.Max.X; x++ {
if mask.AlphaAt(x, y).A == 0 {
continue
}
img.SetRGBA(x, y, estimatedRectBackgroundColor(source, rect, x, y))
}
}
softenMaskedPixels(img, mask, rect)
}
func plainLayerSeparationBackgroundImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) (*image.RGBA, bool) {
if img == nil || mask == nil || rect.Empty() {
return nil, false
}
bounds := img.Bounds()
if bounds.Empty() {
return nil, false
}
radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.02)), 3, 18)
expandedMask, _ := dilateTextMask(mask, bounds, radius)
stats, ok := unmaskedColorStats(img, expandedMask, bounds)
if !ok || stats.StdDev > 24 {
return nil, false
}
bg := image.NewRGBA(bounds)
fill := colorRGBA(int(math.Round(stats.R)), int(math.Round(stats.G)), int(math.Round(stats.B)), int(math.Round(stats.A)))
draw.Draw(bg, bounds, &image.Uniform{C: fill}, image.Point{}, draw.Src)
return bg, true
}
type layerSeparationColorStats struct {
Count int64
R float64
G float64
B float64
A float64
StdDev float64
}
func unmaskedColorStats(img *image.RGBA, mask *image.Alpha, bounds image.Rectangle) (layerSeparationColorStats, bool) {
stride := clampInt(int(math.Ceil(float64(maxInt(bounds.Dx(), bounds.Dy()))/640)), 1, 12)
var count int64
var r, g, b, a float64
var rr, gg, bb float64
for y := bounds.Min.Y; y < bounds.Max.Y; y += stride {
for x := bounds.Min.X; x < bounds.Max.X; x += stride {
if mask.AlphaAt(x, y).A != 0 {
continue
}
c := img.RGBAAt(x, y)
cfR := float64(c.R)
cfG := float64(c.G)
cfB := float64(c.B)
cfA := float64(c.A)
r += cfR
g += cfG
b += cfB
a += cfA
rr += cfR * cfR
gg += cfG * cfG
bb += cfB * cfB
count++
}
}
if count < 32 {
return layerSeparationColorStats{}, false
}
divisor := float64(count)
meanR := r / divisor
meanG := g / divisor
meanB := b / divisor
varR := math.Max(0, rr/divisor-meanR*meanR)
varG := math.Max(0, gg/divisor-meanG*meanG)
varB := math.Max(0, bb/divisor-meanB*meanB)
return layerSeparationColorStats{
Count: count,
R: meanR,
G: meanG,
B: meanB,
A: a / divisor,
StdDev: math.Sqrt((varR + varG + varB) / 3),
}, true
}
func alphaContentBounds(img image.Image, bounds image.Rectangle, threshold uint8) (image.Rectangle, bool) {
minX, minY := bounds.Max.X, bounds.Max.Y
maxX, maxY := bounds.Min.X, bounds.Min.Y
found := false
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
if alphaAt(img, x, y) <= threshold {
continue
}
if x < minX {
minX = x
}
if y < minY {
minY = y
}
if x+1 > maxX {
maxX = x + 1
}
if y+1 > maxY {
maxY = y + 1
}
found = true
}
}
if !found {
return image.Rectangle{}, false
}
return image.Rect(minX, minY, maxX, maxY), true
}
func alphaMask(img image.Image, bounds image.Rectangle, threshold uint8) (*image.Alpha, int) {
mask := image.NewAlpha(bounds)
count := 0
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
if alphaAt(img, x, y) <= threshold {
continue
}
mask.SetAlpha(x, y, color.Alpha{A: 255})
count++
}
}
return mask, count
}
func alphaAt(img image.Image, x int, y int) uint8 {
_, _, _, a := img.At(x, y).RGBA()
return uint8(a >> 8)
}
func cropImage(img image.Image, rect image.Rectangle) image.Image {
cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy()))
draw.Draw(cropped, cropped.Bounds(), img, rect.Min, draw.Src)
return cropped
}
type layerSeparationTextDocument struct {
Layers []layerSeparationTextLayer `json:"layers"`
CanvasSize layerSeparationCanvasSize `json:"canvas_size"`
}
type layerSeparationCanvasSize struct {
Width int64 `json:"width"`
Height int64 `json:"height"`
}
type layerSeparationTextLayer struct {
TextInfo layerSeparationTextInfo `json:"text_info"`
X float64 `json:"x"`
Y float64 `json:"y"`
W float64 `json:"w"`
H float64 `json:"h"`
}
type layerSeparationTextInfo struct {
Text string `json:"text"`
FontSizePx int64 `json:"font_size_px"`
FontFamily string `json:"font_family"`
FontWeight any `json:"font_weight"`
FontStyle string `json:"font_style"`
ColorCSS string `json:"color_css"`
TextAlign string `json:"text_align"`
Leading float64 `json:"leading"`
Tracking float64 `json:"tracking"`
Rotation float64 `json:"rotation"`
LeadingTrim string `json:"leading_trim"`
}
func layerSeparationTextRenderData(extraction design.TextExtraction, canvasWidth int64, canvasHeight int64) (string, error) {
doc := layerSeparationTextDocument{
Layers: make([]layerSeparationTextLayer, 0, len(extraction.Layers)),
CanvasSize: layerSeparationCanvasSize{
Width: canvasWidth,
Height: canvasHeight,
},
}
for _, layer := range extraction.Layers {
text := strings.TrimSpace(layer.Text)
if text == "" {
continue
}
x := clamp01(layer.X)
y := clamp01(layer.Y)
width := clamp01(layer.Width)
height := clamp01(layer.Height)
if x+width > 1 {
width = 1 - x
}
if y+height > 1 {
height = 1 - y
}
if width <= 0 || height <= 0 {
continue
}
fontSize := layer.FontSize
if fontSize <= 0 {
fontSize = math.Max(10, float64(canvasHeight)*height*0.72)
}
lineHeight := layer.LineHeight
if lineHeight <= 0 {
lineHeight = 1.2
}
fontFamily := strings.TrimSpace(layer.FontFamily)
if fontFamily == "" {
fontFamily = "Inter"
}
doc.Layers = append(doc.Layers, layerSeparationTextLayer{
TextInfo: layerSeparationTextInfo{
Text: text,
FontSizePx: int64(math.Round(fontSize)),
FontFamily: fontFamily,
FontWeight: layerSeparationFontWeight(layer.FontWeight),
FontStyle: layerSeparationFontStyle(layer.FontWeight),
ColorCSS: layerSeparationColorCSS(layer.Color),
TextAlign: layerSeparationTextAlign(layer.TextAlign),
Leading: lineHeight,
Tracking: layer.LetterSpacing,
Rotation: layer.Rotation,
LeadingTrim: "CAP_HEIGHT",
},
X: x * float64(canvasWidth),
Y: y * float64(canvasHeight),
W: width * float64(canvasWidth),
H: height * float64(canvasHeight),
})
}
data, err := json.Marshal(doc)
if err != nil {
return "", err
}
return string(data), nil
}
func layerSeparationFontWeight(value string) any {
value = strings.TrimSpace(value)
if value == "" {
return 400
}
if weight, err := strconv.Atoi(value); err == nil {
return weight
}
return value
}
func layerSeparationFontStyle(value string) string {
if strings.Contains(strings.ToLower(strings.TrimSpace(value)), "italic") {
return "italic"
}
return "normal"
}
func layerSeparationTextAlign(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "left":
return "start"
case "right":
return "end"
case "center":
return "center"
default:
return "start"
}
}
func layerSeparationColorCSS(value string) string {
parsed, ok := parseCanvasColor(value)
if !ok {
return "rgba(25, 25, 25, 1.0)"
}
alpha := float64(parsed.A) / 255
return fmt.Sprintf("rgba(%d, %d, %d, %.3g)", parsed.R, parsed.G, parsed.B, alpha)
}
@@ -0,0 +1,410 @@
package application
import (
"bytes"
"context"
"image"
"image/color"
"image/png"
"testing"
"time"
"img_infinite_canvas/internal/domain/design"
"img_infinite_canvas/internal/infrastructure/memory"
)
func TestRunEditElementsSeparatesBackgroundForegroundAndTextLayers(t *testing.T) {
ctx := context.Background()
repo := memory.NewProjectRepository()
assets := newFakeAssetStorage()
queue := &fakeJobQueue{}
extractor := &countingTextExtractor{
result: design.TextExtraction{
Layers: []design.ExtractedTextLayer{
{Text: "Hi", X: 0.5, Y: 0.05, Width: 0.2, Height: 0.1, FontSize: 18, FontFamily: "Inter", FontWeight: "700", Color: "#111111"},
},
},
}
remover := &fakeBackgroundRemover{result: design.BackgroundRemoval{
Image: testLayerSeparationForegroundPNG(t),
ContentType: "image/png",
Width: 4,
Height: 4,
}}
service := NewDesignService(repo, nil, assets, nil, extractor)
service.SetJobQueue(queue)
service.SetBackgroundRemover(remover)
source := testLayerSeparationSourceDataURL()
project := design.Project{
ID: "project-layer-separation",
Title: "Layer separation",
Status: design.StatusReady,
UpdatedAt: time.Now(),
Nodes: []design.Node{{
ID: "image-1",
Type: design.NodeTypeImage,
Title: "Source",
X: 100,
Y: 120,
Width: 400,
Height: 400,
Content: source,
Status: "success",
}},
}
if err := repo.Save(ctx, project); err != nil {
t.Fatal(err)
}
started, err := service.RunNodeActionAsync(ctx, project.ID, "image-1", design.NodeActionRequest{Action: "edit-elements"})
if err != nil {
t.Fatal(err)
}
if len(queue.jobs) != 1 {
t.Fatalf("expected one queued job, got %#v", queue.jobs)
}
if len(started.Nodes) != 2 {
t.Fatalf("expected source plus separated placeholder, got %#v", started.Nodes)
}
if started.Nodes[0].Status != "success" || started.Nodes[0].Content != source {
t.Fatalf("expected source node to remain unchanged, got %#v", started.Nodes[0])
}
if started.Nodes[1].Type != design.NodeTypeFrame || started.Nodes[1].Status != "generating" || started.Nodes[1].LayerRole != "manual-frame" || started.Nodes[1].ParentID != "" || started.Nodes[1].FillColor != "transparent" {
t.Fatalf("expected separated placeholder beside source, got %#v", started.Nodes[1])
}
if started.Nodes[1].Content != source {
t.Fatalf("expected separated placeholder to preview source image, got %#v", started.Nodes[1])
}
task, err := service.GeneratorTask(ctx, project.ID, started.LastThreadID)
if err != nil {
t.Fatal(err)
}
if task.GeneratorName != layerSeparationGeneratorName || task.Status != "submitted" {
t.Fatalf("expected submitted layer separation task, got %#v", task)
}
if err := service.ProcessJob(ctx, queue.jobs[0]); err != nil {
t.Fatal(err)
}
saved, err := repo.Get(ctx, project.ID)
if err != nil {
t.Fatal(err)
}
if saved.Status != design.StatusReady {
t.Fatalf("expected ready project, got %s", saved.Status)
}
if len(saved.Nodes) != 5 {
t.Fatalf("expected original, background, frame, foreground, and text nodes, got %#v", saved.Nodes)
}
if saved.Nodes[0].Status != "success" || saved.Nodes[0].Content != source {
t.Fatalf("expected original source to remain unchanged, got %#v", saved.Nodes[0])
}
background := saved.Nodes[1]
if background.Type != design.NodeTypeImage || background.LayerRole != "clean-image" || background.Status != "success" {
t.Fatalf("expected clean background node, got %#v", background)
}
if background.X != 532 || background.Y != 120 {
t.Fatalf("expected clean background at separated result origin, got %#v", background)
}
frame := saved.Nodes[2]
if frame.Type != design.NodeTypeFrame || frame.LayerRole != "manual-frame" || frame.Status != "success" {
t.Fatalf("expected separated frame beside source, got %#v", frame)
}
if frame.X != 532 || frame.Y != 120 {
t.Fatalf("expected frame beside source, got %#v", frame)
}
if frame.ParentID != "" || frame.FillColor != "transparent" {
t.Fatalf("expected top-level transparent separated frame, got %#v", frame)
}
if background.ParentID != "" {
t.Fatalf("expected clean background to be a bottom image layer outside frame, got %#v", background)
}
foreground := saved.Nodes[3]
if foreground.LayerRole != "foreground-image" || foreground.ParentID != frame.ID {
t.Fatalf("expected separated foreground node, got %#v", foreground)
}
if foreground.X != 632 || foreground.Y != 220 || foreground.Width != 200 || foreground.Height != 200 {
t.Fatalf("unexpected foreground geometry: %#v", foreground)
}
text := saved.Nodes[4]
if text.LayerRole != "editable-text" || text.ParentID != frame.ID || text.Content != "Hi" {
t.Fatalf("expected editable text node, got %#v", text)
}
task, err = service.GeneratorTask(ctx, project.ID, started.LastThreadID)
if err != nil {
t.Fatal(err)
}
if task.Status != "completed" {
t.Fatalf("expected completed task, got %s", task.Status)
}
if len(task.Artifacts) != 3 {
t.Fatalf("expected foreground, background, and text artifacts, got %#v", task.Artifacts)
}
if task.Artifacts[0].Metadata.Label != layerSeparationForegroundLabel || task.Artifacts[1].Metadata.Label != layerSeparationBackgroundLabel || task.Artifacts[2].Metadata.Label != layerSeparationTextLabel {
t.Fatalf("unexpected artifact labels: %#v", task.Artifacts)
}
if got := task.Artifacts[0].Metadata.BBox; len(got) != 4 || got[0] != 1 || got[1] != 1 || got[2] != 3 || got[3] != 3 {
t.Fatalf("unexpected foreground bbox: %#v", got)
}
if task.Artifacts[2].Type != "text" || !bytes.Contains([]byte(task.Artifacts[2].Content), []byte(`"text":"Hi"`)) {
t.Fatalf("expected text render data artifact, got %#v", task.Artifacts[2])
}
}
func TestSubmitLayerSeparationTaskCompletesStandaloneArtifacts(t *testing.T) {
ctx := context.Background()
assets := newFakeAssetStorage()
extractor := &countingTextExtractor{
result: design.TextExtraction{
Layers: []design.ExtractedTextLayer{{Text: "Title", X: 0.1, Y: 0.1, Width: 0.3, Height: 0.1}},
},
}
remover := &fakeBackgroundRemover{result: design.BackgroundRemoval{
Image: testLayerSeparationForegroundPNG(t),
ContentType: "image/png",
Width: 4,
Height: 4,
}}
service := NewDesignService(nil, nil, assets, nil, extractor)
service.SetBackgroundRemover(remover)
task, err := service.SubmitGeneratorTask(ctx, design.GeneratorTaskSubmitRequest{
ProjectID: "project-standalone-layer-separation",
GeneratorName: layerSeparationGeneratorName,
InputArgs: design.GeneratorTaskInputArgs{
ImageURL: testLayerSeparationSourceDataURL(),
},
})
if err != nil {
t.Fatal(err)
}
completed := waitForStandaloneGeneratorTask(t, service, task.GeneratorTaskID)
if completed.Status != "completed" {
t.Fatalf("expected completed standalone task, got %#v", completed)
}
if completed.GeneratorName != layerSeparationGeneratorName || len(completed.Artifacts) != 3 {
t.Fatalf("unexpected standalone task: %#v", completed)
}
if completed.Artifacts[0].Metadata.Label != layerSeparationForegroundLabel || completed.Artifacts[2].Metadata.Label != layerSeparationTextLabel {
t.Fatalf("unexpected standalone artifacts: %#v", completed.Artifacts)
}
}
func TestPlainLayerSeparationBackgroundUsesSolidColorWhenBackdropIsPlain(t *testing.T) {
img := image.NewRGBA(image.Rect(0, 0, 24, 24))
for y := 0; y < 24; y++ {
for x := 0; x < 24; x++ {
img.SetRGBA(x, y, color.RGBA{R: 188, G: 185, B: 178, A: 255})
}
}
mask := image.NewAlpha(img.Bounds())
rect := image.Rect(8, 4, 16, 22)
for y := rect.Min.Y; y < rect.Max.Y; y++ {
for x := rect.Min.X; x < rect.Max.X; x++ {
img.SetRGBA(x, y, color.RGBA{R: 32, G: 26, B: 22, A: 255})
mask.SetAlpha(x, y, color.Alpha{A: 255})
}
}
bg, ok := plainLayerSeparationBackgroundImage(img, mask, rect)
if !ok {
t.Fatal("expected plain background branch")
}
center := bg.RGBAAt(12, 12)
if center.R != 188 || center.G != 185 || center.B != 178 || center.A != 255 {
t.Fatalf("expected solid background color, got %#v", center)
}
}
func TestPlainLayerSeparationBackgroundSkipsComplexBackdrop(t *testing.T) {
img := image.NewRGBA(image.Rect(0, 0, 24, 24))
for y := 0; y < 24; y++ {
for x := 0; x < 24; x++ {
img.SetRGBA(x, y, color.RGBA{R: uint8(20 + x*8), G: uint8(40 + y*7), B: uint8(80 + (x+y)*3), A: 255})
}
}
mask := image.NewAlpha(img.Bounds())
rect := image.Rect(9, 6, 15, 20)
for y := rect.Min.Y; y < rect.Max.Y; y++ {
for x := rect.Min.X; x < rect.Max.X; x++ {
mask.SetAlpha(x, y, color.Alpha{A: 255})
}
}
if _, ok := plainLayerSeparationBackgroundImage(img, mask, rect); ok {
t.Fatal("expected complex background to use repair branch")
}
}
func TestLayerSeparationPosterKeepsComplexBackgroundAndExtractsSmallElements(t *testing.T) {
ctx := context.Background()
extractor := &countingTextExtractor{
result: design.TextExtraction{
Layers: []design.ExtractedTextLayer{{
Text: "SALE",
X: 0.1,
Y: 0.1,
Width: 0.35,
Height: 0.125,
FontSize: 12,
FontFamily: "Inter",
FontWeight: "700",
Color: "#111111",
}},
},
}
remover := &fakeBackgroundRemover{result: design.BackgroundRemoval{
Image: testPosterForegroundPNG(t),
ContentType: "image/png",
Width: 80,
Height: 80,
}}
service := NewDesignService(nil, nil, nil, nil, extractor)
service.SetBackgroundRemover(remover)
result, err := service.createLayerSeparationResult(ctx, "project-poster-layer-separation", design.Node{
ID: "poster",
Type: design.NodeTypeImage,
Title: "Poster",
Content: testPosterSourceDataURL(),
Status: "success",
}, design.NodeActionRequest{Action: "edit-elements"})
if err != nil {
t.Fatal(err)
}
if len(result.TextExtraction.Layers) != 1 {
t.Fatalf("expected editable text extraction to remain, got %#v", result.TextExtraction.Layers)
}
if len(result.Foregrounds) != 1 {
t.Fatalf("expected only the small decorative foreground to be extracted, got %#v", result.Foregrounds)
}
if got := result.Foregrounds[0].BBox; len(got) != 4 || got[0] != 54 || got[1] != 44 || got[2] != 66 || got[3] != 56 {
t.Fatalf("unexpected decorative foreground bbox: %#v", got)
}
backgroundData, _, err := loadImageContent(ctx, result.BackgroundURL)
if err != nil {
t.Fatal(err)
}
background, _, err := image.Decode(bytes.NewReader(backgroundData))
if err != nil {
t.Fatal(err)
}
topLeft := color.NRGBAModel.Convert(background.At(2, 2)).(color.NRGBA)
bottomRight := color.NRGBAModel.Convert(background.At(76, 76)).(color.NRGBA)
if topLeft == bottomRight {
t.Fatalf("expected complex poster background to be preserved, got solid-looking pixels %#v", topLeft)
}
textPixel := color.NRGBAModel.Convert(background.At(14, 12)).(color.NRGBA)
if textPixel.R < 32 && textPixel.G < 32 && textPixel.B < 32 {
t.Fatalf("expected text pixels to be removed from background, got %#v", textPixel)
}
ornamentPixel := color.NRGBAModel.Convert(background.At(59, 49)).(color.NRGBA)
if ornamentPixel.R > 200 && ornamentPixel.G < 80 && ornamentPixel.B < 80 {
t.Fatalf("expected extracted ornament to be removed from background, got %#v", ornamentPixel)
}
scenePixel := color.NRGBAModel.Convert(background.At(20, 48)).(color.NRGBA)
if scenePixel.B < 170 {
t.Fatalf("expected large poster scenery to stay on the background, got %#v", scenePixel)
}
}
func waitForStandaloneGeneratorTask(t *testing.T, service *DesignService, taskID string) design.GeneratorTask {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
task, err := service.GeneratorTask(context.Background(), "", taskID)
if err != nil {
t.Fatal(err)
}
if task.Status == "completed" || task.Status == "failed" {
return task
}
time.Sleep(10 * time.Millisecond)
}
task, _ := service.GeneratorTask(context.Background(), "", taskID)
t.Fatalf("timed out waiting for standalone task: %#v", task)
return design.GeneratorTask{}
}
func testLayerSeparationSourceDataURL() string {
img := image.NewNRGBA(image.Rect(0, 0, 4, 4))
for y := 0; y < 4; y++ {
for x := 0; x < 4; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 240, G: 240, B: 240, A: 255})
}
}
for y := 1; y < 3; y++ {
for x := 1; x < 3; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 200, G: 20, B: 30, A: 255})
}
}
return testPNGDataURLFromImage(img)
}
func testLayerSeparationForegroundPNG(t *testing.T) []byte {
t.Helper()
img := image.NewNRGBA(image.Rect(0, 0, 4, 4))
for y := 1; y < 3; y++ {
for x := 1; x < 3; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 200, G: 20, B: 30, A: 255})
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
func testPosterSourceDataURL() string {
img := image.NewNRGBA(image.Rect(0, 0, 80, 80))
for y := 0; y < 80; y++ {
for x := 0; x < 80; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: uint8(190 + x/3), G: uint8(186 + y/4), B: uint8(174 + (x+y)/8), A: 255})
}
}
for y := 38; y < 70; y++ {
for x := 8; x < 50; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 64, G: 106, B: 204, A: 255})
}
}
for y := 8; y < 18; y++ {
for x := 8; x < 36; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 17, G: 17, B: 17, A: 255})
}
}
for y := 44; y < 56; y++ {
for x := 54; x < 66; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 225, G: 32, B: 44, A: 255})
}
}
return testPNGDataURLFromImage(img)
}
func testPosterForegroundPNG(t *testing.T) []byte {
t.Helper()
img := image.NewNRGBA(image.Rect(0, 0, 80, 80))
for y := 8; y < 18; y++ {
for x := 8; x < 36; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 17, G: 17, B: 17, A: 255})
}
}
for y := 38; y < 70; y++ {
for x := 8; x < 50; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 64, G: 106, B: 204, A: 255})
}
}
for y := 44; y < 56; y++ {
for x := 54; x < 66; x++ {
img.SetNRGBA(x, y, color.NRGBA{R: 225, G: 32, B: 44, A: 255})
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
+10 -8
View File
@@ -107,14 +107,16 @@ type GeneratorTaskQueueInfo struct {
}
type GeneratorTaskArtifactMetadata struct {
ArtifactID string
Format string
Height int64
Label string
NodeID string
NodeName string
SizeBytes int64
Width int64
ArtifactID string
BBox []int64
Format string
GeneratorName string
Height int64
Label string
NodeID string
NodeName string
SizeBytes int64
Width int64
}
type GeneratorTaskArtifact struct {
+10 -8
View File
@@ -133,14 +133,16 @@ func toAPIGeneratorTaskArtifacts(items []design.GeneratorTaskArtifact) []types.G
Type: item.Type,
Content: item.Content,
Metadata: types.GeneratorTaskArtifactMetadata{
ArtifactId: item.Metadata.ArtifactID,
Format: item.Metadata.Format,
Height: item.Metadata.Height,
Label: item.Metadata.Label,
NodeId: item.Metadata.NodeID,
NodeName: item.Metadata.NodeName,
SizeBytes: item.Metadata.SizeBytes,
Width: item.Metadata.Width,
ArtifactId: item.Metadata.ArtifactID,
BBox: append([]int64(nil), item.Metadata.BBox...),
Format: item.Metadata.Format,
GeneratorName: item.Metadata.GeneratorName,
Height: item.Metadata.Height,
Label: item.Metadata.Label,
NodeId: item.Metadata.NodeID,
NodeName: item.Metadata.NodeName,
SizeBytes: item.Metadata.SizeBytes,
Width: item.Metadata.Width,
},
})
}
+10 -8
View File
@@ -417,14 +417,16 @@ type GeneratorTaskArtifact struct {
}
type GeneratorTaskArtifactMetadata struct {
ArtifactId string `json:"artifact_id,optional"`
Format string `json:"format,optional"`
Height int64 `json:"height,optional"`
Label string `json:"label,optional"`
NodeId string `json:"node_id,optional"`
NodeName string `json:"node_name,optional"`
SizeBytes int64 `json:"size_bytes,optional"`
Width int64 `json:"width,optional"`
ArtifactId string `json:"artifact_id,optional"`
BBox []int64 `json:"bbox,optional"`
Format string `json:"format,optional"`
GeneratorName string `json:"generator_name,optional"`
Height int64 `json:"height,optional"`
Label string `json:"label,optional"`
NodeId string `json:"node_id,optional"`
NodeName string `json:"node_name,optional"`
SizeBytes int64 `json:"size_bytes,optional"`
Width int64 `json:"width,optional"`
}
type GeneratorTaskData struct {