feat(server): add layer-separation generator task
Introduce the `layer-separation` generator (edit-elements node action) that splits a source image into a clean background layer, foreground image artifacts with bounding boxes, and editable text render data. - New layer_separation.go builds the separated result and artifacts - SubmitGeneratorTask routes layer-separation and move-object via a switch; standalone task completion + polling support - Extend artifact metadata with BBox and GeneratorName across the domain, API types, mapper, .api schema, and API.md - edit-elements gets a transparent manual-frame placeholder holding the separated foreground images and text layers beside the original Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,892 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/png"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"img_infinite_canvas/internal/domain/design"
|
||||
)
|
||||
|
||||
const (
|
||||
layerSeparationForegroundLabel = "fg_image"
|
||||
layerSeparationBackgroundLabel = "bg_image"
|
||||
layerSeparationTextLabel = "text_render_data"
|
||||
)
|
||||
|
||||
type layerSeparationResult struct {
|
||||
CanvasWidth int64
|
||||
CanvasHeight int64
|
||||
BackgroundID string
|
||||
BackgroundURL string
|
||||
Foregrounds []layerSeparationForeground
|
||||
TextExtraction design.TextExtraction
|
||||
Artifacts []design.GeneratorTaskArtifact
|
||||
}
|
||||
|
||||
type layerSeparationForeground struct {
|
||||
ID string
|
||||
URL string
|
||||
BBox []int64
|
||||
Width int64
|
||||
Height int64
|
||||
Mask *image.Alpha
|
||||
Rect image.Rectangle
|
||||
}
|
||||
|
||||
type layerSeparationForegroundCandidate struct {
|
||||
BBox []int64
|
||||
Width int64
|
||||
Height int64
|
||||
Mask *image.Alpha
|
||||
Rect image.Rectangle
|
||||
Image image.Image
|
||||
}
|
||||
|
||||
func (s *DesignService) completeLayerSeparationGeneration(ctx context.Context, projectID string, threadID string, target design.Node, req design.NodeActionRequest) error {
|
||||
resultNodeID := nodeActionResultNodeID(target, req)
|
||||
if err := s.addGenerationStep(ctx, projectID, threadID, "tool", "图层分离", "正在拆分底图、前景元素和可编辑文字层。"); err != nil {
|
||||
return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err)
|
||||
}
|
||||
|
||||
resultTarget := target
|
||||
resultTarget.ID = resultNodeID
|
||||
result, err := s.createLayerSeparationResult(ctx, projectID, resultTarget, req)
|
||||
if err != nil {
|
||||
return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err)
|
||||
}
|
||||
|
||||
project, err := s.repo.Get(ctx, projectID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nodeIndex := findNodeIndex(project.Nodes, resultNodeID)
|
||||
if nodeIndex < 0 {
|
||||
project.Nodes = append(project.Nodes, newLayerSeparationPlaceholder(target))
|
||||
nodeIndex = len(project.Nodes) - 1
|
||||
project.Nodes[nodeIndex].ID = resultNodeID
|
||||
}
|
||||
|
||||
frameNode := project.Nodes[nodeIndex]
|
||||
if frameNode.Width <= 0 {
|
||||
frameNode.Width = float64(result.CanvasWidth)
|
||||
}
|
||||
if frameNode.Height <= 0 {
|
||||
frameNode.Height = float64(result.CanvasHeight)
|
||||
}
|
||||
frameNode.Type = design.NodeTypeFrame
|
||||
frameNode.Title = "Frame"
|
||||
frameNode.Content = ""
|
||||
frameNode.Tone = "frame"
|
||||
frameNode.Status = "success"
|
||||
frameNode.ParentID = ""
|
||||
frameNode.LayerRole = "manual-frame"
|
||||
frameNode.FillColor = "transparent"
|
||||
frameNode.StrokeColor = "#C7C7C7"
|
||||
frameNode.StrokeWidth = 1
|
||||
frameNode.StrokeStyle = "solid"
|
||||
frameNode.MockupSurface = design.MockupSurface{}
|
||||
frameNode.MockupModel = design.MockupModel{}
|
||||
frameNode.MockupSourceContent = ""
|
||||
|
||||
now := s.now()
|
||||
backgroundNode := newLayerSeparationBackgroundNode(frameNode, result)
|
||||
foregroundNodes := newForegroundImageLayers(frameNode, result.Foregrounds, result.CanvasWidth, result.CanvasHeight)
|
||||
editableTextNodes := newEditableTextLayers(frameNode, result.TextExtraction)
|
||||
replacementNodes := make([]design.Node, 0, len(foregroundNodes)+len(editableTextNodes)+2)
|
||||
replacementNodes = append(replacementNodes, backgroundNode, frameNode)
|
||||
replacementNodes = append(replacementNodes, foregroundNodes...)
|
||||
replacementNodes = append(replacementNodes, editableTextNodes...)
|
||||
project.Nodes = append(project.Nodes[:nodeIndex], append(replacementNodes, project.Nodes[nodeIndex+1:]...)...)
|
||||
project.Messages = append(project.Messages, layerSeparationArtifactsMessage(threadID, result.Artifacts, now.Add(-time.Millisecond)))
|
||||
project.Messages = append(project.Messages, design.Message{
|
||||
ID: newID(),
|
||||
Role: "assistant",
|
||||
Type: "assistant",
|
||||
Title: nodeActionDoneTitle(req.Action),
|
||||
Content: nodeActionDoneContent(req.Action, req.Prompt),
|
||||
ThreadID: threadID,
|
||||
ActionID: threadID,
|
||||
StepID: newID(),
|
||||
Name: "canvas_action",
|
||||
ToolHint: "canvas_action",
|
||||
Status: "success",
|
||||
CreatedAt: now,
|
||||
})
|
||||
project.Connections = nil
|
||||
project.Status = projectStatusAfterCanvasTask(project.Nodes, design.StatusReady)
|
||||
project.UpdatedAt = now
|
||||
project = s.refreshCanvasSnapshot(project, now)
|
||||
|
||||
return s.repo.Save(ctx, project)
|
||||
}
|
||||
|
||||
func (s *DesignService) createLayerSeparationResult(ctx context.Context, projectID string, target design.Node, req design.NodeActionRequest) (layerSeparationResult, error) {
|
||||
sourceData, sourceContentType, err := loadImageContent(ctx, target.Content)
|
||||
if err != nil {
|
||||
return layerSeparationResult{}, err
|
||||
}
|
||||
source, _, err := image.Decode(bytes.NewReader(sourceData))
|
||||
if err != nil {
|
||||
return layerSeparationResult{}, err
|
||||
}
|
||||
bounds := source.Bounds()
|
||||
canvasWidth := int64(bounds.Dx())
|
||||
canvasHeight := int64(bounds.Dy())
|
||||
canvas := image.NewRGBA(bounds)
|
||||
draw.Draw(canvas, bounds, source, bounds.Min, draw.Src)
|
||||
|
||||
extraction, err := s.extractLayerSeparationText(ctx, target, req)
|
||||
if err != nil {
|
||||
return layerSeparationResult{}, err
|
||||
}
|
||||
for _, layer := range extraction.Layers {
|
||||
removeTextLayerFromImage(canvas, layer)
|
||||
}
|
||||
|
||||
foregrounds := make([]layerSeparationForeground, 0, 1)
|
||||
if fullForeground, elementForegrounds, ok := s.createLayerSeparationForegroundCandidates(ctx, target, sourceData, sourceContentType, bounds, extraction); ok {
|
||||
if len(extraction.Layers) == 0 {
|
||||
if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, fullForeground.Mask, fullForeground.Rect); ok {
|
||||
if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok {
|
||||
foregrounds = append(foregrounds, foreground)
|
||||
}
|
||||
canvas = plainBackground
|
||||
}
|
||||
}
|
||||
if len(foregrounds) == 0 {
|
||||
for _, candidate := range elementForegrounds {
|
||||
foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, candidate)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect)
|
||||
foregrounds = append(foregrounds, foreground)
|
||||
}
|
||||
}
|
||||
if len(foregrounds) == 0 && len(extraction.Layers) == 0 {
|
||||
if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok {
|
||||
removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect)
|
||||
foregrounds = append(foregrounds, foreground)
|
||||
}
|
||||
}
|
||||
if len(extraction.Layers) == 0 && len(foregrounds) == 1 {
|
||||
if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, foregrounds[0].Mask, foregrounds[0].Rect); ok {
|
||||
canvas = plainBackground
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
backgroundID := newID()
|
||||
backgroundURL, backgroundSize, err := s.persistLayerSeparationImage(ctx, projectID, backgroundID, "background", canvas)
|
||||
if err != nil {
|
||||
return layerSeparationResult{}, err
|
||||
}
|
||||
|
||||
textRenderData, err := layerSeparationTextRenderData(extraction, canvasWidth, canvasHeight)
|
||||
if err != nil {
|
||||
return layerSeparationResult{}, err
|
||||
}
|
||||
|
||||
artifacts := make([]design.GeneratorTaskArtifact, 0, len(foregrounds)+2)
|
||||
for _, foreground := range foregrounds {
|
||||
artifacts = append(artifacts, design.GeneratorTaskArtifact{
|
||||
Type: "image",
|
||||
Content: foreground.URL,
|
||||
Metadata: design.GeneratorTaskArtifactMetadata{
|
||||
ArtifactID: foreground.ID,
|
||||
BBox: append([]int64(nil), foreground.BBox...),
|
||||
Format: generatorArtifactFormat("", foreground.URL),
|
||||
GeneratorName: layerSeparationGeneratorName,
|
||||
Height: foreground.Height,
|
||||
Label: layerSeparationForegroundLabel,
|
||||
NodeID: foreground.ID,
|
||||
NodeName: "Foreground Layer",
|
||||
SizeBytes: 0,
|
||||
Width: foreground.Width,
|
||||
},
|
||||
})
|
||||
}
|
||||
artifacts = append(artifacts, design.GeneratorTaskArtifact{
|
||||
Type: "image",
|
||||
Content: backgroundURL,
|
||||
Metadata: design.GeneratorTaskArtifactMetadata{
|
||||
ArtifactID: backgroundID,
|
||||
Format: generatorArtifactFormat("", backgroundURL),
|
||||
GeneratorName: layerSeparationGeneratorName,
|
||||
Height: canvasHeight,
|
||||
Label: layerSeparationBackgroundLabel,
|
||||
NodeID: backgroundID,
|
||||
NodeName: "Image",
|
||||
SizeBytes: backgroundSize,
|
||||
Width: canvasWidth,
|
||||
},
|
||||
}, design.GeneratorTaskArtifact{
|
||||
Type: "text",
|
||||
Content: textRenderData,
|
||||
Metadata: design.GeneratorTaskArtifactMetadata{
|
||||
ArtifactID: newID(),
|
||||
GeneratorName: layerSeparationGeneratorName,
|
||||
Height: canvasHeight,
|
||||
Label: layerSeparationTextLabel,
|
||||
Width: canvasWidth,
|
||||
},
|
||||
})
|
||||
|
||||
return layerSeparationResult{
|
||||
CanvasWidth: canvasWidth,
|
||||
CanvasHeight: canvasHeight,
|
||||
BackgroundID: backgroundID,
|
||||
BackgroundURL: backgroundURL,
|
||||
Foregrounds: foregrounds,
|
||||
TextExtraction: extraction,
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *DesignService) extractLayerSeparationText(ctx context.Context, target design.Node, req design.NodeActionRequest) (design.TextExtraction, error) {
|
||||
if extraction, ok, err := textExtractionFromOptions(req.Options); ok || err != nil {
|
||||
return extraction, err
|
||||
}
|
||||
if s.textExtractor == nil {
|
||||
return design.TextExtraction{}, nil
|
||||
}
|
||||
return s.extractTextExtraction(ctx, target, req)
|
||||
}
|
||||
|
||||
func (s *DesignService) createLayerSeparationForegroundCandidates(ctx context.Context, target design.Node, sourceData []byte, sourceContentType string, bounds image.Rectangle, extraction design.TextExtraction) (layerSeparationForegroundCandidate, []layerSeparationForegroundCandidate, bool) {
|
||||
if s.backgroundRemover == nil {
|
||||
return layerSeparationForegroundCandidate{}, nil, false
|
||||
}
|
||||
removed, err := s.backgroundRemover.RemoveBackground(ctx, design.BackgroundRemovalRequest{
|
||||
Image: sourceData,
|
||||
ContentType: sourceContentType,
|
||||
Width: bounds.Dx(),
|
||||
Height: bounds.Dy(),
|
||||
})
|
||||
if err != nil || len(removed.Image) == 0 {
|
||||
return layerSeparationForegroundCandidate{}, nil, false
|
||||
}
|
||||
foregroundImage, _, err := image.Decode(bytes.NewReader(removed.Image))
|
||||
if err != nil {
|
||||
return layerSeparationForegroundCandidate{}, nil, false
|
||||
}
|
||||
foreground := image.NewNRGBA(foregroundImage.Bounds())
|
||||
draw.Draw(foreground, foreground.Bounds(), foregroundImage, foregroundImage.Bounds().Min, draw.Src)
|
||||
eraseTextLayersFromForeground(foreground, extraction)
|
||||
foregroundBounds := foreground.Bounds()
|
||||
rect, ok := alphaContentBounds(foreground, foregroundBounds, 12)
|
||||
if !ok {
|
||||
return layerSeparationForegroundCandidate{}, nil, false
|
||||
}
|
||||
mask, count := alphaMask(foreground, foregroundBounds, 12)
|
||||
if count == 0 {
|
||||
return layerSeparationForegroundCandidate{}, nil, false
|
||||
}
|
||||
full := newLayerSeparationForegroundCandidate(foreground, mask, rect, foregroundBounds)
|
||||
return full, layerSeparationElementCandidates(foreground, foregroundBounds), true
|
||||
}
|
||||
|
||||
func (s *DesignService) persistLayerSeparationForeground(ctx context.Context, projectID string, candidate layerSeparationForegroundCandidate) (layerSeparationForeground, bool) {
|
||||
if candidate.Image == nil || candidate.Mask == nil || candidate.Rect.Empty() || len(candidate.BBox) < 4 {
|
||||
return layerSeparationForeground{}, false
|
||||
}
|
||||
foregroundID := newID()
|
||||
foregroundURL, _, err := s.persistLayerSeparationImage(ctx, projectID, foregroundID, "foreground", candidate.Image)
|
||||
if err != nil {
|
||||
return layerSeparationForeground{}, false
|
||||
}
|
||||
return layerSeparationForeground{
|
||||
ID: foregroundID,
|
||||
URL: foregroundURL,
|
||||
BBox: append([]int64(nil), candidate.BBox...),
|
||||
Width: candidate.Width,
|
||||
Height: candidate.Height,
|
||||
Mask: candidate.Mask,
|
||||
Rect: candidate.Rect,
|
||||
}, true
|
||||
}
|
||||
|
||||
func newLayerSeparationForegroundCandidate(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle, bounds image.Rectangle) layerSeparationForegroundCandidate {
|
||||
return layerSeparationForegroundCandidate{
|
||||
BBox: []int64{
|
||||
int64(rect.Min.X - bounds.Min.X),
|
||||
int64(rect.Min.Y - bounds.Min.Y),
|
||||
int64(rect.Max.X - bounds.Min.X),
|
||||
int64(rect.Max.Y - bounds.Min.Y),
|
||||
},
|
||||
Width: int64(rect.Dx()),
|
||||
Height: int64(rect.Dy()),
|
||||
Mask: mask,
|
||||
Rect: rect,
|
||||
Image: foregroundImageFromMask(foreground, mask, rect),
|
||||
}
|
||||
}
|
||||
|
||||
func eraseTextLayersFromForeground(foreground *image.NRGBA, extraction design.TextExtraction) {
|
||||
if foreground == nil || len(extraction.Layers) == 0 {
|
||||
return
|
||||
}
|
||||
bounds := foreground.Bounds()
|
||||
if bounds.Dx()*bounds.Dy() < 4096 {
|
||||
return
|
||||
}
|
||||
for _, layer := range extraction.Layers {
|
||||
rect := normalizedTextMaskRect(bounds, layer)
|
||||
if rect.Empty() {
|
||||
continue
|
||||
}
|
||||
padding := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.14)), 2, 18)
|
||||
rect = image.Rect(rect.Min.X-padding, rect.Min.Y-padding, rect.Max.X+padding, rect.Max.Y+padding).Intersect(bounds)
|
||||
for y := rect.Min.Y; y < rect.Max.Y; y++ {
|
||||
for x := rect.Min.X; x < rect.Max.X; x++ {
|
||||
offset := foreground.PixOffset(x, y)
|
||||
foreground.Pix[offset+3] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func layerSeparationElementCandidates(foreground *image.NRGBA, bounds image.Rectangle) []layerSeparationForegroundCandidate {
|
||||
if foreground == nil || bounds.Empty() {
|
||||
return nil
|
||||
}
|
||||
width := bounds.Dx()
|
||||
height := bounds.Dy()
|
||||
visited := make([]bool, width*height)
|
||||
components := make([]layerSeparationForegroundCandidate, 0, 4)
|
||||
indexOf := func(x int, y int) int {
|
||||
return (y-bounds.Min.Y)*width + (x - bounds.Min.X)
|
||||
}
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
index := indexOf(x, y)
|
||||
if visited[index] || foreground.NRGBAAt(x, y).A <= 12 {
|
||||
visited[index] = true
|
||||
continue
|
||||
}
|
||||
mask, rect, count := layerSeparationComponentMask(foreground, bounds, visited, x, y, indexOf)
|
||||
if !keepLayerSeparationElement(rect, count, bounds) {
|
||||
continue
|
||||
}
|
||||
components = append(components, newLayerSeparationForegroundCandidate(foreground, mask, rect, bounds))
|
||||
}
|
||||
}
|
||||
sort.Slice(components, func(i int, j int) bool {
|
||||
leftArea := components[i].Rect.Dx() * components[i].Rect.Dy()
|
||||
rightArea := components[j].Rect.Dx() * components[j].Rect.Dy()
|
||||
return leftArea > rightArea
|
||||
})
|
||||
if len(components) > 12 {
|
||||
components = components[:12]
|
||||
}
|
||||
return components
|
||||
}
|
||||
|
||||
func layerSeparationComponentMask(foreground *image.NRGBA, bounds image.Rectangle, visited []bool, startX int, startY int, indexOf func(int, int) int) (*image.Alpha, image.Rectangle, int) {
|
||||
mask := image.NewAlpha(bounds)
|
||||
stack := []image.Point{{X: startX, Y: startY}}
|
||||
minX, minY := startX, startY
|
||||
maxX, maxY := startX+1, startY+1
|
||||
count := 0
|
||||
for len(stack) > 0 {
|
||||
point := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
if point.X < bounds.Min.X || point.X >= bounds.Max.X || point.Y < bounds.Min.Y || point.Y >= bounds.Max.Y {
|
||||
continue
|
||||
}
|
||||
index := indexOf(point.X, point.Y)
|
||||
if visited[index] {
|
||||
continue
|
||||
}
|
||||
visited[index] = true
|
||||
if foreground.NRGBAAt(point.X, point.Y).A <= 12 {
|
||||
continue
|
||||
}
|
||||
mask.SetAlpha(point.X, point.Y, color.Alpha{A: 255})
|
||||
count++
|
||||
if point.X < minX {
|
||||
minX = point.X
|
||||
}
|
||||
if point.Y < minY {
|
||||
minY = point.Y
|
||||
}
|
||||
if point.X+1 > maxX {
|
||||
maxX = point.X + 1
|
||||
}
|
||||
if point.Y+1 > maxY {
|
||||
maxY = point.Y + 1
|
||||
}
|
||||
stack = append(stack,
|
||||
image.Point{X: point.X + 1, Y: point.Y},
|
||||
image.Point{X: point.X - 1, Y: point.Y},
|
||||
image.Point{X: point.X, Y: point.Y + 1},
|
||||
image.Point{X: point.X, Y: point.Y - 1},
|
||||
)
|
||||
}
|
||||
return mask, image.Rect(minX, minY, maxX, maxY), count
|
||||
}
|
||||
|
||||
func keepLayerSeparationElement(rect image.Rectangle, count int, bounds image.Rectangle) bool {
|
||||
if rect.Empty() || count <= 0 {
|
||||
return false
|
||||
}
|
||||
totalArea := bounds.Dx() * bounds.Dy()
|
||||
if totalArea <= 0 {
|
||||
return false
|
||||
}
|
||||
if totalArea < 4096 {
|
||||
return true
|
||||
}
|
||||
bboxArea := rect.Dx() * rect.Dy()
|
||||
minArea := maxInt(96, totalArea/6000)
|
||||
if count < minArea || bboxArea < minArea {
|
||||
return false
|
||||
}
|
||||
if float64(bboxArea)/float64(totalArea) > 0.18 {
|
||||
return false
|
||||
}
|
||||
if float64(rect.Dx())/float64(bounds.Dx()) > 0.62 || float64(rect.Dy())/float64(bounds.Dy()) > 0.62 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func foregroundImageFromMask(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle) image.Image {
|
||||
cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy()))
|
||||
for y := rect.Min.Y; y < rect.Max.Y; y++ {
|
||||
for x := rect.Min.X; x < rect.Max.X; x++ {
|
||||
if mask.AlphaAt(x, y).A == 0 {
|
||||
continue
|
||||
}
|
||||
cropped.SetNRGBA(x-rect.Min.X, y-rect.Min.Y, foreground.NRGBAAt(x, y))
|
||||
}
|
||||
}
|
||||
return cropped
|
||||
}
|
||||
|
||||
func (s *DesignService) persistLayerSeparationImage(ctx context.Context, projectID string, nodeID string, suffix string, img image.Image) (string, int64, error) {
|
||||
var output bytes.Buffer
|
||||
if err := png.Encode(&output, img); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if s.assets == nil {
|
||||
data := output.Bytes()
|
||||
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(data), int64(len(data)), nil
|
||||
}
|
||||
data, contentType, err := encodeRasterAssetAsWebP(output.Bytes(), "image/png")
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
object, err := s.assets.PutObject(ctx, design.AssetObjectRequest{
|
||||
FileName: fmt.Sprintf("%s-%s-%s.png", projectID, nodeID, suffix),
|
||||
ContentType: contentType,
|
||||
Data: data,
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if strings.TrimSpace(object.PublicURL) == "" {
|
||||
return "", 0, fmt.Errorf("layer separation upload returned empty public URL")
|
||||
}
|
||||
return object.PublicURL, int64(len(data)), nil
|
||||
}
|
||||
|
||||
func newLayerSeparationBackgroundNode(frame design.Node, result layerSeparationResult) design.Node {
|
||||
return design.Node{
|
||||
ID: result.BackgroundID,
|
||||
Type: design.NodeTypeImage,
|
||||
Title: "Image",
|
||||
X: frame.X,
|
||||
Y: frame.Y,
|
||||
Width: frame.Width,
|
||||
Height: frame.Height,
|
||||
Content: result.BackgroundURL,
|
||||
Tone: "visual",
|
||||
Status: "success",
|
||||
LayerRole: "clean-image",
|
||||
}
|
||||
}
|
||||
|
||||
func newForegroundImageLayers(target design.Node, foregrounds []layerSeparationForeground, canvasWidth int64, canvasHeight int64) []design.Node {
|
||||
if len(foregrounds) == 0 || canvasWidth <= 0 || canvasHeight <= 0 {
|
||||
return nil
|
||||
}
|
||||
nodes := make([]design.Node, 0, len(foregrounds))
|
||||
for index, foreground := range foregrounds {
|
||||
if len(foreground.BBox) < 4 || strings.TrimSpace(foreground.URL) == "" {
|
||||
continue
|
||||
}
|
||||
x1 := clampFloat(float64(foreground.BBox[0])/float64(canvasWidth), 0, 1)
|
||||
y1 := clampFloat(float64(foreground.BBox[1])/float64(canvasHeight), 0, 1)
|
||||
x2 := clampFloat(float64(foreground.BBox[2])/float64(canvasWidth), x1, 1)
|
||||
y2 := clampFloat(float64(foreground.BBox[3])/float64(canvasHeight), y1, 1)
|
||||
width := target.Width * (x2 - x1)
|
||||
height := target.Height * (y2 - y1)
|
||||
if width <= 1 || height <= 1 {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, design.Node{
|
||||
ID: foreground.ID,
|
||||
Type: design.NodeTypeImage,
|
||||
Title: fmt.Sprintf("Foreground Layer %d", index+1),
|
||||
X: target.X + target.Width*x1,
|
||||
Y: target.Y + target.Height*y1,
|
||||
Width: width,
|
||||
Height: height,
|
||||
Content: foreground.URL,
|
||||
Tone: "separated-element",
|
||||
Status: "success",
|
||||
ParentID: target.ID,
|
||||
LayerRole: "foreground-image",
|
||||
})
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func layerSeparationArtifactsMessage(threadID string, artifacts []design.GeneratorTaskArtifact, createdAt time.Time) design.Message {
|
||||
content, _ := json.Marshal(toLayerSeparationArtifactWire(artifacts))
|
||||
return design.Message{
|
||||
ID: newID(),
|
||||
Role: "tool",
|
||||
Type: "tool_use",
|
||||
Title: layerSeparationMessageName,
|
||||
Content: string(content),
|
||||
ThreadID: threadID,
|
||||
ActionID: threadID,
|
||||
StepID: newID(),
|
||||
ToolCallID: "call_" + strings.ReplaceAll(newID(), "-", ""),
|
||||
Name: layerSeparationMessageName,
|
||||
ToolHint: layerSeparationMessageName,
|
||||
Status: "success",
|
||||
CreatedAt: createdAt,
|
||||
}
|
||||
}
|
||||
|
||||
func toLayerSeparationArtifactWire(artifacts []design.GeneratorTaskArtifact) []map[string]any {
|
||||
items := make([]map[string]any, 0, len(artifacts))
|
||||
for _, artifact := range artifacts {
|
||||
items = append(items, map[string]any{
|
||||
"type": artifact.Type,
|
||||
"content": artifact.Content,
|
||||
"metadata": map[string]any{
|
||||
"artifact_id": artifact.Metadata.ArtifactID,
|
||||
"bbox": artifact.Metadata.BBox,
|
||||
"format": artifact.Metadata.Format,
|
||||
"generator_name": artifact.Metadata.GeneratorName,
|
||||
"height": artifact.Metadata.Height,
|
||||
"label": artifact.Metadata.Label,
|
||||
"node_id": artifact.Metadata.NodeID,
|
||||
"node_name": artifact.Metadata.NodeName,
|
||||
"size_bytes": artifact.Metadata.SizeBytes,
|
||||
"width": artifact.Metadata.Width,
|
||||
},
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func removeForegroundMaskFromImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) {
|
||||
if mask == nil || rect.Empty() {
|
||||
return
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
rect = rect.Intersect(bounds)
|
||||
if rect.Empty() {
|
||||
return
|
||||
}
|
||||
radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.025)), 2, 10)
|
||||
mask, count := dilateTextMask(mask, rect, radius)
|
||||
if count == 0 {
|
||||
return
|
||||
}
|
||||
source := image.NewRGBA(bounds)
|
||||
draw.Draw(source, bounds, img, bounds.Min, draw.Src)
|
||||
for y := rect.Min.Y; y < rect.Max.Y; y++ {
|
||||
for x := rect.Min.X; x < rect.Max.X; x++ {
|
||||
if mask.AlphaAt(x, y).A == 0 {
|
||||
continue
|
||||
}
|
||||
img.SetRGBA(x, y, estimatedRectBackgroundColor(source, rect, x, y))
|
||||
}
|
||||
}
|
||||
softenMaskedPixels(img, mask, rect)
|
||||
}
|
||||
|
||||
func plainLayerSeparationBackgroundImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) (*image.RGBA, bool) {
|
||||
if img == nil || mask == nil || rect.Empty() {
|
||||
return nil, false
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
if bounds.Empty() {
|
||||
return nil, false
|
||||
}
|
||||
radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.02)), 3, 18)
|
||||
expandedMask, _ := dilateTextMask(mask, bounds, radius)
|
||||
stats, ok := unmaskedColorStats(img, expandedMask, bounds)
|
||||
if !ok || stats.StdDev > 24 {
|
||||
return nil, false
|
||||
}
|
||||
bg := image.NewRGBA(bounds)
|
||||
fill := colorRGBA(int(math.Round(stats.R)), int(math.Round(stats.G)), int(math.Round(stats.B)), int(math.Round(stats.A)))
|
||||
draw.Draw(bg, bounds, &image.Uniform{C: fill}, image.Point{}, draw.Src)
|
||||
return bg, true
|
||||
}
|
||||
|
||||
type layerSeparationColorStats struct {
|
||||
Count int64
|
||||
R float64
|
||||
G float64
|
||||
B float64
|
||||
A float64
|
||||
StdDev float64
|
||||
}
|
||||
|
||||
func unmaskedColorStats(img *image.RGBA, mask *image.Alpha, bounds image.Rectangle) (layerSeparationColorStats, bool) {
|
||||
stride := clampInt(int(math.Ceil(float64(maxInt(bounds.Dx(), bounds.Dy()))/640)), 1, 12)
|
||||
var count int64
|
||||
var r, g, b, a float64
|
||||
var rr, gg, bb float64
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y += stride {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x += stride {
|
||||
if mask.AlphaAt(x, y).A != 0 {
|
||||
continue
|
||||
}
|
||||
c := img.RGBAAt(x, y)
|
||||
cfR := float64(c.R)
|
||||
cfG := float64(c.G)
|
||||
cfB := float64(c.B)
|
||||
cfA := float64(c.A)
|
||||
r += cfR
|
||||
g += cfG
|
||||
b += cfB
|
||||
a += cfA
|
||||
rr += cfR * cfR
|
||||
gg += cfG * cfG
|
||||
bb += cfB * cfB
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count < 32 {
|
||||
return layerSeparationColorStats{}, false
|
||||
}
|
||||
divisor := float64(count)
|
||||
meanR := r / divisor
|
||||
meanG := g / divisor
|
||||
meanB := b / divisor
|
||||
varR := math.Max(0, rr/divisor-meanR*meanR)
|
||||
varG := math.Max(0, gg/divisor-meanG*meanG)
|
||||
varB := math.Max(0, bb/divisor-meanB*meanB)
|
||||
return layerSeparationColorStats{
|
||||
Count: count,
|
||||
R: meanR,
|
||||
G: meanG,
|
||||
B: meanB,
|
||||
A: a / divisor,
|
||||
StdDev: math.Sqrt((varR + varG + varB) / 3),
|
||||
}, true
|
||||
}
|
||||
|
||||
func alphaContentBounds(img image.Image, bounds image.Rectangle, threshold uint8) (image.Rectangle, bool) {
|
||||
minX, minY := bounds.Max.X, bounds.Max.Y
|
||||
maxX, maxY := bounds.Min.X, bounds.Min.Y
|
||||
found := false
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
if alphaAt(img, x, y) <= threshold {
|
||||
continue
|
||||
}
|
||||
if x < minX {
|
||||
minX = x
|
||||
}
|
||||
if y < minY {
|
||||
minY = y
|
||||
}
|
||||
if x+1 > maxX {
|
||||
maxX = x + 1
|
||||
}
|
||||
if y+1 > maxY {
|
||||
maxY = y + 1
|
||||
}
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return image.Rectangle{}, false
|
||||
}
|
||||
return image.Rect(minX, minY, maxX, maxY), true
|
||||
}
|
||||
|
||||
func alphaMask(img image.Image, bounds image.Rectangle, threshold uint8) (*image.Alpha, int) {
|
||||
mask := image.NewAlpha(bounds)
|
||||
count := 0
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
if alphaAt(img, x, y) <= threshold {
|
||||
continue
|
||||
}
|
||||
mask.SetAlpha(x, y, color.Alpha{A: 255})
|
||||
count++
|
||||
}
|
||||
}
|
||||
return mask, count
|
||||
}
|
||||
|
||||
func alphaAt(img image.Image, x int, y int) uint8 {
|
||||
_, _, _, a := img.At(x, y).RGBA()
|
||||
return uint8(a >> 8)
|
||||
}
|
||||
|
||||
func cropImage(img image.Image, rect image.Rectangle) image.Image {
|
||||
cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy()))
|
||||
draw.Draw(cropped, cropped.Bounds(), img, rect.Min, draw.Src)
|
||||
return cropped
|
||||
}
|
||||
|
||||
type layerSeparationTextDocument struct {
|
||||
Layers []layerSeparationTextLayer `json:"layers"`
|
||||
CanvasSize layerSeparationCanvasSize `json:"canvas_size"`
|
||||
}
|
||||
|
||||
type layerSeparationCanvasSize struct {
|
||||
Width int64 `json:"width"`
|
||||
Height int64 `json:"height"`
|
||||
}
|
||||
|
||||
type layerSeparationTextLayer struct {
|
||||
TextInfo layerSeparationTextInfo `json:"text_info"`
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
W float64 `json:"w"`
|
||||
H float64 `json:"h"`
|
||||
}
|
||||
|
||||
type layerSeparationTextInfo struct {
|
||||
Text string `json:"text"`
|
||||
FontSizePx int64 `json:"font_size_px"`
|
||||
FontFamily string `json:"font_family"`
|
||||
FontWeight any `json:"font_weight"`
|
||||
FontStyle string `json:"font_style"`
|
||||
ColorCSS string `json:"color_css"`
|
||||
TextAlign string `json:"text_align"`
|
||||
Leading float64 `json:"leading"`
|
||||
Tracking float64 `json:"tracking"`
|
||||
Rotation float64 `json:"rotation"`
|
||||
LeadingTrim string `json:"leading_trim"`
|
||||
}
|
||||
|
||||
func layerSeparationTextRenderData(extraction design.TextExtraction, canvasWidth int64, canvasHeight int64) (string, error) {
|
||||
doc := layerSeparationTextDocument{
|
||||
Layers: make([]layerSeparationTextLayer, 0, len(extraction.Layers)),
|
||||
CanvasSize: layerSeparationCanvasSize{
|
||||
Width: canvasWidth,
|
||||
Height: canvasHeight,
|
||||
},
|
||||
}
|
||||
for _, layer := range extraction.Layers {
|
||||
text := strings.TrimSpace(layer.Text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
x := clamp01(layer.X)
|
||||
y := clamp01(layer.Y)
|
||||
width := clamp01(layer.Width)
|
||||
height := clamp01(layer.Height)
|
||||
if x+width > 1 {
|
||||
width = 1 - x
|
||||
}
|
||||
if y+height > 1 {
|
||||
height = 1 - y
|
||||
}
|
||||
if width <= 0 || height <= 0 {
|
||||
continue
|
||||
}
|
||||
fontSize := layer.FontSize
|
||||
if fontSize <= 0 {
|
||||
fontSize = math.Max(10, float64(canvasHeight)*height*0.72)
|
||||
}
|
||||
lineHeight := layer.LineHeight
|
||||
if lineHeight <= 0 {
|
||||
lineHeight = 1.2
|
||||
}
|
||||
fontFamily := strings.TrimSpace(layer.FontFamily)
|
||||
if fontFamily == "" {
|
||||
fontFamily = "Inter"
|
||||
}
|
||||
doc.Layers = append(doc.Layers, layerSeparationTextLayer{
|
||||
TextInfo: layerSeparationTextInfo{
|
||||
Text: text,
|
||||
FontSizePx: int64(math.Round(fontSize)),
|
||||
FontFamily: fontFamily,
|
||||
FontWeight: layerSeparationFontWeight(layer.FontWeight),
|
||||
FontStyle: layerSeparationFontStyle(layer.FontWeight),
|
||||
ColorCSS: layerSeparationColorCSS(layer.Color),
|
||||
TextAlign: layerSeparationTextAlign(layer.TextAlign),
|
||||
Leading: lineHeight,
|
||||
Tracking: layer.LetterSpacing,
|
||||
Rotation: layer.Rotation,
|
||||
LeadingTrim: "CAP_HEIGHT",
|
||||
},
|
||||
X: x * float64(canvasWidth),
|
||||
Y: y * float64(canvasHeight),
|
||||
W: width * float64(canvasWidth),
|
||||
H: height * float64(canvasHeight),
|
||||
})
|
||||
}
|
||||
data, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func layerSeparationFontWeight(value string) any {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return 400
|
||||
}
|
||||
if weight, err := strconv.Atoi(value); err == nil {
|
||||
return weight
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func layerSeparationFontStyle(value string) string {
|
||||
if strings.Contains(strings.ToLower(strings.TrimSpace(value)), "italic") {
|
||||
return "italic"
|
||||
}
|
||||
return "normal"
|
||||
}
|
||||
|
||||
func layerSeparationTextAlign(value string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "left":
|
||||
return "start"
|
||||
case "right":
|
||||
return "end"
|
||||
case "center":
|
||||
return "center"
|
||||
default:
|
||||
return "start"
|
||||
}
|
||||
}
|
||||
|
||||
func layerSeparationColorCSS(value string) string {
|
||||
parsed, ok := parseCanvasColor(value)
|
||||
if !ok {
|
||||
return "rgba(25, 25, 25, 1.0)"
|
||||
}
|
||||
alpha := float64(parsed.A) / 255
|
||||
return fmt.Sprintf("rgba(%d, %d, %d, %.3g)", parsed.R, parsed.G, parsed.B, alpha)
|
||||
}
|
||||
Reference in New Issue
Block a user