893 lines
28 KiB
Go
893 lines
28 KiB
Go
|
package application
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"encoding/base64"
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"image"
|
||
|
|
"image/color"
|
||
|
|
"image/draw"
|
||
|
|
"image/png"
|
||
|
|
"math"
|
||
|
|
"sort"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"img_infinite_canvas/internal/domain/design"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
layerSeparationForegroundLabel = "fg_image"
|
||
|
|
layerSeparationBackgroundLabel = "bg_image"
|
||
|
|
layerSeparationTextLabel = "text_render_data"
|
||
|
|
)
|
||
|
|
|
||
|
|
type layerSeparationResult struct {
|
||
|
|
CanvasWidth int64
|
||
|
|
CanvasHeight int64
|
||
|
|
BackgroundID string
|
||
|
|
BackgroundURL string
|
||
|
|
Foregrounds []layerSeparationForeground
|
||
|
|
TextExtraction design.TextExtraction
|
||
|
|
Artifacts []design.GeneratorTaskArtifact
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationForeground struct {
|
||
|
|
ID string
|
||
|
|
URL string
|
||
|
|
BBox []int64
|
||
|
|
Width int64
|
||
|
|
Height int64
|
||
|
|
Mask *image.Alpha
|
||
|
|
Rect image.Rectangle
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationForegroundCandidate struct {
|
||
|
|
BBox []int64
|
||
|
|
Width int64
|
||
|
|
Height int64
|
||
|
|
Mask *image.Alpha
|
||
|
|
Rect image.Rectangle
|
||
|
|
Image image.Image
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *DesignService) completeLayerSeparationGeneration(ctx context.Context, projectID string, threadID string, target design.Node, req design.NodeActionRequest) error {
|
||
|
|
resultNodeID := nodeActionResultNodeID(target, req)
|
||
|
|
if err := s.addGenerationStep(ctx, projectID, threadID, "tool", "图层分离", "正在拆分底图、前景元素和可编辑文字层。"); err != nil {
|
||
|
|
return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
resultTarget := target
|
||
|
|
resultTarget.ID = resultNodeID
|
||
|
|
result, err := s.createLayerSeparationResult(ctx, projectID, resultTarget, req)
|
||
|
|
if err != nil {
|
||
|
|
return s.markNodeActionFailed(detachedUserContext(ctx), projectID, threadID, resultNodeID, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
project, err := s.repo.Get(ctx, projectID)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
nodeIndex := findNodeIndex(project.Nodes, resultNodeID)
|
||
|
|
if nodeIndex < 0 {
|
||
|
|
project.Nodes = append(project.Nodes, newLayerSeparationPlaceholder(target))
|
||
|
|
nodeIndex = len(project.Nodes) - 1
|
||
|
|
project.Nodes[nodeIndex].ID = resultNodeID
|
||
|
|
}
|
||
|
|
|
||
|
|
frameNode := project.Nodes[nodeIndex]
|
||
|
|
if frameNode.Width <= 0 {
|
||
|
|
frameNode.Width = float64(result.CanvasWidth)
|
||
|
|
}
|
||
|
|
if frameNode.Height <= 0 {
|
||
|
|
frameNode.Height = float64(result.CanvasHeight)
|
||
|
|
}
|
||
|
|
frameNode.Type = design.NodeTypeFrame
|
||
|
|
frameNode.Title = "Frame"
|
||
|
|
frameNode.Content = ""
|
||
|
|
frameNode.Tone = "frame"
|
||
|
|
frameNode.Status = "success"
|
||
|
|
frameNode.ParentID = ""
|
||
|
|
frameNode.LayerRole = "manual-frame"
|
||
|
|
frameNode.FillColor = "transparent"
|
||
|
|
frameNode.StrokeColor = "#C7C7C7"
|
||
|
|
frameNode.StrokeWidth = 1
|
||
|
|
frameNode.StrokeStyle = "solid"
|
||
|
|
frameNode.MockupSurface = design.MockupSurface{}
|
||
|
|
frameNode.MockupModel = design.MockupModel{}
|
||
|
|
frameNode.MockupSourceContent = ""
|
||
|
|
|
||
|
|
now := s.now()
|
||
|
|
backgroundNode := newLayerSeparationBackgroundNode(frameNode, result)
|
||
|
|
foregroundNodes := newForegroundImageLayers(frameNode, result.Foregrounds, result.CanvasWidth, result.CanvasHeight)
|
||
|
|
editableTextNodes := newEditableTextLayers(frameNode, result.TextExtraction)
|
||
|
|
replacementNodes := make([]design.Node, 0, len(foregroundNodes)+len(editableTextNodes)+2)
|
||
|
|
replacementNodes = append(replacementNodes, backgroundNode, frameNode)
|
||
|
|
replacementNodes = append(replacementNodes, foregroundNodes...)
|
||
|
|
replacementNodes = append(replacementNodes, editableTextNodes...)
|
||
|
|
project.Nodes = append(project.Nodes[:nodeIndex], append(replacementNodes, project.Nodes[nodeIndex+1:]...)...)
|
||
|
|
project.Messages = append(project.Messages, layerSeparationArtifactsMessage(threadID, result.Artifacts, now.Add(-time.Millisecond)))
|
||
|
|
project.Messages = append(project.Messages, design.Message{
|
||
|
|
ID: newID(),
|
||
|
|
Role: "assistant",
|
||
|
|
Type: "assistant",
|
||
|
|
Title: nodeActionDoneTitle(req.Action),
|
||
|
|
Content: nodeActionDoneContent(req.Action, req.Prompt),
|
||
|
|
ThreadID: threadID,
|
||
|
|
ActionID: threadID,
|
||
|
|
StepID: newID(),
|
||
|
|
Name: "canvas_action",
|
||
|
|
ToolHint: "canvas_action",
|
||
|
|
Status: "success",
|
||
|
|
CreatedAt: now,
|
||
|
|
})
|
||
|
|
project.Connections = nil
|
||
|
|
project.Status = projectStatusAfterCanvasTask(project.Nodes, design.StatusReady)
|
||
|
|
project.UpdatedAt = now
|
||
|
|
project = s.refreshCanvasSnapshot(project, now)
|
||
|
|
|
||
|
|
return s.repo.Save(ctx, project)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *DesignService) createLayerSeparationResult(ctx context.Context, projectID string, target design.Node, req design.NodeActionRequest) (layerSeparationResult, error) {
|
||
|
|
sourceData, sourceContentType, err := loadImageContent(ctx, target.Content)
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationResult{}, err
|
||
|
|
}
|
||
|
|
source, _, err := image.Decode(bytes.NewReader(sourceData))
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationResult{}, err
|
||
|
|
}
|
||
|
|
bounds := source.Bounds()
|
||
|
|
canvasWidth := int64(bounds.Dx())
|
||
|
|
canvasHeight := int64(bounds.Dy())
|
||
|
|
canvas := image.NewRGBA(bounds)
|
||
|
|
draw.Draw(canvas, bounds, source, bounds.Min, draw.Src)
|
||
|
|
|
||
|
|
extraction, err := s.extractLayerSeparationText(ctx, target, req)
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationResult{}, err
|
||
|
|
}
|
||
|
|
for _, layer := range extraction.Layers {
|
||
|
|
removeTextLayerFromImage(canvas, layer)
|
||
|
|
}
|
||
|
|
|
||
|
|
foregrounds := make([]layerSeparationForeground, 0, 1)
|
||
|
|
if fullForeground, elementForegrounds, ok := s.createLayerSeparationForegroundCandidates(ctx, target, sourceData, sourceContentType, bounds, extraction); ok {
|
||
|
|
if len(extraction.Layers) == 0 {
|
||
|
|
if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, fullForeground.Mask, fullForeground.Rect); ok {
|
||
|
|
if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok {
|
||
|
|
foregrounds = append(foregrounds, foreground)
|
||
|
|
}
|
||
|
|
canvas = plainBackground
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(foregrounds) == 0 {
|
||
|
|
for _, candidate := range elementForegrounds {
|
||
|
|
foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, candidate)
|
||
|
|
if !ok {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect)
|
||
|
|
foregrounds = append(foregrounds, foreground)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(foregrounds) == 0 && len(extraction.Layers) == 0 {
|
||
|
|
if foreground, ok := s.persistLayerSeparationForeground(ctx, projectID, fullForeground); ok {
|
||
|
|
removeForegroundMaskFromImage(canvas, foreground.Mask, foreground.Rect)
|
||
|
|
foregrounds = append(foregrounds, foreground)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(extraction.Layers) == 0 && len(foregrounds) == 1 {
|
||
|
|
if plainBackground, ok := plainLayerSeparationBackgroundImage(canvas, foregrounds[0].Mask, foregrounds[0].Rect); ok {
|
||
|
|
canvas = plainBackground
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
backgroundID := newID()
|
||
|
|
backgroundURL, backgroundSize, err := s.persistLayerSeparationImage(ctx, projectID, backgroundID, "background", canvas)
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationResult{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
textRenderData, err := layerSeparationTextRenderData(extraction, canvasWidth, canvasHeight)
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationResult{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
artifacts := make([]design.GeneratorTaskArtifact, 0, len(foregrounds)+2)
|
||
|
|
for _, foreground := range foregrounds {
|
||
|
|
artifacts = append(artifacts, design.GeneratorTaskArtifact{
|
||
|
|
Type: "image",
|
||
|
|
Content: foreground.URL,
|
||
|
|
Metadata: design.GeneratorTaskArtifactMetadata{
|
||
|
|
ArtifactID: foreground.ID,
|
||
|
|
BBox: append([]int64(nil), foreground.BBox...),
|
||
|
|
Format: generatorArtifactFormat("", foreground.URL),
|
||
|
|
GeneratorName: layerSeparationGeneratorName,
|
||
|
|
Height: foreground.Height,
|
||
|
|
Label: layerSeparationForegroundLabel,
|
||
|
|
NodeID: foreground.ID,
|
||
|
|
NodeName: "Foreground Layer",
|
||
|
|
SizeBytes: 0,
|
||
|
|
Width: foreground.Width,
|
||
|
|
},
|
||
|
|
})
|
||
|
|
}
|
||
|
|
artifacts = append(artifacts, design.GeneratorTaskArtifact{
|
||
|
|
Type: "image",
|
||
|
|
Content: backgroundURL,
|
||
|
|
Metadata: design.GeneratorTaskArtifactMetadata{
|
||
|
|
ArtifactID: backgroundID,
|
||
|
|
Format: generatorArtifactFormat("", backgroundURL),
|
||
|
|
GeneratorName: layerSeparationGeneratorName,
|
||
|
|
Height: canvasHeight,
|
||
|
|
Label: layerSeparationBackgroundLabel,
|
||
|
|
NodeID: backgroundID,
|
||
|
|
NodeName: "Image",
|
||
|
|
SizeBytes: backgroundSize,
|
||
|
|
Width: canvasWidth,
|
||
|
|
},
|
||
|
|
}, design.GeneratorTaskArtifact{
|
||
|
|
Type: "text",
|
||
|
|
Content: textRenderData,
|
||
|
|
Metadata: design.GeneratorTaskArtifactMetadata{
|
||
|
|
ArtifactID: newID(),
|
||
|
|
GeneratorName: layerSeparationGeneratorName,
|
||
|
|
Height: canvasHeight,
|
||
|
|
Label: layerSeparationTextLabel,
|
||
|
|
Width: canvasWidth,
|
||
|
|
},
|
||
|
|
})
|
||
|
|
|
||
|
|
return layerSeparationResult{
|
||
|
|
CanvasWidth: canvasWidth,
|
||
|
|
CanvasHeight: canvasHeight,
|
||
|
|
BackgroundID: backgroundID,
|
||
|
|
BackgroundURL: backgroundURL,
|
||
|
|
Foregrounds: foregrounds,
|
||
|
|
TextExtraction: extraction,
|
||
|
|
Artifacts: artifacts,
|
||
|
|
}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *DesignService) extractLayerSeparationText(ctx context.Context, target design.Node, req design.NodeActionRequest) (design.TextExtraction, error) {
|
||
|
|
if extraction, ok, err := textExtractionFromOptions(req.Options); ok || err != nil {
|
||
|
|
return extraction, err
|
||
|
|
}
|
||
|
|
if s.textExtractor == nil {
|
||
|
|
return design.TextExtraction{}, nil
|
||
|
|
}
|
||
|
|
return s.extractTextExtraction(ctx, target, req)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *DesignService) createLayerSeparationForegroundCandidates(ctx context.Context, target design.Node, sourceData []byte, sourceContentType string, bounds image.Rectangle, extraction design.TextExtraction) (layerSeparationForegroundCandidate, []layerSeparationForegroundCandidate, bool) {
|
||
|
|
if s.backgroundRemover == nil {
|
||
|
|
return layerSeparationForegroundCandidate{}, nil, false
|
||
|
|
}
|
||
|
|
removed, err := s.backgroundRemover.RemoveBackground(ctx, design.BackgroundRemovalRequest{
|
||
|
|
Image: sourceData,
|
||
|
|
ContentType: sourceContentType,
|
||
|
|
Width: bounds.Dx(),
|
||
|
|
Height: bounds.Dy(),
|
||
|
|
})
|
||
|
|
if err != nil || len(removed.Image) == 0 {
|
||
|
|
return layerSeparationForegroundCandidate{}, nil, false
|
||
|
|
}
|
||
|
|
foregroundImage, _, err := image.Decode(bytes.NewReader(removed.Image))
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationForegroundCandidate{}, nil, false
|
||
|
|
}
|
||
|
|
foreground := image.NewNRGBA(foregroundImage.Bounds())
|
||
|
|
draw.Draw(foreground, foreground.Bounds(), foregroundImage, foregroundImage.Bounds().Min, draw.Src)
|
||
|
|
eraseTextLayersFromForeground(foreground, extraction)
|
||
|
|
foregroundBounds := foreground.Bounds()
|
||
|
|
rect, ok := alphaContentBounds(foreground, foregroundBounds, 12)
|
||
|
|
if !ok {
|
||
|
|
return layerSeparationForegroundCandidate{}, nil, false
|
||
|
|
}
|
||
|
|
mask, count := alphaMask(foreground, foregroundBounds, 12)
|
||
|
|
if count == 0 {
|
||
|
|
return layerSeparationForegroundCandidate{}, nil, false
|
||
|
|
}
|
||
|
|
full := newLayerSeparationForegroundCandidate(foreground, mask, rect, foregroundBounds)
|
||
|
|
return full, layerSeparationElementCandidates(foreground, foregroundBounds), true
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *DesignService) persistLayerSeparationForeground(ctx context.Context, projectID string, candidate layerSeparationForegroundCandidate) (layerSeparationForeground, bool) {
|
||
|
|
if candidate.Image == nil || candidate.Mask == nil || candidate.Rect.Empty() || len(candidate.BBox) < 4 {
|
||
|
|
return layerSeparationForeground{}, false
|
||
|
|
}
|
||
|
|
foregroundID := newID()
|
||
|
|
foregroundURL, _, err := s.persistLayerSeparationImage(ctx, projectID, foregroundID, "foreground", candidate.Image)
|
||
|
|
if err != nil {
|
||
|
|
return layerSeparationForeground{}, false
|
||
|
|
}
|
||
|
|
return layerSeparationForeground{
|
||
|
|
ID: foregroundID,
|
||
|
|
URL: foregroundURL,
|
||
|
|
BBox: append([]int64(nil), candidate.BBox...),
|
||
|
|
Width: candidate.Width,
|
||
|
|
Height: candidate.Height,
|
||
|
|
Mask: candidate.Mask,
|
||
|
|
Rect: candidate.Rect,
|
||
|
|
}, true
|
||
|
|
}
|
||
|
|
|
||
|
|
func newLayerSeparationForegroundCandidate(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle, bounds image.Rectangle) layerSeparationForegroundCandidate {
|
||
|
|
return layerSeparationForegroundCandidate{
|
||
|
|
BBox: []int64{
|
||
|
|
int64(rect.Min.X - bounds.Min.X),
|
||
|
|
int64(rect.Min.Y - bounds.Min.Y),
|
||
|
|
int64(rect.Max.X - bounds.Min.X),
|
||
|
|
int64(rect.Max.Y - bounds.Min.Y),
|
||
|
|
},
|
||
|
|
Width: int64(rect.Dx()),
|
||
|
|
Height: int64(rect.Dy()),
|
||
|
|
Mask: mask,
|
||
|
|
Rect: rect,
|
||
|
|
Image: foregroundImageFromMask(foreground, mask, rect),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func eraseTextLayersFromForeground(foreground *image.NRGBA, extraction design.TextExtraction) {
|
||
|
|
if foreground == nil || len(extraction.Layers) == 0 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
bounds := foreground.Bounds()
|
||
|
|
if bounds.Dx()*bounds.Dy() < 4096 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, layer := range extraction.Layers {
|
||
|
|
rect := normalizedTextMaskRect(bounds, layer)
|
||
|
|
if rect.Empty() {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
padding := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.14)), 2, 18)
|
||
|
|
rect = image.Rect(rect.Min.X-padding, rect.Min.Y-padding, rect.Max.X+padding, rect.Max.Y+padding).Intersect(bounds)
|
||
|
|
for y := rect.Min.Y; y < rect.Max.Y; y++ {
|
||
|
|
for x := rect.Min.X; x < rect.Max.X; x++ {
|
||
|
|
offset := foreground.PixOffset(x, y)
|
||
|
|
foreground.Pix[offset+3] = 0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationElementCandidates(foreground *image.NRGBA, bounds image.Rectangle) []layerSeparationForegroundCandidate {
|
||
|
|
if foreground == nil || bounds.Empty() {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
width := bounds.Dx()
|
||
|
|
height := bounds.Dy()
|
||
|
|
visited := make([]bool, width*height)
|
||
|
|
components := make([]layerSeparationForegroundCandidate, 0, 4)
|
||
|
|
indexOf := func(x int, y int) int {
|
||
|
|
return (y-bounds.Min.Y)*width + (x - bounds.Min.X)
|
||
|
|
}
|
||
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||
|
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||
|
|
index := indexOf(x, y)
|
||
|
|
if visited[index] || foreground.NRGBAAt(x, y).A <= 12 {
|
||
|
|
visited[index] = true
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
mask, rect, count := layerSeparationComponentMask(foreground, bounds, visited, x, y, indexOf)
|
||
|
|
if !keepLayerSeparationElement(rect, count, bounds) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
components = append(components, newLayerSeparationForegroundCandidate(foreground, mask, rect, bounds))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
sort.Slice(components, func(i int, j int) bool {
|
||
|
|
leftArea := components[i].Rect.Dx() * components[i].Rect.Dy()
|
||
|
|
rightArea := components[j].Rect.Dx() * components[j].Rect.Dy()
|
||
|
|
return leftArea > rightArea
|
||
|
|
})
|
||
|
|
if len(components) > 12 {
|
||
|
|
components = components[:12]
|
||
|
|
}
|
||
|
|
return components
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationComponentMask(foreground *image.NRGBA, bounds image.Rectangle, visited []bool, startX int, startY int, indexOf func(int, int) int) (*image.Alpha, image.Rectangle, int) {
|
||
|
|
mask := image.NewAlpha(bounds)
|
||
|
|
stack := []image.Point{{X: startX, Y: startY}}
|
||
|
|
minX, minY := startX, startY
|
||
|
|
maxX, maxY := startX+1, startY+1
|
||
|
|
count := 0
|
||
|
|
for len(stack) > 0 {
|
||
|
|
point := stack[len(stack)-1]
|
||
|
|
stack = stack[:len(stack)-1]
|
||
|
|
if point.X < bounds.Min.X || point.X >= bounds.Max.X || point.Y < bounds.Min.Y || point.Y >= bounds.Max.Y {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
index := indexOf(point.X, point.Y)
|
||
|
|
if visited[index] {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
visited[index] = true
|
||
|
|
if foreground.NRGBAAt(point.X, point.Y).A <= 12 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
mask.SetAlpha(point.X, point.Y, color.Alpha{A: 255})
|
||
|
|
count++
|
||
|
|
if point.X < minX {
|
||
|
|
minX = point.X
|
||
|
|
}
|
||
|
|
if point.Y < minY {
|
||
|
|
minY = point.Y
|
||
|
|
}
|
||
|
|
if point.X+1 > maxX {
|
||
|
|
maxX = point.X + 1
|
||
|
|
}
|
||
|
|
if point.Y+1 > maxY {
|
||
|
|
maxY = point.Y + 1
|
||
|
|
}
|
||
|
|
stack = append(stack,
|
||
|
|
image.Point{X: point.X + 1, Y: point.Y},
|
||
|
|
image.Point{X: point.X - 1, Y: point.Y},
|
||
|
|
image.Point{X: point.X, Y: point.Y + 1},
|
||
|
|
image.Point{X: point.X, Y: point.Y - 1},
|
||
|
|
)
|
||
|
|
}
|
||
|
|
return mask, image.Rect(minX, minY, maxX, maxY), count
|
||
|
|
}
|
||
|
|
|
||
|
|
func keepLayerSeparationElement(rect image.Rectangle, count int, bounds image.Rectangle) bool {
|
||
|
|
if rect.Empty() || count <= 0 {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
totalArea := bounds.Dx() * bounds.Dy()
|
||
|
|
if totalArea <= 0 {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
if totalArea < 4096 {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
bboxArea := rect.Dx() * rect.Dy()
|
||
|
|
minArea := maxInt(96, totalArea/6000)
|
||
|
|
if count < minArea || bboxArea < minArea {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
if float64(bboxArea)/float64(totalArea) > 0.18 {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
if float64(rect.Dx())/float64(bounds.Dx()) > 0.62 || float64(rect.Dy())/float64(bounds.Dy()) > 0.62 {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
|
||
|
|
func foregroundImageFromMask(foreground *image.NRGBA, mask *image.Alpha, rect image.Rectangle) image.Image {
|
||
|
|
cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy()))
|
||
|
|
for y := rect.Min.Y; y < rect.Max.Y; y++ {
|
||
|
|
for x := rect.Min.X; x < rect.Max.X; x++ {
|
||
|
|
if mask.AlphaAt(x, y).A == 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
cropped.SetNRGBA(x-rect.Min.X, y-rect.Min.Y, foreground.NRGBAAt(x, y))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return cropped
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *DesignService) persistLayerSeparationImage(ctx context.Context, projectID string, nodeID string, suffix string, img image.Image) (string, int64, error) {
|
||
|
|
var output bytes.Buffer
|
||
|
|
if err := png.Encode(&output, img); err != nil {
|
||
|
|
return "", 0, err
|
||
|
|
}
|
||
|
|
if s.assets == nil {
|
||
|
|
data := output.Bytes()
|
||
|
|
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(data), int64(len(data)), nil
|
||
|
|
}
|
||
|
|
data, contentType, err := encodeRasterAssetAsWebP(output.Bytes(), "image/png")
|
||
|
|
if err != nil {
|
||
|
|
return "", 0, err
|
||
|
|
}
|
||
|
|
object, err := s.assets.PutObject(ctx, design.AssetObjectRequest{
|
||
|
|
FileName: fmt.Sprintf("%s-%s-%s.png", projectID, nodeID, suffix),
|
||
|
|
ContentType: contentType,
|
||
|
|
Data: data,
|
||
|
|
})
|
||
|
|
if err != nil {
|
||
|
|
return "", 0, err
|
||
|
|
}
|
||
|
|
if strings.TrimSpace(object.PublicURL) == "" {
|
||
|
|
return "", 0, fmt.Errorf("layer separation upload returned empty public URL")
|
||
|
|
}
|
||
|
|
return object.PublicURL, int64(len(data)), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func newLayerSeparationBackgroundNode(frame design.Node, result layerSeparationResult) design.Node {
|
||
|
|
return design.Node{
|
||
|
|
ID: result.BackgroundID,
|
||
|
|
Type: design.NodeTypeImage,
|
||
|
|
Title: "Image",
|
||
|
|
X: frame.X,
|
||
|
|
Y: frame.Y,
|
||
|
|
Width: frame.Width,
|
||
|
|
Height: frame.Height,
|
||
|
|
Content: result.BackgroundURL,
|
||
|
|
Tone: "visual",
|
||
|
|
Status: "success",
|
||
|
|
LayerRole: "clean-image",
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func newForegroundImageLayers(target design.Node, foregrounds []layerSeparationForeground, canvasWidth int64, canvasHeight int64) []design.Node {
|
||
|
|
if len(foregrounds) == 0 || canvasWidth <= 0 || canvasHeight <= 0 {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
nodes := make([]design.Node, 0, len(foregrounds))
|
||
|
|
for index, foreground := range foregrounds {
|
||
|
|
if len(foreground.BBox) < 4 || strings.TrimSpace(foreground.URL) == "" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
x1 := clampFloat(float64(foreground.BBox[0])/float64(canvasWidth), 0, 1)
|
||
|
|
y1 := clampFloat(float64(foreground.BBox[1])/float64(canvasHeight), 0, 1)
|
||
|
|
x2 := clampFloat(float64(foreground.BBox[2])/float64(canvasWidth), x1, 1)
|
||
|
|
y2 := clampFloat(float64(foreground.BBox[3])/float64(canvasHeight), y1, 1)
|
||
|
|
width := target.Width * (x2 - x1)
|
||
|
|
height := target.Height * (y2 - y1)
|
||
|
|
if width <= 1 || height <= 1 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
nodes = append(nodes, design.Node{
|
||
|
|
ID: foreground.ID,
|
||
|
|
Type: design.NodeTypeImage,
|
||
|
|
Title: fmt.Sprintf("Foreground Layer %d", index+1),
|
||
|
|
X: target.X + target.Width*x1,
|
||
|
|
Y: target.Y + target.Height*y1,
|
||
|
|
Width: width,
|
||
|
|
Height: height,
|
||
|
|
Content: foreground.URL,
|
||
|
|
Tone: "separated-element",
|
||
|
|
Status: "success",
|
||
|
|
ParentID: target.ID,
|
||
|
|
LayerRole: "foreground-image",
|
||
|
|
})
|
||
|
|
}
|
||
|
|
return nodes
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationArtifactsMessage(threadID string, artifacts []design.GeneratorTaskArtifact, createdAt time.Time) design.Message {
|
||
|
|
content, _ := json.Marshal(toLayerSeparationArtifactWire(artifacts))
|
||
|
|
return design.Message{
|
||
|
|
ID: newID(),
|
||
|
|
Role: "tool",
|
||
|
|
Type: "tool_use",
|
||
|
|
Title: layerSeparationMessageName,
|
||
|
|
Content: string(content),
|
||
|
|
ThreadID: threadID,
|
||
|
|
ActionID: threadID,
|
||
|
|
StepID: newID(),
|
||
|
|
ToolCallID: "call_" + strings.ReplaceAll(newID(), "-", ""),
|
||
|
|
Name: layerSeparationMessageName,
|
||
|
|
ToolHint: layerSeparationMessageName,
|
||
|
|
Status: "success",
|
||
|
|
CreatedAt: createdAt,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func toLayerSeparationArtifactWire(artifacts []design.GeneratorTaskArtifact) []map[string]any {
|
||
|
|
items := make([]map[string]any, 0, len(artifacts))
|
||
|
|
for _, artifact := range artifacts {
|
||
|
|
items = append(items, map[string]any{
|
||
|
|
"type": artifact.Type,
|
||
|
|
"content": artifact.Content,
|
||
|
|
"metadata": map[string]any{
|
||
|
|
"artifact_id": artifact.Metadata.ArtifactID,
|
||
|
|
"bbox": artifact.Metadata.BBox,
|
||
|
|
"format": artifact.Metadata.Format,
|
||
|
|
"generator_name": artifact.Metadata.GeneratorName,
|
||
|
|
"height": artifact.Metadata.Height,
|
||
|
|
"label": artifact.Metadata.Label,
|
||
|
|
"node_id": artifact.Metadata.NodeID,
|
||
|
|
"node_name": artifact.Metadata.NodeName,
|
||
|
|
"size_bytes": artifact.Metadata.SizeBytes,
|
||
|
|
"width": artifact.Metadata.Width,
|
||
|
|
},
|
||
|
|
})
|
||
|
|
}
|
||
|
|
return items
|
||
|
|
}
|
||
|
|
|
||
|
|
func removeForegroundMaskFromImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) {
|
||
|
|
if mask == nil || rect.Empty() {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
bounds := img.Bounds()
|
||
|
|
rect = rect.Intersect(bounds)
|
||
|
|
if rect.Empty() {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.025)), 2, 10)
|
||
|
|
mask, count := dilateTextMask(mask, rect, radius)
|
||
|
|
if count == 0 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
source := image.NewRGBA(bounds)
|
||
|
|
draw.Draw(source, bounds, img, bounds.Min, draw.Src)
|
||
|
|
for y := rect.Min.Y; y < rect.Max.Y; y++ {
|
||
|
|
for x := rect.Min.X; x < rect.Max.X; x++ {
|
||
|
|
if mask.AlphaAt(x, y).A == 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
img.SetRGBA(x, y, estimatedRectBackgroundColor(source, rect, x, y))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
softenMaskedPixels(img, mask, rect)
|
||
|
|
}
|
||
|
|
|
||
|
|
func plainLayerSeparationBackgroundImage(img *image.RGBA, mask *image.Alpha, rect image.Rectangle) (*image.RGBA, bool) {
|
||
|
|
if img == nil || mask == nil || rect.Empty() {
|
||
|
|
return nil, false
|
||
|
|
}
|
||
|
|
bounds := img.Bounds()
|
||
|
|
if bounds.Empty() {
|
||
|
|
return nil, false
|
||
|
|
}
|
||
|
|
radius := clampInt(int(math.Round(math.Min(float64(rect.Dx()), float64(rect.Dy()))*0.02)), 3, 18)
|
||
|
|
expandedMask, _ := dilateTextMask(mask, bounds, radius)
|
||
|
|
stats, ok := unmaskedColorStats(img, expandedMask, bounds)
|
||
|
|
if !ok || stats.StdDev > 24 {
|
||
|
|
return nil, false
|
||
|
|
}
|
||
|
|
bg := image.NewRGBA(bounds)
|
||
|
|
fill := colorRGBA(int(math.Round(stats.R)), int(math.Round(stats.G)), int(math.Round(stats.B)), int(math.Round(stats.A)))
|
||
|
|
draw.Draw(bg, bounds, &image.Uniform{C: fill}, image.Point{}, draw.Src)
|
||
|
|
return bg, true
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationColorStats struct {
|
||
|
|
Count int64
|
||
|
|
R float64
|
||
|
|
G float64
|
||
|
|
B float64
|
||
|
|
A float64
|
||
|
|
StdDev float64
|
||
|
|
}
|
||
|
|
|
||
|
|
func unmaskedColorStats(img *image.RGBA, mask *image.Alpha, bounds image.Rectangle) (layerSeparationColorStats, bool) {
|
||
|
|
stride := clampInt(int(math.Ceil(float64(maxInt(bounds.Dx(), bounds.Dy()))/640)), 1, 12)
|
||
|
|
var count int64
|
||
|
|
var r, g, b, a float64
|
||
|
|
var rr, gg, bb float64
|
||
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y += stride {
|
||
|
|
for x := bounds.Min.X; x < bounds.Max.X; x += stride {
|
||
|
|
if mask.AlphaAt(x, y).A != 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
c := img.RGBAAt(x, y)
|
||
|
|
cfR := float64(c.R)
|
||
|
|
cfG := float64(c.G)
|
||
|
|
cfB := float64(c.B)
|
||
|
|
cfA := float64(c.A)
|
||
|
|
r += cfR
|
||
|
|
g += cfG
|
||
|
|
b += cfB
|
||
|
|
a += cfA
|
||
|
|
rr += cfR * cfR
|
||
|
|
gg += cfG * cfG
|
||
|
|
bb += cfB * cfB
|
||
|
|
count++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if count < 32 {
|
||
|
|
return layerSeparationColorStats{}, false
|
||
|
|
}
|
||
|
|
divisor := float64(count)
|
||
|
|
meanR := r / divisor
|
||
|
|
meanG := g / divisor
|
||
|
|
meanB := b / divisor
|
||
|
|
varR := math.Max(0, rr/divisor-meanR*meanR)
|
||
|
|
varG := math.Max(0, gg/divisor-meanG*meanG)
|
||
|
|
varB := math.Max(0, bb/divisor-meanB*meanB)
|
||
|
|
return layerSeparationColorStats{
|
||
|
|
Count: count,
|
||
|
|
R: meanR,
|
||
|
|
G: meanG,
|
||
|
|
B: meanB,
|
||
|
|
A: a / divisor,
|
||
|
|
StdDev: math.Sqrt((varR + varG + varB) / 3),
|
||
|
|
}, true
|
||
|
|
}
|
||
|
|
|
||
|
|
func alphaContentBounds(img image.Image, bounds image.Rectangle, threshold uint8) (image.Rectangle, bool) {
|
||
|
|
minX, minY := bounds.Max.X, bounds.Max.Y
|
||
|
|
maxX, maxY := bounds.Min.X, bounds.Min.Y
|
||
|
|
found := false
|
||
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||
|
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||
|
|
if alphaAt(img, x, y) <= threshold {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if x < minX {
|
||
|
|
minX = x
|
||
|
|
}
|
||
|
|
if y < minY {
|
||
|
|
minY = y
|
||
|
|
}
|
||
|
|
if x+1 > maxX {
|
||
|
|
maxX = x + 1
|
||
|
|
}
|
||
|
|
if y+1 > maxY {
|
||
|
|
maxY = y + 1
|
||
|
|
}
|
||
|
|
found = true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if !found {
|
||
|
|
return image.Rectangle{}, false
|
||
|
|
}
|
||
|
|
return image.Rect(minX, minY, maxX, maxY), true
|
||
|
|
}
|
||
|
|
|
||
|
|
func alphaMask(img image.Image, bounds image.Rectangle, threshold uint8) (*image.Alpha, int) {
|
||
|
|
mask := image.NewAlpha(bounds)
|
||
|
|
count := 0
|
||
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||
|
|
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||
|
|
if alphaAt(img, x, y) <= threshold {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
mask.SetAlpha(x, y, color.Alpha{A: 255})
|
||
|
|
count++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return mask, count
|
||
|
|
}
|
||
|
|
|
||
|
|
func alphaAt(img image.Image, x int, y int) uint8 {
|
||
|
|
_, _, _, a := img.At(x, y).RGBA()
|
||
|
|
return uint8(a >> 8)
|
||
|
|
}
|
||
|
|
|
||
|
|
func cropImage(img image.Image, rect image.Rectangle) image.Image {
|
||
|
|
cropped := image.NewNRGBA(image.Rect(0, 0, rect.Dx(), rect.Dy()))
|
||
|
|
draw.Draw(cropped, cropped.Bounds(), img, rect.Min, draw.Src)
|
||
|
|
return cropped
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationTextDocument struct {
|
||
|
|
Layers []layerSeparationTextLayer `json:"layers"`
|
||
|
|
CanvasSize layerSeparationCanvasSize `json:"canvas_size"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationCanvasSize struct {
|
||
|
|
Width int64 `json:"width"`
|
||
|
|
Height int64 `json:"height"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationTextLayer struct {
|
||
|
|
TextInfo layerSeparationTextInfo `json:"text_info"`
|
||
|
|
X float64 `json:"x"`
|
||
|
|
Y float64 `json:"y"`
|
||
|
|
W float64 `json:"w"`
|
||
|
|
H float64 `json:"h"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type layerSeparationTextInfo struct {
|
||
|
|
Text string `json:"text"`
|
||
|
|
FontSizePx int64 `json:"font_size_px"`
|
||
|
|
FontFamily string `json:"font_family"`
|
||
|
|
FontWeight any `json:"font_weight"`
|
||
|
|
FontStyle string `json:"font_style"`
|
||
|
|
ColorCSS string `json:"color_css"`
|
||
|
|
TextAlign string `json:"text_align"`
|
||
|
|
Leading float64 `json:"leading"`
|
||
|
|
Tracking float64 `json:"tracking"`
|
||
|
|
Rotation float64 `json:"rotation"`
|
||
|
|
LeadingTrim string `json:"leading_trim"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationTextRenderData(extraction design.TextExtraction, canvasWidth int64, canvasHeight int64) (string, error) {
|
||
|
|
doc := layerSeparationTextDocument{
|
||
|
|
Layers: make([]layerSeparationTextLayer, 0, len(extraction.Layers)),
|
||
|
|
CanvasSize: layerSeparationCanvasSize{
|
||
|
|
Width: canvasWidth,
|
||
|
|
Height: canvasHeight,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
for _, layer := range extraction.Layers {
|
||
|
|
text := strings.TrimSpace(layer.Text)
|
||
|
|
if text == "" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
x := clamp01(layer.X)
|
||
|
|
y := clamp01(layer.Y)
|
||
|
|
width := clamp01(layer.Width)
|
||
|
|
height := clamp01(layer.Height)
|
||
|
|
if x+width > 1 {
|
||
|
|
width = 1 - x
|
||
|
|
}
|
||
|
|
if y+height > 1 {
|
||
|
|
height = 1 - y
|
||
|
|
}
|
||
|
|
if width <= 0 || height <= 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
fontSize := layer.FontSize
|
||
|
|
if fontSize <= 0 {
|
||
|
|
fontSize = math.Max(10, float64(canvasHeight)*height*0.72)
|
||
|
|
}
|
||
|
|
lineHeight := layer.LineHeight
|
||
|
|
if lineHeight <= 0 {
|
||
|
|
lineHeight = 1.2
|
||
|
|
}
|
||
|
|
fontFamily := strings.TrimSpace(layer.FontFamily)
|
||
|
|
if fontFamily == "" {
|
||
|
|
fontFamily = "Inter"
|
||
|
|
}
|
||
|
|
doc.Layers = append(doc.Layers, layerSeparationTextLayer{
|
||
|
|
TextInfo: layerSeparationTextInfo{
|
||
|
|
Text: text,
|
||
|
|
FontSizePx: int64(math.Round(fontSize)),
|
||
|
|
FontFamily: fontFamily,
|
||
|
|
FontWeight: layerSeparationFontWeight(layer.FontWeight),
|
||
|
|
FontStyle: layerSeparationFontStyle(layer.FontWeight),
|
||
|
|
ColorCSS: layerSeparationColorCSS(layer.Color),
|
||
|
|
TextAlign: layerSeparationTextAlign(layer.TextAlign),
|
||
|
|
Leading: lineHeight,
|
||
|
|
Tracking: layer.LetterSpacing,
|
||
|
|
Rotation: layer.Rotation,
|
||
|
|
LeadingTrim: "CAP_HEIGHT",
|
||
|
|
},
|
||
|
|
X: x * float64(canvasWidth),
|
||
|
|
Y: y * float64(canvasHeight),
|
||
|
|
W: width * float64(canvasWidth),
|
||
|
|
H: height * float64(canvasHeight),
|
||
|
|
})
|
||
|
|
}
|
||
|
|
data, err := json.Marshal(doc)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
return string(data), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationFontWeight(value string) any {
|
||
|
|
value = strings.TrimSpace(value)
|
||
|
|
if value == "" {
|
||
|
|
return 400
|
||
|
|
}
|
||
|
|
if weight, err := strconv.Atoi(value); err == nil {
|
||
|
|
return weight
|
||
|
|
}
|
||
|
|
return value
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationFontStyle(value string) string {
|
||
|
|
if strings.Contains(strings.ToLower(strings.TrimSpace(value)), "italic") {
|
||
|
|
return "italic"
|
||
|
|
}
|
||
|
|
return "normal"
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationTextAlign(value string) string {
|
||
|
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
||
|
|
case "left":
|
||
|
|
return "start"
|
||
|
|
case "right":
|
||
|
|
return "end"
|
||
|
|
case "center":
|
||
|
|
return "center"
|
||
|
|
default:
|
||
|
|
return "start"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func layerSeparationColorCSS(value string) string {
|
||
|
|
parsed, ok := parseCanvasColor(value)
|
||
|
|
if !ok {
|
||
|
|
return "rgba(25, 25, 25, 1.0)"
|
||
|
|
}
|
||
|
|
alpha := float64(parsed.A) / 255
|
||
|
|
return fmt.Sprintf("rgba(%d, %d, %d, %.3g)", parsed.R, parsed.G, parsed.B, alpha)
|
||
|
|
}
|