root
44406b72db
Moteva-style AI design workbench replica. Home prompt creates a project; the project page provides an infinite canvas with node drag, zoom/pan, a design chat panel, history replay, image asset upload, and project save/regenerate. - Backend: go-zero, DDD layering, sqlc/pgx, optional PostgreSQL; memory or Redis cache; asynq job queue; MinIO/S3/R2/OSS object storage; sky-valley/pi agent runtime adapter. - Frontend: Next.js App Router + Vite artifact build, TypeScript, i18n, shadcn/ui components, auth (OTP/Turnstile/Google/WeChat). - Deploy: Docker Compose and k3s manifests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
419 lines
12 KiB
Go
419 lines
12 KiB
Go
package textextraction
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"img_infinite_canvas/internal/domain/design"
|
|
)
|
|
|
|
type TextExtractionClientOptions struct {
|
|
BaseURL string
|
|
APIKey string
|
|
Model string
|
|
TimeoutSeconds int
|
|
}
|
|
|
|
type TextExtractionClient struct {
|
|
baseURL string
|
|
apiKey string
|
|
model string
|
|
client *http.Client
|
|
}
|
|
|
|
func NewTextExtractionClient(opts TextExtractionClientOptions) *TextExtractionClient {
|
|
if strings.TrimSpace(opts.BaseURL) == "" || strings.TrimSpace(opts.APIKey) == "" {
|
|
return nil
|
|
}
|
|
timeout := time.Duration(opts.TimeoutSeconds) * time.Second
|
|
if timeout <= 0 {
|
|
timeout = 45 * time.Second
|
|
}
|
|
model := strings.TrimSpace(opts.Model)
|
|
if model == "" {
|
|
model = "gpt-5.4-mini"
|
|
}
|
|
|
|
return &TextExtractionClient{
|
|
baseURL: strings.TrimRight(opts.BaseURL, "/"),
|
|
apiKey: opts.APIKey,
|
|
model: model,
|
|
client: &http.Client{Timeout: timeout},
|
|
}
|
|
}
|
|
|
|
func (c *TextExtractionClient) ExtractText(ctx context.Context, req design.TextExtractionRequest) (design.TextExtraction, error) {
|
|
imageURL := strings.TrimSpace(req.ImageURL)
|
|
if imageURL == "" {
|
|
return design.TextExtraction{}, fmt.Errorf("text extraction image url is required")
|
|
}
|
|
imagePayloadURL := c.inlineImageURL(ctx, imageURL)
|
|
|
|
body, err := json.Marshal(map[string]any{
|
|
"model": c.model,
|
|
"messages": []map[string]any{
|
|
{
|
|
"role": "system",
|
|
"content": strings.Join([]string{
|
|
"You are an OCR layout extraction engine for a design canvas.",
|
|
"Return strict JSON only.",
|
|
"Detect every visible text block in the image, including Chinese and English.",
|
|
"Use normalized coordinates from 0 to 1 relative to the original image: x, y, width, height.",
|
|
"Coordinates must cover the visible text ink and text effects only: glyph fill, anti-aliased edges, stroke, outline, shadow, or glow that should be removed from the base image.",
|
|
"Keep each coordinate box tight, with at most 2-6 pixels of padding around those text pixels.",
|
|
"Do not include large empty background, product area, logo container whitespace, or surrounding layout region in a text box.",
|
|
"Do not duplicate the same text layer. Merge words that belong to one visual line or paragraph into one item.",
|
|
"Return no placeholder text. If no text is visible, return {\"items\":[]}.",
|
|
"Estimate CSS-like text rendering fields: fontSize in image pixels, fontFamily, fontWeight, color as hex, textAlign, lineHeight as a unitless multiplier, letterSpacing in pixels, strokeColor and strokeWidth if the text has an outline.",
|
|
"The app will keep the original image as the clean base, remove only detected text pixels from that base, and recreate the words with editable canvas text components. Accurate tight boxes and faithful text style are more important than visual restyling.",
|
|
}, " "),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": []map[string]any{
|
|
{
|
|
"type": "text",
|
|
"text": "Extract OCR text layers for editable overlay reconstruction. Return tight text-effect bounding boxes, not full design regions. Preserve the original words exactly and output each visible text layer once. JSON shape: {\"items\":[{\"text\":\"...\",\"x\":0.1,\"y\":0.2,\"width\":0.3,\"height\":0.06,\"fontSize\":42,\"fontFamily\":\"Inter\",\"fontWeight\":\"700\",\"color\":\"#111111\",\"textAlign\":\"center\",\"lineHeight\":1.05,\"letterSpacing\":0,\"strokeColor\":\"#ffffff\",\"strokeWidth\":0,\"rotation\":0,\"confidence\":0.9}]}",
|
|
},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": map[string]any{
|
|
"url": imagePayloadURL,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return design.TextExtraction{}, err
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/chat/completions", bytes.NewReader(body))
|
|
if err != nil {
|
|
return design.TextExtraction{}, err
|
|
}
|
|
httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := c.client.Do(httpReq)
|
|
if err != nil {
|
|
return design.TextExtraction{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var payload struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
}
|
|
respBody, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if err != nil {
|
|
return design.TextExtraction{}, err
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return design.TextExtraction{}, fmt.Errorf("text extraction failed: %s: %s", resp.Status, trimErrorBody(respBody))
|
|
}
|
|
if err := json.Unmarshal(respBody, &payload); err != nil {
|
|
return design.TextExtraction{}, err
|
|
}
|
|
if len(payload.Choices) == 0 {
|
|
return design.TextExtraction{}, fmt.Errorf("text extraction returned no choices")
|
|
}
|
|
|
|
content := decodeTextExtractionContent(payload.Choices[0].Message.Content)
|
|
return parseTextExtraction(content)
|
|
}
|
|
|
|
func (c *TextExtractionClient) inlineImageURL(ctx context.Context, imageURL string) string {
|
|
if strings.HasPrefix(imageURL, "data:image/") {
|
|
return imageURL
|
|
}
|
|
if !strings.HasPrefix(imageURL, "http://") && !strings.HasPrefix(imageURL, "https://") {
|
|
return imageURL
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
|
if err != nil {
|
|
return imageURL
|
|
}
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
return imageURL
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return imageURL
|
|
}
|
|
|
|
data, err := io.ReadAll(io.LimitReader(resp.Body, 20<<20))
|
|
if err != nil || len(data) == 0 {
|
|
return imageURL
|
|
}
|
|
contentType := strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0])
|
|
if contentType == "" || !strings.HasPrefix(contentType, "image/") {
|
|
contentType = http.DetectContentType(data)
|
|
}
|
|
if !strings.HasPrefix(contentType, "image/") {
|
|
return imageURL
|
|
}
|
|
return "data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(data)
|
|
}
|
|
|
|
func trimErrorBody(body []byte) string {
|
|
value := strings.TrimSpace(string(body))
|
|
if value == "" {
|
|
return "empty error body"
|
|
}
|
|
if len(value) > 1200 {
|
|
return value[:1200]
|
|
}
|
|
return value
|
|
}
|
|
|
|
func decodeTextExtractionContent(raw json.RawMessage) string {
|
|
var text string
|
|
if err := json.Unmarshal(raw, &text); err == nil {
|
|
return text
|
|
}
|
|
|
|
var parts []struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
}
|
|
if err := json.Unmarshal(raw, &parts); err == nil {
|
|
var builder strings.Builder
|
|
for _, part := range parts {
|
|
if strings.TrimSpace(part.Text) == "" {
|
|
continue
|
|
}
|
|
if builder.Len() > 0 {
|
|
builder.WriteString("\n")
|
|
}
|
|
builder.WriteString(part.Text)
|
|
}
|
|
return builder.String()
|
|
}
|
|
return string(raw)
|
|
}
|
|
|
|
func parseTextExtraction(content string) (design.TextExtraction, error) {
|
|
content = strings.TrimSpace(content)
|
|
if content == "" {
|
|
return design.TextExtraction{}, fmt.Errorf("text extraction returned empty content")
|
|
}
|
|
if strings.HasPrefix(content, "```") {
|
|
content = strings.Trim(content, "`")
|
|
content = strings.TrimSpace(strings.TrimPrefix(content, "json"))
|
|
}
|
|
if start := strings.Index(content, "{"); start >= 0 {
|
|
if end := strings.LastIndex(content, "}"); end >= start {
|
|
content = content[start : end+1]
|
|
}
|
|
}
|
|
|
|
type rawLayer struct {
|
|
Text string `json:"text"`
|
|
X float64 `json:"x"`
|
|
Y float64 `json:"y"`
|
|
Width float64 `json:"width"`
|
|
Height float64 `json:"height"`
|
|
BBox []float64 `json:"bbox"`
|
|
Box []float64 `json:"box"`
|
|
FontSize float64 `json:"fontSize"`
|
|
FontFamily string `json:"fontFamily"`
|
|
FontWeight string `json:"fontWeight"`
|
|
Color string `json:"color"`
|
|
TextAlign string `json:"textAlign"`
|
|
LineHeight float64 `json:"lineHeight"`
|
|
LetterSpacing float64 `json:"letterSpacing"`
|
|
StrokeColor string `json:"strokeColor"`
|
|
StrokeWidth float64 `json:"strokeWidth"`
|
|
Opacity float64 `json:"opacity"`
|
|
Rotation float64 `json:"rotation"`
|
|
Confidence float64 `json:"confidence"`
|
|
}
|
|
var doc struct {
|
|
Items []rawLayer `json:"items"`
|
|
Layers []rawLayer `json:"layers"`
|
|
TextLayers []rawLayer `json:"textLayers"`
|
|
TextBlocks []rawLayer `json:"textBlocks"`
|
|
}
|
|
if err := json.Unmarshal([]byte(content), &doc); err != nil {
|
|
return design.TextExtraction{}, err
|
|
}
|
|
|
|
rawLayers := doc.Items
|
|
if len(rawLayers) == 0 {
|
|
rawLayers = doc.Layers
|
|
}
|
|
if len(rawLayers) == 0 {
|
|
rawLayers = doc.TextLayers
|
|
}
|
|
if len(rawLayers) == 0 {
|
|
rawLayers = doc.TextBlocks
|
|
}
|
|
|
|
layers := make([]design.ExtractedTextLayer, 0, len(rawLayers))
|
|
for _, item := range rawLayers {
|
|
text := strings.TrimSpace(item.Text)
|
|
if text == "" {
|
|
continue
|
|
}
|
|
x, y, width, height := item.X, item.Y, item.Width, item.Height
|
|
if len(item.BBox) >= 4 {
|
|
x, y, width, height = item.BBox[0], item.BBox[1], item.BBox[2], item.BBox[3]
|
|
}
|
|
if len(item.Box) >= 4 {
|
|
x, y, width, height = item.Box[0], item.Box[1], item.Box[2], item.Box[3]
|
|
}
|
|
width = clampUnit(width)
|
|
height = clampUnit(height)
|
|
if width <= 0 || height <= 0 {
|
|
continue
|
|
}
|
|
layer := design.ExtractedTextLayer{
|
|
Text: text,
|
|
OriginalText: text,
|
|
X: clampUnit(x),
|
|
Y: clampUnit(y),
|
|
Width: width,
|
|
Height: height,
|
|
FontSize: item.FontSize,
|
|
FontFamily: strings.TrimSpace(item.FontFamily),
|
|
FontWeight: strings.TrimSpace(item.FontWeight),
|
|
Color: normalizeHexColor(item.Color),
|
|
TextAlign: normalizeTextAlign(item.TextAlign),
|
|
LineHeight: normalizeLineHeight(item.LineHeight, item.FontSize),
|
|
LetterSpacing: normalizeLetterSpacing(item.LetterSpacing),
|
|
StrokeColor: normalizeHexColor(item.StrokeColor),
|
|
StrokeWidth: normalizeStrokeWidth(item.StrokeWidth),
|
|
Opacity: normalizeOpacity(item.Opacity),
|
|
Rotation: item.Rotation,
|
|
Confidence: item.Confidence,
|
|
}
|
|
if isDuplicateTextLayer(layers, layer) {
|
|
continue
|
|
}
|
|
layers = append(layers, layer)
|
|
}
|
|
return design.TextExtraction{Layers: layers}, nil
|
|
}
|
|
|
|
func clampUnit(value float64) float64 {
|
|
if value < 0 {
|
|
return 0
|
|
}
|
|
if value > 1 {
|
|
return 1
|
|
}
|
|
return value
|
|
}
|
|
|
|
func normalizeHexColor(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return ""
|
|
}
|
|
if strings.HasPrefix(value, "#") && (len(value) == 4 || len(value) == 7 || len(value) == 9) {
|
|
return value
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func normalizeTextAlign(value string) string {
|
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
|
case "left", "center", "right":
|
|
return strings.ToLower(strings.TrimSpace(value))
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func normalizeLineHeight(value float64, fontSize float64) float64 {
|
|
if value <= 0 {
|
|
return 0
|
|
}
|
|
if value > 4 && fontSize > 0 {
|
|
value = value / fontSize
|
|
}
|
|
if value < 0.7 || value > 2.5 {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|
|
|
|
func normalizeLetterSpacing(value float64) float64 {
|
|
if value < 0 || value > 80 {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|
|
|
|
func normalizeStrokeWidth(value float64) float64 {
|
|
if value < 0 || value > 32 {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|
|
|
|
func normalizeOpacity(value float64) float64 {
|
|
if value <= 0 || value > 1 {
|
|
return 0
|
|
}
|
|
return value
|
|
}
|
|
|
|
func isDuplicateTextLayer(existing []design.ExtractedTextLayer, next design.ExtractedTextLayer) bool {
|
|
for _, layer := range existing {
|
|
if !strings.EqualFold(strings.TrimSpace(layer.Text), strings.TrimSpace(next.Text)) {
|
|
continue
|
|
}
|
|
if textLayerOverlap(layer, next) >= 0.86 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func textLayerOverlap(a design.ExtractedTextLayer, b design.ExtractedTextLayer) float64 {
|
|
ax2 := a.X + a.Width
|
|
ay2 := a.Y + a.Height
|
|
bx2 := b.X + b.Width
|
|
by2 := b.Y + b.Height
|
|
interWidth := minFloat64(ax2, bx2) - maxFloat64(a.X, b.X)
|
|
interHeight := minFloat64(ay2, by2) - maxFloat64(a.Y, b.Y)
|
|
if interWidth <= 0 || interHeight <= 0 {
|
|
return 0
|
|
}
|
|
intersection := interWidth * interHeight
|
|
smallerArea := minFloat64(a.Width*a.Height, b.Width*b.Height)
|
|
if smallerArea <= 0 {
|
|
return 0
|
|
}
|
|
return intersection / smallerArea
|
|
}
|
|
|
|
func minFloat64(a float64, b float64) float64 {
|
|
if a < b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func maxFloat64(a float64, b float64) float64 {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|