diff --git a/frontend/src/domain/design.ts b/frontend/src/domain/design.ts index 1a28ea0..61698b7 100644 --- a/frontend/src/domain/design.ts +++ b/frontend/src/domain/design.ts @@ -268,6 +268,7 @@ export type DeleteProjectsResponse = { export type NodeActionRequest = { action: | "quick-edit" + | "visual-annotation" | "upscale" | "remove-background" | "eraser" @@ -285,6 +286,7 @@ export type NodeActionRequest = { imageSize?: string; mask?: string; selection?: string; + annotationImage?: string; options?: Record; }; diff --git a/frontend/src/i18n/locales/en-US.json b/frontend/src/i18n/locales/en-US.json index de113eb..a6118ac 100644 --- a/frontend/src/i18n/locales/en-US.json +++ b/frontend/src/i18n/locales/en-US.json @@ -504,6 +504,18 @@ "quickEdit": "Quick edit", "quickEditTab": "Quick edit Tab", "quickEditPlaceholder": "Describe the edit for this image", + "visualAnnotation": "Annotated edit", + "visualAnnotationTools": "Annotation tools", + "visualAnnotationBrush": "Brush mark", + "visualAnnotationCircle": "Circle mark", + "visualAnnotationRectangle": "Rectangle mark", + "visualAnnotationCross": "Cross mark", + "visualAnnotationList": "Mark list", + "visualAnnotationEmpty": "No marks yet", + "visualAnnotationInstructionPlaceholder": "Tell AI how to change this area", + "visualAnnotationInstructionLabel": "Edit instruction for mark {index}", + "visualAnnotationDelete": "Delete mark {index}", + "visualAnnotationPrompt": "Use the unmarked source image as the base and edit only the numbered marked regions. The marks, numbers, and annotated guide are location cues only and must not appear in the final image. Keep every unmarked area unchanged.\n{instructions}", "upscale": "Upscale", "upscalePrompt": "Upscale this image to {scale} level with the model. Preserve composition and subject while improving detail, edges, and texture clarity.", "removeBackground": "Remove background", @@ -619,6 +631,8 @@ "cropPromptPlaceholder": "Describe the cropped composition", "apply": "Apply", "cancel": "Cancel", + "undo": "Undo", + "delete": "Delete", "clipboardCopied": "Copied to system clipboard", "clipboardCopyFailed": "Could not write to system clipboard", "contextCopy": "Copy", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 17de494..018f344 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -504,6 +504,18 @@ "quickEdit": "快捷编辑", "quickEditTab": "快捷编辑Tab", "quickEditPlaceholder": "输入这张图的局部或整体修改要求", + "visualAnnotation": "标注改图", + "visualAnnotationTools": "标注工具", + "visualAnnotationBrush": "画笔标记", + "visualAnnotationCircle": "圆形标记", + "visualAnnotationRectangle": "矩形标记", + "visualAnnotationCross": "叉号标记", + "visualAnnotationList": "标记列表", + "visualAnnotationEmpty": "暂无标记", + "visualAnnotationInstructionPlaceholder": "告诉 AI 这里怎么改", + "visualAnnotationInstructionLabel": "标记 {index} 的修改说明", + "visualAnnotationDelete": "删除标记 {index}", + "visualAnnotationPrompt": "请以未标注原图为底图，严格按编号标记修改对应区域；标记线、编号和标注引导图只用于定位，最终图中不得保留任何标记。未标注区域保持不变。\n{instructions}", "upscale": "放大", "upscalePrompt": "请将这张图通过模型放大到 {scale} 级别，保持构图和主体不变，提升细节、边缘和纹理清晰度。", "removeBackground": "去背景", @@ -619,6 +631,8 @@ "cropPromptPlaceholder": "描述裁剪后的构图用途", "apply": "应用", "cancel": "取消", + "undo": "撤销", + "delete": "删除", "clipboardCopied": "已复制到系统剪贴板", "clipboardCopyFailed": "无法写入系统剪贴板", "contextCopy": "复制", diff --git a/frontend/src/ui/components/ImageNodeActions/index.tsx b/frontend/src/ui/components/ImageNodeActions/index.tsx index c7b9886..71d6964 100644 --- a/frontend/src/ui/components/ImageNodeActions/index.tsx +++ b/frontend/src/ui/components/ImageNodeActions/index.tsx @@ -57,6 +57,7 @@ export type { MultiAngleOptions } from "@/ui/components/MultiAnglePanel"; export type ImageToolbarItemId = | "quickEdit" + | "visualAnnotation" | "upscale" | "removeBackground" | "eraser" @@ -154,6 +155,7 @@ type BrushPath = { export const defaultToolbarVisibility: ImageToolbarVisibility = { quickEdit: true, + visualAnnotation: true, upscale: true, removeBackground: true, eraser: true, @@ -170,6 +172,7 @@ export const defaultToolbarVisibility: ImageToolbarVisibility = { }; const defaultToolbarOrder: ImageToolbarItemId[] = [ + "visualAnnotation", "upscale", "removeBackground", "eraser", @@ -360,6 +363,7 @@ export function ImageNodeToolbar({ visibility, onVisibilityChange, onQuickEdit, + onVisualAnnotation, onUpscale, onRemoveBackground, onEraser, @@ -381,6 +385,7 @@ export function ImageNodeToolbar({ visibility: ImageToolbarVisibility; onVisibilityChange: Dispatch>; onQuickEdit: () => void; + onVisualAnnotation: () => void; onUpscale: (scale: string) => void; onRemoveBackground: () => void; onEraser: () => void; @@ -416,6 +421,7 @@ export function ImageNodeToolbar({ primary?: boolean; }> = [ { id: "quickEdit", label: t("quickEditTab"), icon: Wand2, onClick: onQuickEdit, primary: true }, + { id: "visualAnnotation", label: t("visualAnnotation"), icon: PenLine, onClick: onVisualAnnotation }, { id: "removeBackground", label: t("removeBackground"), icon: Circle, onClick: onRemoveBackground }, { id: "eraser", label: t("eraserTool"), icon: EraserIcon, onClick: onEraser }, { id: "editElements", label: t("editElements"), icon: Layers, onClick: onEditElements }, diff --git a/frontend/src/ui/components/VisualAnnotationPanel/index.css b/frontend/src/ui/components/VisualAnnotationPanel/index.css new file mode 100644 index 0000000..d2aed68 --- /dev/null +++ b/frontend/src/ui/components/VisualAnnotationPanel/index.css @@ -0,0 +1,398 @@ +.visual-annotation-overlay { + position: absolute; + z-index: 38; + overflow: hidden; + border: 1px solid rgba(47, 128, 255, 0.82); + background: rgba(47, 128, 255, 0.015); + cursor: crosshair; + touch-action: none; +} + +.visual-annotation-overlay.is-brush { + cursor: cell; +} + +.visual-annotation-overlay svg { + width: 100%; + height: 100%; + display: block; + overflow: visible; + pointer-events: none; +} + +.visual-annotation-mark { + fill: none; + stroke: #2f80ff; + stroke-linecap: round; + stroke-linejoin: round; + vector-effect: non-scaling-stroke; + filter: drop-shadow(0 0 1px rgba(255, 255, 255, 0.96)); +} + +circle.visual-annotation-mark { + fill: #2f80ff; + stroke: #fff; + stroke-width: 1.5px; +} + +.visual-annotation-mark.selected { + stroke: #1673f9; + filter: drop-shadow(0 0 2px rgba(255, 255, 255, 1)); +} + +.visual-annotation-mark.draft { + opacity: 0.72; +} + +.visual-annotation-badge circle { + fill: #2f80ff; + stroke: #fff; + stroke-width: 2px; + vector-effect: non-scaling-stroke; +} + +.visual-annotation-badge text { + fill: #fff; + font-family: inherit; + font-size: 11px; + font-weight: 760; + letter-spacing: 0; + user-select: none; +} + +.visual-annotation-panel { + z-index: 42; + display: grid; + grid-template-rows: auto auto auto minmax(52px, 1fr) auto; + overflow: hidden; + border-color: rgba(222, 226, 232, 0.98); + border-radius: 12px; + background: rgba(255, 255, 255, 0.985); + box-shadow: + 0 1px 2px rgba(17, 24, 39, 0.06), + 0 18px 48px rgba(17, 24, 39, 0.17); + transform: none; + backdrop-filter: blur(16px); +} + +.visual-annotation-header { + min-height: 46px; + display: flex; + align-items: center; + gap: 10px; + padding: 9px 12px 7px; +} + +.visual-annotation-header strong { + min-width: 0; + flex: 1; + color: #20242b; + font-size: 14px; + font-weight: 720; + line-height: 20px; + letter-spacing: 0; +} + +.visual-annotation-header-actions { + display: inline-flex; + align-items: center; + gap: 2px; +} + +.visual-annotation-header-actions button { + width: 28px; + height: 28px; + display: inline-grid; + place-items: center; + border-radius: 8px; + color: #626975; + background: transparent; +} + +.visual-annotation-header-actions button:hover:not(:disabled), +.visual-annotation-header-actions button:focus-visible { + color: #222831; + background: #f1f3f5; +} + +.visual-annotation-header-actions button:disabled { + opacity: 0.36; + cursor: not-allowed; +} + +.visual-annotation-tools { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 4px; + padding: 2px 12px 12px; +} + +.visual-annotation-tools button { + height: 38px; + display: inline-grid; + place-items: center; + border: 1px solid transparent; + border-radius: 8px; + color: #707783; + background: #f7f8f9; + transition: background 0.14s ease, border-color 0.14s ease, color 0.14s ease, box-shadow 0.14s ease; +} + +.visual-annotation-tools button:hover:not(:disabled), +.visual-annotation-tools button:focus-visible { + color: #282e37; + background: #f0f2f4; +} + +.visual-annotation-tools button.active { + border-color: rgba(47, 128, 255, 0.42); + color: #176fe9; + background: rgba(47, 128, 255, 0.09); + box-shadow: inset 0 0 0 1px rgba(47, 128, 255, 0.05); +} + +.visual-annotation-tools button:disabled { + opacity: 0.44; + cursor: not-allowed; +} + +.visual-annotation-list-heading { + min-height: 28px; + display: flex; + align-items: center; + gap: 7px; + padding: 0 12px 6px; + color: #6d7480; + font-size: 12px; + font-weight: 650; +} + +.visual-annotation-list-heading small { + min-width: 18px; + height: 18px; + display: inline-grid; + place-items: center; + border-radius: 9px; + color: #657080; + background: #eff1f4; + font-size: 10px; + font-weight: 700; +} + +.visual-annotation-list { + min-height: 0; + display: flex; + flex-direction: column; + gap: 6px; + overflow-x: hidden; + overflow-y: auto; + padding: 0 12px 12px; + scrollbar-width: thin; + scrollbar-color: rgba(160, 166, 176, 0.56) transparent; +} + +.visual-annotation-list::-webkit-scrollbar { + width: 7px; +} + +.visual-annotation-list::-webkit-scrollbar-thumb { + border-radius: 999px; + background: rgba(160, 166, 176, 0.56); +} + +.visual-annotation-empty { + min-height: 62px; + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + border: 1px dashed #dde1e6; + border-radius: 8px; + color: #9aa1ac; + background: #fafafa; + font-size: 12px; + line-height: 18px; +} + +.visual-annotation-item { + min-width: 0; + min-height: 42px; + display: grid; + grid-template-columns: 24px 16px minmax(0, 1fr) 28px; + align-items: center; + gap: 6px; + padding: 4px 5px 4px 7px; + border: 1px solid #e4e7eb; + border-radius: 8px; + background: #fff; + transition: border-color 0.14s ease, background 0.14s ease, box-shadow 0.14s ease; +} + +.visual-annotation-item:hover, +.visual-annotation-item:focus-within { + border-color: #d7dce3; + background: #fbfbfc; +} + +.visual-annotation-item.selected { + border-color: rgba(47, 128, 255, 0.58); + background: rgba(47, 128, 255, 0.055); + box-shadow: 0 0 0 1px rgba(47, 128, 255, 0.06); +} + +.visual-annotation-item.needs-instruction:not(.selected) { + border-color: rgba(237, 170, 76, 0.5); +} + +.visual-annotation-index { + width: 24px; + height: 24px; + display: inline-grid; + place-items: center; + border-radius: 7px; + color: #176fe9; + background: rgba(47, 128, 255, 0.1); + font-size: 11px; + font-weight: 760; +} + +.visual-annotation-item-tool { + color: #77808d; +} + +.visual-annotation-item input { + width: 100%; + min-width: 0; + height: 30px; + padding: 0; + border: 0; + outline: 0; + color: #2a3039; + background: transparent; + font: inherit; + font-size: 12px; + font-weight: 520; + letter-spacing: 0; +} + +.visual-annotation-item input::placeholder { + color: #a0a6b0; +} + +.visual-annotation-item > button { + width: 28px; + height: 28px; + display: inline-grid; + place-items: center; + border-radius: 7px; + color: #8a919c; + background: transparent; +} + +.visual-annotation-item > button:hover:not(:disabled), +.visual-annotation-item > button:focus-visible { + color: #d14343; + background: rgba(209, 67, 67, 0.08); +} + +.visual-annotation-footer { + min-width: 0; + display: grid; + grid-template-columns: minmax(0, 1fr) 92px; + gap: 8px; + padding: 11px 12px 12px; + border-top: 1px solid #eceef1; +} + +.visual-annotation-model, +.visual-annotation-generate { + min-width: 0; + height: 36px; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 7px; + border-radius: 9px; + font-size: 12px; + font-weight: 650; +} + +.visual-annotation-model { + justify-content: flex-start; + padding: 0 10px; + border: 1px solid #e1e4e8; + color: #303640; + background: #f8f8f8; +} + +.visual-annotation-model > span { + min-width: 0; + flex: 1; + overflow: hidden; + text-align: left; + text-overflow: ellipsis; + white-space: nowrap; +} + +.visual-annotation-model:hover:not(:disabled), +.visual-annotation-model:focus-visible { + background: #f1f2f4; +} + +.visual-annotation-generate { + color: #fff; + background: #22262d; +} + +.visual-annotation-generate:hover:not(:disabled), +.visual-annotation-generate:focus-visible { + background: #0f1216; +} + +.visual-annotation-model:disabled, +.visual-annotation-generate:disabled { + opacity: 0.44; + cursor: not-allowed; +} + +.visual-annotation-model-menu { + width: 250px; +} + +.visual-annotation-model-menu .ui-dropdown-menu-item { + display: grid; + grid-template-columns: 18px minmax(0, 1fr) auto 16px; + gap: 8px; +} + +.visual-annotation-model-menu .ui-dropdown-menu-item span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.visual-annotation-model-menu .ui-dropdown-menu-item small { + color: #8a919c; + font-size: 11px; +} + +.visual-annotation-spin { + animation: visual-annotation-spin 0.8s linear infinite; +} + +@keyframes visual-annotation-spin { + to { + transform: rotate(360deg); + } +} + +@media (max-width: 680px) { + .visual-annotation-panel { + grid-template-rows: auto auto auto minmax(52px, 150px) auto; + } + + .visual-annotation-footer { + grid-template-columns: minmax(0, 1fr) 84px; + } +} diff --git a/frontend/src/ui/components/VisualAnnotationPanel/index.tsx b/frontend/src/ui/components/VisualAnnotationPanel/index.tsx new file mode 100644 index 0000000..d8f65e8 --- /dev/null +++ b/frontend/src/ui/components/VisualAnnotationPanel/index.tsx @@ -0,0 +1,554 @@ +"use client"; + +import { Brush, Check, ChevronDown, Circle, Loader2, Sparkles, Square, Trash2, Undo2, X, type LucideIcon } from "lucide-react"; +import { useEffect, useMemo, useRef, useState, type CSSProperties, type PointerEvent as ReactPointerEvent, type ReactNode } from "react"; +import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"; +import { useI18n } from "@/i18n/i18n"; +import type { CanvasComposerModel } from "@/ui/components/CanvasAgentComposer"; +import type { NodeScreenBounds } from "@/ui/pages/CanvasWorkspace/canvas/types"; +import "./index.css"; + +export type VisualAnnotationTool = "brush" | "circle" | "rectangle" | "cross"; + +export type VisualAnnotationPoint = { + x: number; + y: number; +}; + +export type VisualAnnotationMark = { + id: string; + index: number; + tool: VisualAnnotationTool; + points: VisualAnnotationPoint[]; + instruction: string; +}; + +type DrawingState = { + pointerId: number; + mark: VisualAnnotationMark; +}; + +type ToolOption = { + id: VisualAnnotationTool; + label: string; + icon: LucideIcon; +}; + +const annotationColor = "#2F80FF"; + +export function VisualAnnotationPanel({ + bounds, + stageWidth, + stageHeight, + marks, + models, + selectedModel, + disabled, + submitting, + onMarksChange, + onModelChange, + onCancel, + onApply +}: { + bounds: NodeScreenBounds; + stageWidth: number; + stageHeight: number; + marks: VisualAnnotationMark[]; + models: CanvasComposerModel[]; + selectedModel: CanvasComposerModel; + disabled: boolean; + submitting: boolean; + onMarksChange: (marks: VisualAnnotationMark[]) => void; + onModelChange: (modelId: string) => void; + onCancel: () => void; + onApply: () => void; +}) { + const { t } = useI18n(); + const drawingRef = useRef(null); + const [activeTool, setActiveTool] = useState("brush"); + const [draftMark, setDraftMark] = useState(null); + const [selectedMarkId, setSelectedMarkId] = useState(null); + const [modelOpen, setModelOpen] = useState(false); + const nextIndex = useMemo(() => marks.reduce((max, mark) => Math.max(max, mark.index), 0) + 1, [marks]); + const canApply = marks.length > 0 && marks.every((mark) => mark.instruction.trim()); + const toolOptions: ToolOption[] = [ + { id: "brush", label: t("visualAnnotationBrush"), icon: Brush }, + { id: "circle", label: t("visualAnnotationCircle"), icon: Circle }, + { id: "rectangle", label: t("visualAnnotationRectangle"), icon: Square }, + { id: "cross", label: t("visualAnnotationCross"), icon: X } + ]; + + useEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape") return; + if (drawingRef.current) { + drawingRef.current = null; + setDraftMark(null); + return; + } + onCancel(); + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [onCancel]); + + const beginMark = (event: ReactPointerEvent) => { + event.preventDefault(); + event.stopPropagation(); + if (disabled) return; + const point = normalizedPointer(event); + const mark: VisualAnnotationMark = { + id: visualAnnotationId(), + index: nextIndex, + tool: activeTool, + points: activeTool === "brush" ? [point] : [point, point], + instruction: "" + }; + drawingRef.current = { pointerId: event.pointerId, mark }; + setDraftMark(mark); + setSelectedMarkId(mark.id); + event.currentTarget.setPointerCapture(event.pointerId); + }; + + const updateMark = (event: ReactPointerEvent) => { + const drawing = drawingRef.current; + if (!drawing || drawing.pointerId !== event.pointerId) return; + event.preventDefault(); + event.stopPropagation(); + const point = normalizedPointer(event); + const points = drawing.mark.tool === "brush" ? appendBrushPoint(drawing.mark.points, point, bounds) : [drawing.mark.points[0], point]; + drawing.mark = { ...drawing.mark, points }; + drawingRef.current = drawing; + setDraftMark(drawing.mark); + }; + + const finishMark = (event: ReactPointerEvent) => { + const drawing = drawingRef.current; + if (!drawing || drawing.pointerId !== event.pointerId) return; + event.preventDefault(); + event.stopPropagation(); + const mark = finalizeMark(drawing.mark, bounds); + drawingRef.current = null; + setDraftMark(null); + onMarksChange([...marks, mark]); + window.requestAnimationFrame(() => { + document.querySelector(`[data-visual-annotation-input="${mark.id}"]`)?.focus(); + }); + }; + + const cancelDraft = (event: ReactPointerEvent) => { + if (drawingRef.current?.pointerId !== event.pointerId) return; + event.preventDefault(); + event.stopPropagation(); + drawingRef.current = null; + setDraftMark(null); + }; + + const updateInstruction = (id: string, instruction: string) => { + onMarksChange(marks.map((mark) => (mark.id === id ? { ...mark, instruction } : mark))); + }; + + const removeMark = (id: string) => { + const nextMarks = marks.filter((mark) => mark.id !== id); + onMarksChange(nextMarks); + setSelectedMarkId((current) => (current === id ? nextMarks.at(-1)?.id ?? null : current)); + }; + + const undoLastMark = () => { + if (marks.length === 0 || disabled) return; + const nextMarks = marks.slice(0, -1); + onMarksChange(nextMarks); + setSelectedMarkId(nextMarks.at(-1)?.id ?? null); + }; + + return ( + <> +
+ +
+ +
event.stopPropagation()} + onWheel={(event) => event.stopPropagation()} + > +
+ {t("visualAnnotation")} +
+ + +
+
+ +
+ {toolOptions.map((tool) => { + const Icon = tool.icon; + const active = tool.id === activeTool; + return ( + + ); + })} +
+ +
+ {t("visualAnnotationList")} + {marks.length > 0 && {marks.length}} +
+ +
+ {marks.length === 0 ? ( +
+ + {t("visualAnnotationEmpty")} +
+ ) : ( + marks.map((mark) => { + const ToolIcon = toolOptions.find((tool) => tool.id === mark.tool)?.icon ?? Brush; + const selected = mark.id === selectedMarkId; + return ( +
setSelectedMarkId(mark.id)}> + {mark.index} + + setSelectedMarkId(mark.id)} + onChange={(event) => updateInstruction(mark.id, event.target.value)} + /> + +
+ ); + }) + )} +
+ +
+ + + + + + {models.map((model) => ( + { + onModelChange(model.id); + setModelOpen(false); + }} + > + + {model.label} + {model.credits} + {model.id === selectedModel.id && } + + ))} + + + +
+
+ + ); +} + +function VisualMark({ mark, bounds, selected, draft = false }: { mark: VisualAnnotationMark; bounds: NodeScreenBounds; selected: boolean; draft?: boolean }) { + const strokeWidth = annotationStrokeWidth(bounds); + const points = mark.points.map((point) => ({ x: point.x * bounds.width, y: point.y * bounds.height })); + const shapeClass = `visual-annotation-mark ${selected ? "selected" : ""} ${draft ? "draft" : ""}`; + const anchor = markAnchor(points, mark.tool); + const badgeAnchor = { + x: clamp(anchor.x, 13, Math.max(13, bounds.width - 13)), + y: clamp(anchor.y, 13, Math.max(13, bounds.height - 13)) + }; + let shape: ReactNode = null; + + if (mark.tool === "brush") { + if (points.length === 1) { + shape = ; + } else { + shape = ; + } + } else { + const rect = pointRect(points); + if (mark.tool === "circle") { + shape = ; + } else if (mark.tool === "rectangle") { + shape = ; + } else { + shape = ( + + + + + ); + } + } + + return ( + + {shape} + {!draft && ( + + + + {mark.index} + + + )} + + ); +} + +export function visualAnnotationPayload(marks: VisualAnnotationMark[]) { + return { + annotations: marks.map((mark) => ({ + index: mark.index, + type: mark.tool, + instruction: mark.instruction.trim(), + points: mark.points.map((point) => ({ x: roundCoordinate(point.x), y: roundCoordinate(point.y) })) + })) + }; +} + +export async function renderVisualAnnotationGuide(source: Blob, marks: VisualAnnotationMark[]) { + const image = await loadBlobImage(source); + const width = Math.max(1, image.naturalWidth || image.width); + const height = Math.max(1, image.naturalHeight || image.height); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + if (!context) throw new Error("Failed to create annotation canvas"); + context.drawImage(image, 0, 0, width, height); + const strokeWidth = Math.max(5, Math.min(28, Math.min(width, height) * 0.012)); + + marks.forEach((mark) => { + const points = mark.points.map((point) => ({ x: point.x * width, y: point.y * height })); + context.save(); + context.strokeStyle = annotationColor; + context.fillStyle = annotationColor; + context.lineWidth = strokeWidth; + context.lineCap = "round"; + context.lineJoin = "round"; + context.shadowColor = "rgba(255, 255, 255, 0.86)"; + context.shadowBlur = Math.max(2, strokeWidth * 0.18); + drawCanvasMark(context, mark.tool, points, strokeWidth); + context.restore(); + drawCanvasBadge(context, mark.index, markAnchor(points, mark.tool), strokeWidth); + }); + + return canvasBlob(canvas, "image/png"); +} + +function drawCanvasMark(context: CanvasRenderingContext2D, tool: VisualAnnotationTool, points: VisualAnnotationPoint[], strokeWidth: number) { + if (tool === "brush") { + if (points.length === 1) { + context.beginPath(); + context.arc(points[0].x, points[0].y, strokeWidth / 2, 0, Math.PI * 2); + context.fill(); + return; + } + context.beginPath(); + points.forEach((point, index) => (index === 0 ? context.moveTo(point.x, point.y) : context.lineTo(point.x, point.y))); + context.stroke(); + return; + } + + const rect = pointRect(points); + context.beginPath(); + if (tool === "circle") { + context.ellipse(rect.x + rect.width / 2, rect.y + rect.height / 2, Math.max(rect.width / 2, 1), Math.max(rect.height / 2, 1), 0, 0, Math.PI * 2); + } else if (tool === "rectangle") { + context.rect(rect.x, rect.y, rect.width, rect.height); + } else { + context.moveTo(rect.x, rect.y); + context.lineTo(rect.x + rect.width, rect.y + rect.height); + context.moveTo(rect.x + rect.width, rect.y); + context.lineTo(rect.x, rect.y + rect.height); + } + context.stroke(); +} + +function drawCanvasBadge(context: CanvasRenderingContext2D, index: number, point: VisualAnnotationPoint, strokeWidth: number) { + const radius = Math.max(13, strokeWidth * 1.35); + const x = clamp(point.x, radius + 2, context.canvas.width - radius - 2); + const y = clamp(point.y, radius + 2, context.canvas.height - radius - 2); + context.save(); + context.beginPath(); + context.arc(x, y, radius, 0, Math.PI * 2); + context.fillStyle = annotationColor; + context.fill(); + context.lineWidth = Math.max(2, strokeWidth * 0.22); + context.strokeStyle = "#FFFFFF"; + context.stroke(); + context.fillStyle = "#FFFFFF"; + context.font = `700 ${Math.max(13, radius * 1.08)}px sans-serif`; + context.textAlign = "center"; + context.textBaseline = "middle"; + context.fillText(String(index), x, y + 0.5); + context.restore(); +} + +function annotationPanelStyle(bounds: NodeScreenBounds, stageWidth: number, stageHeight: number): CSSProperties { + const margin = 12; + const gap = 14; + const panelWidth = Math.min(320, Math.max(260, stageWidth - margin * 2)); + const panelHeight = Math.min(560, Math.max(340, stageHeight - margin * 2)); + const right = bounds.left + bounds.width + gap; + const left = bounds.left - panelWidth - gap; + const fitsRight = right + panelWidth <= stageWidth - margin; + const fitsLeft = left >= margin; + const panelLeft = fitsRight ? right : fitsLeft ? left : clamp(bounds.left + bounds.width / 2 - panelWidth / 2, margin, Math.max(margin, stageWidth - panelWidth - margin)); + const panelTop = clamp(bounds.top + bounds.height / 2 - panelHeight / 2, margin, Math.max(margin, stageHeight - panelHeight - margin)); + return { left: panelLeft, top: panelTop, width: panelWidth, maxHeight: panelHeight }; +} + +function overlayStyle(bounds: NodeScreenBounds): CSSProperties { + return { left: bounds.left, top: bounds.top, width: bounds.width, height: bounds.height }; +} + +function normalizedPointer(event: ReactPointerEvent): VisualAnnotationPoint { + const rect = event.currentTarget.getBoundingClientRect(); + return { + x: clamp((event.clientX - rect.left) / Math.max(rect.width, 1), 0, 1), + y: clamp((event.clientY - rect.top) / Math.max(rect.height, 1), 0, 1) + }; +} + +function appendBrushPoint(points: VisualAnnotationPoint[], point: VisualAnnotationPoint, bounds: NodeScreenBounds) { + const previous = points.at(-1); + if (!previous) return [point]; + const distance = Math.hypot((point.x - previous.x) * bounds.width, (point.y - previous.y) * bounds.height); + if (distance < 1.5) return points; + return [...points, point]; +} + +function finalizeMark(mark: VisualAnnotationMark, bounds: NodeScreenBounds): VisualAnnotationMark { + if (mark.tool === "brush") return mark; + const start = mark.points[0]; + const end = mark.points[1] ?? start; + const minWidth = Math.min(0.16, 28 / Math.max(bounds.width, 1)); + const minHeight = Math.min(0.16, 28 / Math.max(bounds.height, 1)); + const width = Math.abs(end.x - start.x); + const height = Math.abs(end.y - start.y); + if (width >= minWidth || height >= minHeight) return mark; + return { + ...mark, + points: [ + { x: clamp(start.x - minWidth / 2, 0, 1), y: clamp(start.y - minHeight / 2, 0, 1) }, + { x: clamp(start.x + minWidth / 2, 0, 1), y: clamp(start.y + minHeight / 2, 0, 1) } + ] + }; +} + +function pointRect(points: VisualAnnotationPoint[]) { + const start = points[0] ?? { x: 0, y: 0 }; + const end = points.at(-1) ?? start; + return { + x: Math.min(start.x, end.x), + y: Math.min(start.y, end.y), + width: Math.abs(end.x - start.x), + height: Math.abs(end.y - start.y) + }; +} + +function markAnchor(points: VisualAnnotationPoint[], tool: VisualAnnotationTool) { + if (points.length === 0) return { x: 0, y: 0 }; + if (tool === "brush") return points.at(-1) ?? points[0]; + const rect = pointRect(points); + return { x: rect.x, y: rect.y }; +} + +function pathFromPoints(points: VisualAnnotationPoint[]) { + return points.map((point, index) => `${index === 0 ? "M" : "L"} ${point.x.toFixed(2)} ${point.y.toFixed(2)}`).join(" "); +} + +function annotationStrokeWidth(bounds: NodeScreenBounds) { + return Math.max(4, Math.min(10, Math.min(bounds.width, bounds.height) * 0.018)); +} + +function visualAnnotationId() { + if (typeof crypto !== "undefined" && "randomUUID" in crypto) return crypto.randomUUID(); + return `visual-annotation-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +function roundCoordinate(value: number) { + return Math.round(clamp(value, 0, 1) * 10000) / 10000; +} + +function clamp(value: number, min: number, max: number) { + return Math.max(min, Math.min(max, value)); +} + +async function loadBlobImage(blob: Blob) { + const url = URL.createObjectURL(blob); + try { + const image = new Image(); + image.decoding = "async"; + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error("Failed to load annotation source image")); + image.src = url; + }); + return image; + } finally { + URL.revokeObjectURL(url); + } +} + +function canvasBlob(canvas: HTMLCanvasElement, type: string) { + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error("Failed to encode annotation guide"))), type); + }); +} diff --git a/frontend/src/ui/pages/CanvasWorkspace/index.tsx b/frontend/src/ui/pages/CanvasWorkspace/index.tsx index b04de59..7bb013e 100644 --- a/frontend/src/ui/pages/CanvasWorkspace/index.tsx +++ b/frontend/src/ui/pages/CanvasWorkspace/index.tsx @@ -32,6 +32,7 @@ import { AdjustPanel, CropOverlay, defaultToolbarVisibility, EraserOverlay, Expa import { ImageTextEditPanel, type EditableTextLayer } from "@/ui/components/ImageTextEditPanel"; import { MultiSelectionOverlay, type MultiSelectionResizeHandle } from "@/ui/components/MultiSelectionOverlay"; import { TextStyleToolbar, type TextStylePatch } from "@/ui/components/TextStyleToolbar"; +import { renderVisualAnnotationGuide, visualAnnotationPayload, VisualAnnotationPanel, type VisualAnnotationMark, type VisualAnnotationTool } from "@/ui/components/VisualAnnotationPanel"; import { loadMotevaFontCatalog } from "@/ui/components/TextStyleToolbar/fontCatalog"; import { WorkspaceTitle } from "@/ui/components/WorkspaceTitle"; import { SharePopover } from "@/ui/components/SharePopover"; @@ -115,7 +116,7 @@ type HomeImageModel = { credits: number; }; -type ImageActionMode = "quick-edit" | "eraser" | "edit-text" | "multi-angle" | "move-object" | "mockup" | "adjust" | "expand" | "crop" | "flip-rotate" | null; +type ImageActionMode = "quick-edit" | "visual-annotation" | "eraser" | "edit-text" | "multi-angle" | "move-object" | "mockup" | "adjust" | "expand" | "crop" | "flip-rotate" | null; const defaultCropSelection: NormalizedRect = { x: 0, y: 0, width: 1, height: 1 }; const imageModelBlueprints = [ { @@ -630,6 +631,8 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess const [toolbarVisibility, setToolbarVisibility] = useState(defaultToolbarVisibility); const [imageActionMode, setImageActionMode] = useState(null); const [quickEditPrompt, setQuickEditPrompt] = useState(""); + const [visualAnnotationMarks, setVisualAnnotationMarks] = useState([]); + const [visualAnnotationSubmitting, setVisualAnnotationSubmitting] = useState(false); const [eraserBrushSize, setEraserBrushSize] = useState(42); const [eraserBrushColor, setEraserBrushColor] = useState("#5E6CFF"); const [eraserBrushOpacity, setEraserBrushOpacity] = useState(72); @@ -663,7 +666,7 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess zoom: 50, prompt: "" }); - const canvasActionWorkActive = canvasActionGenerating || textEditLoading || nodes.some(isCanvasActionWorkingNode); + const canvasActionWorkActive = canvasActionGenerating || textEditLoading || visualAnnotationSubmitting || nodes.some(isCanvasActionWorkingNode); const agentTimelineActive = agentGenerating || Boolean(project && !projectHasCanvasActionGeneration(project) && nodes.some(isAgentVisibleGeneratingNode)); const mockupPreparing = Boolean(mockupPendingTargetId); const anyGenerating = agentGenerating || imageGeneratorGenerating || canvasActionWorkActive || mockupPreparing; @@ -1147,6 +1150,8 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess useEffect(() => { setImageActionMode(null); setQuickEditPrompt(""); + setVisualAnnotationMarks([]); + setVisualAnnotationSubmitting(false); setEraserMarks([]); setTextEditItems([]); setTextEditLoading(false); @@ -2841,6 +2846,7 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess applyProject(result.project); startProjectEvents(result.project.id, result.project.lastThreadId, "canvas-action"); setImageActionMode(null); + setVisualAnnotationMarks([]); setEraserMarks([]); setTextEditItems([]); setTextEditLoading(false); @@ -3086,6 +3092,31 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess void runImageNodeAction("quick-edit", { prompt: quickEditPrompt.trim() }); }; + const applyVisualAnnotation = async () => { + const targetNode = selectedImageNode; + if (!targetNode || visualAnnotationSubmitting || visualAnnotationMarks.length === 0 || visualAnnotationMarks.some((mark) => !mark.instruction.trim())) return; + setVisualAnnotationSubmitting(true); + setError(""); + try { + const source = await imageNodeRasterBlob(targetNode, "image/png"); + const guide = await renderVisualAnnotationGuide(source, visualAnnotationMarks); + const guideFile = new File([guide], `visual-annotation-${Date.now()}.png`, { type: "image/png" }); + const annotationImage = await designGateway.uploadImage(guideFile); + const instructions = visualAnnotationMarks + .map((mark) => `${mark.index}. ${visualAnnotationToolLabel(mark.tool, t)}: ${mark.instruction.trim()}`) + .join("\n"); + await runImageNodeAction("visual-annotation", { + prompt: t("visualAnnotationPrompt", { instructions }), + annotationImage, + selection: JSON.stringify(visualAnnotationPayload(visualAnnotationMarks)) + }); + } catch (err) { + setError(toUserMessageText(err, "generateError")); + } finally { + setVisualAnnotationSubmitting(false); + } + }; + const openQuickEditPanel = () => { if (!selectedImageNode) return; const stageWidth = canvasStageSize.width; @@ -3743,7 +3774,7 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess exportImageNode(selectedImageNode, "svg")} /> ) : ( <> - {imageActionMode !== "quick-edit" && imageActionMode !== "eraser" && imageActionMode !== "edit-text" && imageActionMode !== "expand" && imageActionMode !== "crop" && imageActionMode !== "flip-rotate" && selectedImageNode.status !== "text-extracting" && mockupPendingTargetId !== selectedImageNode.id && ( + {imageActionMode !== "quick-edit" && imageActionMode !== "visual-annotation" && imageActionMode !== "eraser" && imageActionMode !== "edit-text" && imageActionMode !== "expand" && imageActionMode !== "crop" && imageActionMode !== "flip-rotate" && selectedImageNode.status !== "text-extracting" && mockupPendingTargetId !== selectedImageNode.id && ( { + setVisualAnnotationMarks([]); + setImageActionMode("visual-annotation"); + }} onUpscale={(scale) => void runImageNodeAction("upscale", { prompt: t("upscalePrompt", { scale }), @@ -3794,6 +3829,26 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess onSubmit={applyQuickEdit} /> )} + {imageActionMode === "visual-annotation" && ( + { + if (visualAnnotationSubmitting) return; + setImageActionMode(null); + setVisualAnnotationMarks([]); + }} + onApply={() => void applyVisualAnnotation()} + /> + )} {imageActionMode === "eraser" && ( string) { + const keys: Record = { + brush: "visualAnnotationBrush", + circle: "visualAnnotationCircle", + rectangle: "visualAnnotationRectangle", + cross: "visualAnnotationCross" + }; + return t(keys[tool]); +} + function isVectorizedImageNode(node: CanvasNode) { return node.type === "image" && node.layerRole === "vectorized-image"; } diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo index ffbf35d..3fb05bf 100644 --- a/frontend/tsconfig.tsbuildinfo +++ b/frontend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/main.tsx","./src/vite-env.d.ts","./src/app/layout.tsx","./src/app/page.tsx","./src/app/root.tsx","./src/app/[...path]/page.tsx","./src/app/api/[...path]/route.ts","./src/app/project/[id]/page.tsx","./src/app/projects/page.tsx","./src/app/projects/[id]/page.tsx","./src/application/route.ts","./src/components/ui/button.tsx","./src/components/ui/context-menu.tsx","./src/components/ui/dropdown-menu.tsx","./src/components/ui/popover.tsx","./src/components/ui/scroll-area.tsx","./src/components/ui/switch.tsx","./src/components/ui/tabs.tsx","./src/domain/auth.ts","./src/domain/brandkit.ts","./src/domain/design.ts","./src/i18n/i18n.tsx","./src/infrastructure/authgateway.ts","./src/infrastructure/brandkitrepository.ts","./src/infrastructure/designgateway.ts","./src/infrastructure/imageuploadencoding.ts","./src/infrastructure/usermessages.ts","./src/infrastructure/designgateway/agents.ts","./src/infrastructure/designgateway/assets.ts","./src/infrastructure/designgateway/canvas.ts","./src/infrastructure/designgateway/canvassnapshot.ts","./src/infrastructure/designgateway/clientids.ts","./src/infrastructure/designgateway/events.ts","./src/infrastructure/designgateway/generation.ts","./src/infrastructure/designgateway/projects.ts","./src/infrastructure/designgateway/request.ts","./src/infrastructure/designgateway/responses.ts","./src/infrastructure/designgateway/sharecontext.ts","./src/infrastructure/designgateway/sharing.ts","./src/infrastructure/designgateway/types.ts","./src/infrastructure/designgateway/uicontent.ts","./src/ui/app.tsx","./src/ui/auth/authprovider/index.tsx","./src/ui/auth/logindialog/brandlogo.tsx","./src/ui/auth/logindialog/otpinput.tsx","./src/ui/auth/logindialog/turnstilewidget.tsx","./src/ui/auth/logindialog/index.tsx","./src/ui/components/brandmark.tsx","./src/ui/components/accountmanagementdialog/index.tsx","./src/ui/components/artboardpreview/index.tsx","./src/ui/components/brandkitselector/index.tsx","./src/ui/components/canvasagentcomposer/index.tsx","./src/ui/components/canvasagentemptystate/defaults.ts","./src/ui/components/canvasagentemptystate/index.tsx","./src/ui/components/canvasagenttimeline/index.tsx","./src/ui/components/canvasannotationpreview/index.tsx","./src/ui/components/canvasannotations/index.tsx","./src/ui/components/canvasminimap/index.tsx","./src/ui/components/canvasnodecontextmenu/index.tsx","./src/ui/components/canvaspanels/index.tsx","./src/ui/components/canvastoast/index.tsx","./src/ui/components/canvasworkoverlay/index.tsx","./src/ui/components/canvaszoommenu/index.tsx","./src/ui/components/deleteprojectdialog/index.tsx","./src/ui/components/imagedropoverlay/index.tsx","./src/ui/components/imagegeneratorfloatingcomposer/index.tsx","./src/ui/components/imagenodeactions/index.tsx","./src/ui/components/imagetexteditpanel/index.tsx","./src/ui/components/layermergeskeleton/index.tsx","./src/ui/components/markdowncontent/index.tsx","./src/ui/components/multianglepanel/index.tsx","./src/ui/components/multiselectionoverlay/index.tsx","./src/ui/components/projectcard/index.tsx","./src/ui/components/projectthumb/index.tsx","./src/ui/components/promptcomposer/index.tsx","./src/ui/components/sharepopover/index.tsx","./src/ui/components/textstyletoolbar/fontcatalog.ts","./src/ui/components/textstyletoolbar/index.tsx","./src/ui/components/toolbarcustomizedialog/index.tsx","./src/ui/components/workspacetitle/index.tsx","./src/ui/hooks/usefloatingtoolbarplacement.ts","./src/ui/hooks/usetypewritertext.ts","./src/ui/hooks/useusermessage.ts","./src/ui/lib/friendlyagenterrors.ts","./src/ui/lib/imageadjustments.ts","./src/ui/lib/imagedelivery.ts","./src/ui/lib/promptimagereferences.ts","./src/ui/pages/brandkitpage/index.tsx","./src/ui/pages/brandkitroutepage/index.tsx","./src/ui/pages/canvasworkspace/index.tsx","./src/ui/pages/canvasworkspace/canvas/canvascolorpopover.tsx","./src/ui/pages/canvasworkspace/canvas/framestyletoolbar.tsx","./src/ui/pages/canvasworkspace/canvas/frametooloverlay.tsx","./src/ui/pages/canvasworkspace/canvas/penstroketoolbar.tsx","./src/ui/pages/canvasworkspace/canvas/pentool.tsx","./src/ui/pages/canvasworkspace/canvas/workspacetoolbar.tsx","./src/ui/pages/canvasworkspace/canvas/agentmessagefilters.ts","./src/ui/pages/canvasworkspace/canvas/canvasgeometry.ts","./src/ui/pages/canvasworkspace/canvas/canvasnodeops.ts","./src/ui/pages/canvasworkspace/canvas/framestyle.ts","./src/ui/pages/canvasworkspace/canvas/generatortaskpolling.ts","./src/ui/pages/canvasworkspace/canvas/imagegeneratorsizing.ts","./src/ui/pages/canvasworkspace/canvas/nodeexport.ts","./src/ui/pages/canvasworkspace/canvas/nodereferenceimages.ts","./src/ui/pages/canvasworkspace/canvas/shapenode.ts","./src/ui/pages/canvasworkspace/canvas/textnodedefaults.ts","./src/ui/pages/canvasworkspace/canvas/types.ts","./src/ui/pages/canvasworkspace/canvas/canvasselectionframe/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapenodepreview/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapestyletoolbar/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapetoolmenu/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapetooloverlay/index.tsx","./src/ui/pages/canvasworkspace/canvas/hooks/usecanvasdocument.ts","./src/ui/pages/canvasworkspace/canvas/hooks/usecanvaspersistence.ts","./src/ui/pages/canvasworkspace/canvas/hooks/usecanvasselection.ts","./src/ui/pages/canvasworkspace/canvas/hooks/usegenerationstream.ts","./src/ui/pages/homepage/index.tsx","./src/ui/pages/homeshell/index.tsx","./src/ui/pages/notfoundpage/index.tsx","./src/ui/pages/projectspage/index.tsx","./src/ui/pages/projectsroutepage/index.tsx"],"version":"6.0.3"} \ No newline at end of file +{"root":["./src/main.tsx","./src/vite-env.d.ts","./src/app/layout.tsx","./src/app/page.tsx","./src/app/root.tsx","./src/app/[...path]/page.tsx","./src/app/api/[...path]/route.ts","./src/app/project/[id]/page.tsx","./src/app/projects/page.tsx","./src/app/projects/[id]/page.tsx","./src/application/route.ts","./src/components/ui/button.tsx","./src/components/ui/context-menu.tsx","./src/components/ui/dropdown-menu.tsx","./src/components/ui/popover.tsx","./src/components/ui/scroll-area.tsx","./src/components/ui/switch.tsx","./src/components/ui/tabs.tsx","./src/domain/auth.ts","./src/domain/brandkit.ts","./src/domain/design.ts","./src/i18n/i18n.tsx","./src/infrastructure/authgateway.ts","./src/infrastructure/brandkitrepository.ts","./src/infrastructure/designgateway.ts","./src/infrastructure/imageuploadencoding.ts","./src/infrastructure/usermessages.ts","./src/infrastructure/designgateway/agents.ts","./src/infrastructure/designgateway/assets.ts","./src/infrastructure/designgateway/canvas.ts","./src/infrastructure/designgateway/canvassnapshot.ts","./src/infrastructure/designgateway/clientids.ts","./src/infrastructure/designgateway/events.ts","./src/infrastructure/designgateway/generation.ts","./src/infrastructure/designgateway/projects.ts","./src/infrastructure/designgateway/request.ts","./src/infrastructure/designgateway/responses.ts","./src/infrastructure/designgateway/sharecontext.ts","./src/infrastructure/designgateway/sharing.ts","./src/infrastructure/designgateway/types.ts","./src/infrastructure/designgateway/uicontent.ts","./src/ui/app.tsx","./src/ui/auth/authprovider/index.tsx","./src/ui/auth/logindialog/brandlogo.tsx","./src/ui/auth/logindialog/otpinput.tsx","./src/ui/auth/logindialog/turnstilewidget.tsx","./src/ui/auth/logindialog/index.tsx","./src/ui/components/brandmark.tsx","./src/ui/components/accountmanagementdialog/index.tsx","./src/ui/components/artboardpreview/index.tsx","./src/ui/components/brandkitselector/index.tsx","./src/ui/components/canvasagentcomposer/index.tsx","./src/ui/components/canvasagentemptystate/defaults.ts","./src/ui/components/canvasagentemptystate/index.tsx","./src/ui/components/canvasagenttimeline/index.tsx","./src/ui/components/canvasannotationpreview/index.tsx","./src/ui/components/canvasannotations/index.tsx","./src/ui/components/canvasminimap/index.tsx","./src/ui/components/canvasnodecontextmenu/index.tsx","./src/ui/components/canvaspanels/index.tsx","./src/ui/components/canvastoast/index.tsx","./src/ui/components/canvasworkoverlay/index.tsx","./src/ui/components/canvaszoommenu/index.tsx","./src/ui/components/deleteprojectdialog/index.tsx","./src/ui/components/imagedropoverlay/index.tsx","./src/ui/components/imagegeneratorfloatingcomposer/index.tsx","./src/ui/components/imagenodeactions/index.tsx","./src/ui/components/imagetexteditpanel/index.tsx","./src/ui/components/layermergeskeleton/index.tsx","./src/ui/components/markdowncontent/index.tsx","./src/ui/components/multianglepanel/index.tsx","./src/ui/components/multiselectionoverlay/index.tsx","./src/ui/components/projectcard/index.tsx","./src/ui/components/projectthumb/index.tsx","./src/ui/components/promptcomposer/index.tsx","./src/ui/components/sharepopover/index.tsx","./src/ui/components/textstyletoolbar/fontcatalog.ts","./src/ui/components/textstyletoolbar/index.tsx","./src/ui/components/toolbarcustomizedialog/index.tsx","./src/ui/components/visualannotationpanel/index.tsx","./src/ui/components/workspacetitle/index.tsx","./src/ui/hooks/usefloatingtoolbarplacement.ts","./src/ui/hooks/usetypewritertext.ts","./src/ui/hooks/useusermessage.ts","./src/ui/lib/friendlyagenterrors.ts","./src/ui/lib/imageadjustments.ts","./src/ui/lib/imagedelivery.ts","./src/ui/lib/promptimagereferences.ts","./src/ui/pages/brandkitpage/index.tsx","./src/ui/pages/brandkitroutepage/index.tsx","./src/ui/pages/canvasworkspace/index.tsx","./src/ui/pages/canvasworkspace/canvas/canvascolorpopover.tsx","./src/ui/pages/canvasworkspace/canvas/framestyletoolbar.tsx","./src/ui/pages/canvasworkspace/canvas/frametooloverlay.tsx","./src/ui/pages/canvasworkspace/canvas/penstroketoolbar.tsx","./src/ui/pages/canvasworkspace/canvas/pentool.tsx","./src/ui/pages/canvasworkspace/canvas/workspacetoolbar.tsx","./src/ui/pages/canvasworkspace/canvas/agentmessagefilters.ts","./src/ui/pages/canvasworkspace/canvas/canvasgeometry.ts","./src/ui/pages/canvasworkspace/canvas/canvasnodeops.ts","./src/ui/pages/canvasworkspace/canvas/framestyle.ts","./src/ui/pages/canvasworkspace/canvas/generatortaskpolling.ts","./src/ui/pages/canvasworkspace/canvas/imagegeneratorsizing.ts","./src/ui/pages/canvasworkspace/canvas/nodeexport.ts","./src/ui/pages/canvasworkspace/canvas/nodereferenceimages.ts","./src/ui/pages/canvasworkspace/canvas/shapenode.ts","./src/ui/pages/canvasworkspace/canvas/textnodedefaults.ts","./src/ui/pages/canvasworkspace/canvas/types.ts","./src/ui/pages/canvasworkspace/canvas/canvasselectionframe/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapenodepreview/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapestyletoolbar/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapetoolmenu/index.tsx","./src/ui/pages/canvasworkspace/canvas/shapetooloverlay/index.tsx","./src/ui/pages/canvasworkspace/canvas/hooks/usecanvasdocument.ts","./src/ui/pages/canvasworkspace/canvas/hooks/usecanvaspersistence.ts","./src/ui/pages/canvasworkspace/canvas/hooks/usecanvasselection.ts","./src/ui/pages/canvasworkspace/canvas/hooks/usegenerationstream.ts","./src/ui/pages/homepage/index.tsx","./src/ui/pages/homeshell/index.tsx","./src/ui/pages/notfoundpage/index.tsx","./src/ui/pages/projectspage/index.tsx","./src/ui/pages/projectsroutepage/index.tsx"],"version":"6.0.3"} \ No newline at end of file diff --git a/server/API.md b/server/API.md index 7c448fe..61e3c36 100644 --- a/server/API.md +++ b/server/API.md @@ -434,6 +434,25 @@ Returns the updated project, the assistant message, and generated nodes. The generation path uses the configured design agent runner. The default is the `sky-valley/pi` Go agent adapter, with local desktop/MCP agent behavior disabled. +## Image Node Actions + +`POST /api/projects/:id/nodes/:nodeId/actions/async` + +Visual annotation edits keep the selected clean image as the base and provide a second uploaded guide image containing numbered brush, circle, rectangle, or cross marks. Each number maps to an editable instruction in `prompt` and the normalized geometry in `selection`. The model must use the guide only for location and must not render marks or numbers into the result. + +```json +{ + "action": "visual-annotation", + "prompt": "1. Cross: remove the chest logo\n2. Rectangle: change the trousers to navy", + "imageModel": "gpt-image-2", + "imageSize": "1024x1024", + "annotationImage": "https://cdn.example.com/annotation-guide.png", + "selection": "{\"annotations\":[{\"index\":1,\"type\":\"cross\",\"instruction\":\"remove the chest logo\",\"points\":[{\"x\":0.42,\"y\":0.28},{\"x\":0.58,\"y\":0.5}]}]}" +} +``` + +`annotationImage` and a non-empty `prompt` are required for `visual-annotation`. The original node remains unchanged. Submission creates a generating copy 32 canvas units to its right, and the completed image replaces only that copy through the standard asynchronous event lifecycle. + ## Generator Tasks `POST /api/generator/tasks` diff --git a/server/README.md b/server/README.md index be83815..1fb2eb6 100644 --- a/server/README.md +++ b/server/README.md @@ -12,6 +12,7 @@ go-zero backend for the Moteva-style infinite canvas MVP. - Full canvas project reads - Moteva-style generation mock for image/design workflows - Canvas layout save +- Numbered visual-annotation image edits with clean-source preservation and editable per-mark instructions - Chat history replay - DDD layout with domain, application, infrastructure, and generated handler layers - sky-valley/pi Go agent adapter with local desktop agent disabled diff --git a/server/img_infinite_canvas.api b/server/img_infinite_canvas.api index 4e7da59..b57a8e5 100644 --- a/server/img_infinite_canvas.api +++ b/server/img_infinite_canvas.api @@ -781,15 +781,16 @@ type GenerateResponse { } type NodeActionRequest { - Id string `path:"id"` - NodeId string `path:"nodeId"` - Action string `json:"action"` - Prompt string `json:"prompt,optional"` - ImageModel string `json:"imageModel,optional"` - ImageSize string `json:"imageSize,optional"` - Mask string `json:"mask,optional"` - Selection string `json:"selection,optional"` - Options map[string]string `json:"options,optional"` + Id string `path:"id"` + NodeId string `path:"nodeId"` + Action string `json:"action"` + Prompt string `json:"prompt,optional"` + ImageModel string `json:"imageModel,optional"` + ImageSize string `json:"imageSize,optional"` + Mask string `json:"mask,optional"` + Selection string `json:"selection,optional"` + AnnotationImage string `json:"annotationImage,optional"` + Options map[string]string `json:"options,optional"` } type PrepareMockupSurfaceRequest { diff --git a/server/internal/application/design_service.go b/server/internal/application/design_service.go index 685df0d..4291710 100644 --- a/server/internal/application/design_service.go +++ b/server/internal/application/design_service.go @@ -1021,6 +1021,9 @@ func (s *DesignService) RunNodeActionAsync(ctx context.Context, projectID string if req.Action == "" { return design.Project{}, fmt.Errorf("%w: action is required", design.ErrInvalidInput) } + if req.Action == "visual-annotation" && (strings.TrimSpace(req.AnnotationImage) == "" || strings.TrimSpace(req.Prompt) == "") { + return design.Project{}, fmt.Errorf("%w: visual annotation requires an annotation image and instructions", design.ErrInvalidInput) + } project, err := s.repo.Get(ctx, strings.TrimSpace(projectID)) if err != nil { @@ -1038,7 +1041,7 @@ func (s *DesignService) RunNodeActionAsync(ctx context.Context, projectID string now := s.now() threadID := newID() target := project.Nodes[nodeIndex] - if req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements" { + if req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements" || req.Action == "visual-annotation" { resultNode := newBackgroundRemovalPlaceholder(target) if req.Action == "vectorize" { resultNode = newVectorizePlaceholder(target) @@ -1046,6 +1049,8 @@ func (s *DesignService) RunNodeActionAsync(ctx context.Context, projectID string resultNode = newMoveObjectPlaceholder(target) } else if req.Action == "edit-elements" { resultNode = newLayerSeparationPlaceholder(target) + } else if req.Action == "visual-annotation" { + resultNode = newVisualAnnotationPlaceholder(target) } if req.Options == nil { req.Options = make(map[string]string) @@ -2020,13 +2025,17 @@ func (s *DesignService) completeNodeActionGeneration(ctx context.Context, projec } actionPrompt := buildNodeActionPrompt(req, target) + actionContents := []design.AgentContent{ + {Type: "text", Text: req.Prompt, TextSource: "input"}, + {Type: "image", ImageURL: target.Content, ImageWidth: int64(target.Width), ImageHeight: int64(target.Height)}, + } + if req.Action == "visual-annotation" && strings.TrimSpace(req.AnnotationImage) != "" { + actionContents = append(actionContents, design.AgentContent{Type: "image", ImageURL: strings.TrimSpace(req.AnnotationImage), ImageWidth: int64(target.Width), ImageHeight: int64(target.Height)}) + } actionMessages := []design.AgentChatMessage{ { - Role: "user", - Contents: []design.AgentContent{ - {Type: "text", Text: req.Prompt, TextSource: "input"}, - {Type: "image", ImageURL: target.Content, ImageWidth: int64(target.Width), ImageHeight: int64(target.Height)}, - }, + Role: "user", + Contents: actionContents, }, } projectForPlan := projectWithoutGeneratingNodes(project) @@ -2059,10 +2068,14 @@ func (s *DesignService) completeNodeActionGeneration(ctx context.Context, projec nodeIndex := findNodeIndex(project.Nodes, resultNodeID) if nodeIndex < 0 { - if req.Action != "move-object" { + if req.Action != "move-object" && req.Action != "visual-annotation" { return design.ErrNotFound } - project.Nodes = append(project.Nodes, newMoveObjectPlaceholder(target)) + resultNode := newMoveObjectPlaceholder(target) + if req.Action == "visual-annotation" { + resultNode = newVisualAnnotationPlaceholder(target) + } + project.Nodes = append(project.Nodes, resultNode) nodeIndex = len(project.Nodes) - 1 project.Nodes[nodeIndex].ID = resultNodeID } @@ -4825,7 +4838,7 @@ func normalizeNodeAction(action string) string { action = strings.ToLower(strings.TrimSpace(action)) action = strings.ReplaceAll(action, "_", "-") switch action { - case "quick-edit", "upscale", "remove-background", "eraser", "edit-elements", "edit-text", "multi-angle", "move-object", "mockup", "expand", "adjust", "crop", "vectorize": + case "quick-edit", "visual-annotation", "upscale", "remove-background", "eraser", "edit-elements", "edit-text", "multi-angle", "move-object", "mockup", "expand", "adjust", "crop", "vectorize": return action default: return "" @@ -4878,6 +4891,21 @@ func newMoveObjectPlaceholder(target design.Node) design.Node { return result } +func newVisualAnnotationPlaceholder(target design.Node) design.Node { + result := target + result.ID = newID() + result.Title = nodeActionWorkingTitle("visual-annotation") + result.X = target.X + target.Width + 32 + result.Y = target.Y + result.Status = "generating" + result.ParentID = target.ID + result.LayerRole = "visual-annotation-edit" + result.MockupSurface = design.MockupSurface{} + result.MockupModel = design.MockupModel{} + result.MockupSourceContent = "" + return result +} + func newLayerSeparationPlaceholder(target design.Node) design.Node { result := target result.ID = newID() @@ -4901,7 +4929,7 @@ func newLayerSeparationPlaceholder(target design.Node) design.Node { } func nodeActionResultNodeID(target design.Node, req design.NodeActionRequest) string { - if (req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements") && len(req.Options) > 0 { + if (req.Action == "remove-background" || req.Action == "vectorize" || req.Action == "move-object" || req.Action == "edit-elements" || req.Action == "visual-annotation") && len(req.Options) > 0 { if id := strings.TrimSpace(req.Options[nodeActionResultNodeIDOption]); id != "" { return id } @@ -5324,6 +5352,12 @@ func buildNodeActionPrompt(req design.NodeActionRequest, target design.Node) str switch req.Action { case "quick-edit": builder.WriteString("Action: quick image edit. Apply the user's prompt directly to the selected image.") + case "visual-annotation": + builder.WriteString("Action: visual annotation edit. The first image is the clean source and must remain the base image. The second image is an annotated guide only. Match its numbered blue marks to the numbered user instructions, edit only those indicated regions, preserve every unmarked region, and never reproduce guide marks, lines, badges, or numbers in the final image.") + if strings.TrimSpace(req.AnnotationImage) != "" { + builder.WriteString("\nAnnotated guide image URL: ") + builder.WriteString(strings.TrimSpace(req.AnnotationImage)) + } case "upscale": builder.WriteString("Action: model upscale. Increase detail, sharpness, texture fidelity, and perceived resolution while preserving the image exactly.") if value := req.Options["scale"]; value != "" { @@ -5412,6 +5446,8 @@ func nodeActionWorkingTitle(action string) string { switch action { case "quick-edit": return "快捷编辑中" + case "visual-annotation": + return "标注改图中" case "upscale": return "模型放大中" case "remove-background": @@ -5445,6 +5481,8 @@ func nodeActionResultTitle(action string) string { switch action { case "quick-edit": return "Quick Edit" + case "visual-annotation": + return "Annotated Edit" case "upscale": return "Upscaled Image" case "remove-background": @@ -5502,6 +5540,8 @@ func nodeActionThinkingContent(action string) string { switch action { case "quick-edit": return "正在按提示词对选中图片做快速编辑。" + case "visual-annotation": + return "正在读取图片标记和对应修改说明，按区域执行精准编辑。" case "upscale": return "正在通过模型增强分辨率、细节和边缘质量。" case "remove-background": @@ -5535,6 +5575,8 @@ func nodeActionToolContent(action string) string { switch action { case "quick-edit": return "准备图片引用、用户提示词和目标约束。" + case "visual-annotation": + return "准备未标注原图、标注引导图和逐项修改说明。" case "upscale": return "准备高清放大提示词，约束构图不漂移。" case "remove-background": @@ -5568,6 +5610,8 @@ func nodeActionDoneTitle(action string) string { switch action { case "quick-edit": return "快捷编辑完成" + case "visual-annotation": + return "标注改图完成" case "upscale": return "模型放大完成" case "remove-background": @@ -5611,6 +5655,9 @@ func nodeActionDoneContent(action string, prompt string) string { if action == "vectorize" { return fmt.Sprintf("已完成图片动作：%s。SVG 矢量结果已生成在原图旁边，可直接下载。", prompt) } + if action == "visual-annotation" { + return fmt.Sprintf("已完成图片动作：%s。原图保持不变，标注修改结果已生成在原图旁边。", prompt) + } return fmt.Sprintf("已完成图片动作：%s。结果已回写到无限画布中的选中图片。", prompt) } diff --git a/server/internal/application/generator_task.go b/server/internal/application/generator_task.go index 39af749..ff62fef 100644 --- a/server/internal/application/generator_task.go +++ b/server/internal/application/generator_task.go @@ -786,6 +786,9 @@ func generatorTaskInputArgsForNodeAction(target design.Node, req design.NodeActi Prompt: generatorTaskPromptForNodeAction(req), Resolution: resolutionLabel(imageSize), } + if normalizeNodeAction(req.Action) == "visual-annotation" && strings.TrimSpace(req.AnnotationImage) != "" { + input.Image = append(input.Image, strings.TrimSpace(req.AnnotationImage)) + } if normalizeNodeAction(req.Action) == "move-object" { input.SrcBox, input.DstBox = moveObjectBoxesFromSelection(req.Selection) } diff --git a/server/internal/application/visual_annotation_test.go b/server/internal/application/visual_annotation_test.go new file mode 100644 index 0000000..f6a23c7 --- /dev/null +++ b/server/internal/application/visual_annotation_test.go @@ -0,0 +1,98 @@ +package application + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "img_infinite_canvas/internal/domain/design" + "img_infinite_canvas/internal/infrastructure/memory" +) + +func TestRunVisualAnnotationCarriesCleanSourceAndGuideImage(t *testing.T) { + ctx := context.Background() + repo := memory.NewProjectRepository() + queue := &fakeJobQueue{} + service := NewDesignService(repo, nil, nil, nil, nil) + service.SetJobQueue(queue) + + project := design.Project{ + ID: "project-visual-annotation", + Title: "Annotated edit", + Status: design.StatusReady, + UpdatedAt: time.Now(), + Nodes: []design.Node{{ + ID: "image-1", + Type: design.NodeTypeImage, + Title: "Source", + Width: 640, + Height: 480, + Content: "https://example.com/source.png", + Status: "success", + }}, + } + if err := repo.Save(ctx, project); err != nil { + t.Fatal(err) + } + + _, err := service.RunNodeActionAsync(ctx, project.ID, "image-1", design.NodeActionRequest{ + Action: "visual-annotation", + Prompt: "1. Remove the logo", + }) + if !errors.Is(err, design.ErrInvalidInput) { + t.Fatalf("expected missing guide image to be rejected, got %v", err) + } + + selection := `{"annotations":[{"index":1,"type":"cross","instruction":"Remove the logo","points":[{"x":0.4,"y":0.3},{"x":0.6,"y":0.5}]}]}` + guideURL := "https://example.com/annotation-guide.png" + next, err := service.RunNodeActionAsync(ctx, project.ID, "image-1", design.NodeActionRequest{ + Action: "visual_annotation", + Prompt: "1. Cross: Remove the logo", + Selection: selection, + AnnotationImage: guideURL, + }) + if err != nil { + t.Fatal(err) + } + + if len(next.Nodes) != 2 { + t.Fatalf("expected the source plus an annotated-edit copy, got %#v", next.Nodes) + } + sourceNode := next.Nodes[0] + placeholder := next.Nodes[1] + if sourceNode.Status != "success" || sourceNode.Title != "Source" || sourceNode.Content != project.Nodes[0].Content { + t.Fatalf("expected the source image to remain unchanged, got %#v", sourceNode) + } + if placeholder.Status != "generating" || placeholder.Title != "标注改图中" || placeholder.ParentID != sourceNode.ID || placeholder.LayerRole != "visual-annotation-edit" { + t.Fatalf("expected a generating annotated-edit copy, got %#v", placeholder) + } + if placeholder.X != sourceNode.X+sourceNode.Width+32 || placeholder.Y != sourceNode.Y || placeholder.Content != sourceNode.Content { + t.Fatalf("expected the annotated-edit copy beside the source, got %#v", placeholder) + } + if len(queue.jobs) != 1 { + t.Fatalf("expected one visual annotation job, got %#v", queue.jobs) + } + request := queue.jobs[0].Request + if request.Action != "visual-annotation" || request.AnnotationImage != guideURL || request.Selection != selection { + t.Fatalf("visual annotation request was not preserved in the job: %#v", request) + } + if request.Options[nodeActionResultNodeIDOption] != placeholder.ID { + t.Fatalf("expected the job to target copied node %q, got %#v", placeholder.ID, request.Options) + } + + task, err := service.GeneratorTask(ctx, project.ID, next.LastThreadID) + if err != nil { + t.Fatal(err) + } + if len(task.OriginalInputArgs.Image) != 2 || task.OriginalInputArgs.Image[0] != project.Nodes[0].Content || task.OriginalInputArgs.Image[1] != guideURL { + t.Fatalf("expected clean source followed by annotation guide, got %#v", task.OriginalInputArgs.Image) + } + prompt := buildNodeActionPrompt(request, project.Nodes[0]) + for _, fragment := range []string{"first image is the clean source", "second image is an annotated guide only", "never reproduce guide marks", selection} { + if !strings.Contains(prompt, fragment) { + t.Fatalf("expected visual annotation prompt to contain %q, got %q", fragment, prompt) + } + } +} diff --git a/server/internal/domain/design/project.go b/server/internal/domain/design/project.go index 126d7b2..147a953 100644 --- a/server/internal/domain/design/project.go +++ b/server/internal/domain/design/project.go @@ -158,13 +158,14 @@ type Node struct { } type NodeActionRequest struct { - Action string - Prompt string - ImageModel string - ImageSize string - Mask string - Selection string - Options map[string]string + Action string + Prompt string + ImageModel string + ImageSize string + Mask string + Selection string + AnnotationImage string + Options map[string]string } type Connection struct { diff --git a/server/internal/logic/run_node_action_async_logic.go b/server/internal/logic/run_node_action_async_logic.go index b7a3bb7..e9bd0c1 100644 --- a/server/internal/logic/run_node_action_async_logic.go +++ b/server/internal/logic/run_node_action_async_logic.go @@ -29,13 +29,14 @@ func NewRunNodeActionAsyncLogic(ctx context.Context, svcCtx *svc.ServiceContext) func (l *RunNodeActionAsyncLogic) RunNodeActionAsync(req *types.NodeActionRequest) (resp *types.ProjectResponse, err error) { project, err := l.svcCtx.DesignService.RunNodeActionAsync(l.ctx, req.Id, req.NodeId, design.NodeActionRequest{ - Action: req.Action, - Prompt: req.Prompt, - ImageModel: req.ImageModel, - ImageSize: req.ImageSize, - Mask: req.Mask, - Selection: req.Selection, - Options: req.Options, + Action: req.Action, + Prompt: req.Prompt, + ImageModel: req.ImageModel, + ImageSize: req.ImageSize, + Mask: req.Mask, + Selection: req.Selection, + AnnotationImage: req.AnnotationImage, + Options: req.Options, }) if err != nil { return nil, err diff --git a/server/internal/types/types.go b/server/internal/types/types.go index 3708542..9add9f0 100644 --- a/server/internal/types/types.go +++ b/server/internal/types/types.go @@ -701,15 +701,16 @@ type MockupSurfaceResponse struct { } type NodeActionRequest struct { - Id string `path:"id"` - NodeId string `path:"nodeId"` - Action string `json:"action"` - Prompt string `json:"prompt,optional"` - ImageModel string `json:"imageModel,optional"` - ImageSize string `json:"imageSize,optional"` - Mask string `json:"mask,optional"` - Selection string `json:"selection,optional"` - Options map[string]string `json:"options,optional"` + Id string `path:"id"` + NodeId string `path:"nodeId"` + Action string `json:"action"` + Prompt string `json:"prompt,optional"` + ImageModel string `json:"imageModel,optional"` + ImageSize string `json:"imageSize,optional"` + Mask string `json:"mask,optional"` + Selection string `json:"selection,optional"` + AnnotationImage string `json:"annotationImage,optional"` + Options map[string]string `json:"options,optional"` } type NormalizedBox struct {