root/moteva
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): add visual annotation edit action

Browse Source 
Add a VisualAnnotationPanel letting users mark regions on an image (brush,
circle, rectangle, cross) with per-mark edit instructions, composited into
an annotation guide image. The new "visual-annotation" node action sends
the unmarked source plus the annotated guide to the image model, editing
only numbered regions. Server validates the annotation image/instructions
and threads them through the generator task.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-07-11 10:03:59 +08:00
parent 114198f1d9
commit bb8946b79b
17 changed files with 1271 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/ui/pages/CanvasWorkspace/index.tsx
+68 -3
View File
@@ -32,6 +32,7 @@ import { AdjustPanel, CropOverlay, defaultToolbarVisibility, EraserOverlay, Expa
import { ImageTextEditPanel, type EditableTextLayer } from "@/ui/components/ImageTextEditPanel";
import { MultiSelectionOverlay, type MultiSelectionResizeHandle } from "@/ui/components/MultiSelectionOverlay";
import { TextStyleToolbar, type TextStylePatch } from "@/ui/components/TextStyleToolbar";
import { renderVisualAnnotationGuide, visualAnnotationPayload, VisualAnnotationPanel, type VisualAnnotationMark, type VisualAnnotationTool } from "@/ui/components/VisualAnnotationPanel";
import { loadMotevaFontCatalog } from "@/ui/components/TextStyleToolbar/fontCatalog";
import { WorkspaceTitle } from "@/ui/components/WorkspaceTitle";
import { SharePopover } from "@/ui/components/SharePopover";
@@ -115,7 +116,7 @@ type HomeImageModel = {
credits: number;
};
type ImageActionMode = "quick-edit" | "eraser" | "edit-text" | "multi-angle" | "move-object" | "mockup" | "adjust" | "expand" | "crop" | "flip-rotate" | null;
type ImageActionMode = "quick-edit" | "visual-annotation" | "eraser" | "edit-text" | "multi-angle" | "move-object" | "mockup" | "adjust" | "expand" | "crop" | "flip-rotate" | null;
const defaultCropSelection: NormalizedRect = { x: 0, y: 0, width: 1, height: 1 };
const imageModelBlueprints = [
{
@@ -630,6 +631,8 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
const [toolbarVisibility, setToolbarVisibility] = useState<ImageToolbarVisibility>(defaultToolbarVisibility);
const [imageActionMode, setImageActionMode] = useState<ImageActionMode>(null);
const [quickEditPrompt, setQuickEditPrompt] = useState("");
const [visualAnnotationMarks, setVisualAnnotationMarks] = useState<VisualAnnotationMark[]>([]);
const [visualAnnotationSubmitting, setVisualAnnotationSubmitting] = useState(false);
const [eraserBrushSize, setEraserBrushSize] = useState(42);
const [eraserBrushColor, setEraserBrushColor] = useState("#5E6CFF");
const [eraserBrushOpacity, setEraserBrushOpacity] = useState(72);
@@ -663,7 +666,7 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
zoom: 50,
prompt: ""
});
const canvasActionWorkActive = canvasActionGenerating || textEditLoading || nodes.some(isCanvasActionWorkingNode);
const canvasActionWorkActive = canvasActionGenerating || textEditLoading || visualAnnotationSubmitting || nodes.some(isCanvasActionWorkingNode);
const agentTimelineActive = agentGenerating || Boolean(project && !projectHasCanvasActionGeneration(project) && nodes.some(isAgentVisibleGeneratingNode));
const mockupPreparing = Boolean(mockupPendingTargetId);
const anyGenerating = agentGenerating || imageGeneratorGenerating || canvasActionWorkActive || mockupPreparing;
@@ -1147,6 +1150,8 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
useEffect(() => {
setImageActionMode(null);
setQuickEditPrompt("");
setVisualAnnotationMarks([]);
setVisualAnnotationSubmitting(false);
setEraserMarks([]);
setTextEditItems([]);
setTextEditLoading(false);
@@ -2841,6 +2846,7 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
applyProject(result.project);
startProjectEvents(result.project.id, result.project.lastThreadId, "canvas-action");
setImageActionMode(null);
setVisualAnnotationMarks([]);
setEraserMarks([]);
setTextEditItems([]);
setTextEditLoading(false);
@@ -3086,6 +3092,31 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
void runImageNodeAction("quick-edit", { prompt: quickEditPrompt.trim() });
};
const applyVisualAnnotation = async () => {
const targetNode = selectedImageNode;
if (!targetNode || visualAnnotationSubmitting || visualAnnotationMarks.length === 0 || visualAnnotationMarks.some((mark) => !mark.instruction.trim())) return;
setVisualAnnotationSubmitting(true);
setError("");
try {
const source = await imageNodeRasterBlob(targetNode, "image/png");
const guide = await renderVisualAnnotationGuide(source, visualAnnotationMarks);
const guideFile = new File([guide], `visual-annotation-${Date.now()}.png`, { type: "image/png" });
const annotationImage = await designGateway.uploadImage(guideFile);
const instructions = visualAnnotationMarks
.map((mark) => `${mark.index}. ${visualAnnotationToolLabel(mark.tool, t)}: ${mark.instruction.trim()}`)
.join("\n");
await runImageNodeAction("visual-annotation", {
prompt: t("visualAnnotationPrompt", { instructions }),
annotationImage,
selection: JSON.stringify(visualAnnotationPayload(visualAnnotationMarks))
});
} catch (err) {
setError(toUserMessageText(err, "generateError"));
} finally {
setVisualAnnotationSubmitting(false);
}
};
const openQuickEditPanel = () => {
if (!selectedImageNode) return;
const stageWidth = canvasStageSize.width;
@@ -3743,7 +3774,7 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
<VectorizedImageToolbar bounds={selectedNodeBounds} leftInset={floatingToolbarLeftInset} onDownload={() => exportImageNode(selectedImageNode, "svg")} />
) : (
<>
{imageActionMode !== "quick-edit" && imageActionMode !== "eraser" && imageActionMode !== "edit-text" && imageActionMode !== "expand" && imageActionMode !== "crop" && imageActionMode !== "flip-rotate" && selectedImageNode.status !== "text-extracting" && mockupPendingTargetId !== selectedImageNode.id && (
{imageActionMode !== "quick-edit" && imageActionMode !== "visual-annotation" && imageActionMode !== "eraser" && imageActionMode !== "edit-text" && imageActionMode !== "expand" && imageActionMode !== "crop" && imageActionMode !== "flip-rotate" && selectedImageNode.status !== "text-extracting" && mockupPendingTargetId !== selectedImageNode.id && (
<ImageNodeToolbar
bounds={selectedNodeBounds}
leftInset={floatingToolbarLeftInset}
@@ -3751,6 +3782,10 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
visibility={toolbarVisibility}
onVisibilityChange={setToolbarVisibility}
onQuickEdit={openQuickEditPanel}
onVisualAnnotation={() => {
setVisualAnnotationMarks([]);
setImageActionMode("visual-annotation");
}}
onUpscale={(scale) =>
void runImageNodeAction("upscale", {
prompt: t("upscalePrompt", { scale }),
@@ -3794,6 +3829,26 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
onSubmit={applyQuickEdit}
/>
)}
{imageActionMode === "visual-annotation" && (
<VisualAnnotationPanel
bounds={selectedNodeBounds}
stageWidth={canvasStageSize.width}
stageHeight={canvasStageSize.height}
marks={visualAnnotationMarks}
models={imageModels}
selectedModel={selectedCanvasModel}
disabled={selectedImageNodeBusy || visualAnnotationSubmitting}
submitting={visualAnnotationSubmitting}
onMarksChange={setVisualAnnotationMarks}
onModelChange={setSelectedCanvasModelId}
onCancel={() => {
if (visualAnnotationSubmitting) return;
setImageActionMode(null);
setVisualAnnotationMarks([]);
}}
onApply={() => void applyVisualAnnotation()}
/>
)}
{imageActionMode === "eraser" && (
<EraserOverlay
bounds={selectedNodeBounds}
@@ -5738,6 +5793,16 @@ function isImageActionBusy(node: CanvasNode) {
return node.status === "generating" || node.status === "uploading" || node.status === "text-extracting" || node.status === "mockup-modeling";
}
function visualAnnotationToolLabel(tool: VisualAnnotationTool, t: (key: string) => string) {
const keys: Record<VisualAnnotationTool, string> = {
brush: "visualAnnotationBrush",
circle: "visualAnnotationCircle",
rectangle: "visualAnnotationRectangle",
cross: "visualAnnotationCross"
};
return t(keys[tool]);
}
function isVectorizedImageNode(node: CanvasNode) {
return node.type === "image" && node.layerRole === "vectorized-image";
}