From 09b961fcc4ed9d7adb17a0a3591c2e26d4ece094 Mon Sep 17 00:00:00 2001 From: liangxu Date: Wed, 15 Jul 2026 12:08:43 +0800 Subject: [PATCH] feat(canvas): attach uploaded image names to model prompts Add modelImageReferences to preprocess agent contents, uploading local image references and surfacing their names in the prompt so the model can cite them. Wire the preparer into the canvas agent submission path, pass imageName through the floating composer references, and relax the prompt directive parser to accept names containing arbitrary characters. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ImageGeneratorFloatingComposer/index.tsx | 3 +- frontend/src/ui/lib/modelImageReferences.ts | 177 ++++++++++++++++++ frontend/src/ui/lib/promptImageReferences.ts | 2 +- .../src/ui/pages/CanvasWorkspace/index.tsx | 5 +- frontend/tests/modelImageReferences.test.mjs | 91 +++++++++ frontend/tests/promptImageReferences.test.mjs | 39 ++++ 6 files changed, 314 insertions(+), 3 deletions(-) create mode 100644 frontend/src/ui/lib/modelImageReferences.ts create mode 100644 frontend/tests/modelImageReferences.test.mjs create mode 100644 frontend/tests/promptImageReferences.test.mjs diff --git a/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx b/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx index 0517e9f..e75790e 100644 --- a/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx +++ b/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx @@ -161,7 +161,8 @@ export function ImageGeneratorFloatingComposer({ .filter((reference) => reference.publicUrl.trim()) .map((reference): AgentContent => ({ type: "image", - imageUrl: reference.publicUrl + imageUrl: reference.publicUrl, + imageName: reference.name })); const contents = [...referenceContents, ...promptContents]; onSubmit(contents, resetComposer); diff --git a/frontend/src/ui/lib/modelImageReferences.ts b/frontend/src/ui/lib/modelImageReferences.ts new file mode 100644 index 0000000..51130b1 --- /dev/null +++ b/frontend/src/ui/lib/modelImageReferences.ts @@ -0,0 +1,177 @@ +import type { AgentContent } from "@/domain/design"; + +type ImageContent = Extract; + +type ModelImageReferenceDependencies = { + uploadImage: (file: File) => Promise; + fetchImage?: (url: string) => Promise; + rasterizeSvg?: (source: Blob, fileName: string) => Promise; +}; + +type ImageFormatHint = "svg" | "raster" | "unknown"; + +const modelReferenceMaxEdge = 2048; +const svgMimeType = "image/svg+xml"; +const svgExtensionPattern = /\.svg(?:$|[?#])/i; +const rasterExtensionPattern = /\.(?:png|jpe?g|gif|webp)(?:$|[?#])/i; +const rasterDataUrlPattern = /^data:image\/(?:png|jpeg|gif|webp)[;,]/i; + +export function createModelAgentContentPreparer(dependencies: ModelImageReferenceDependencies) { + const preparedUrls = new Map>(); + const fetchImage = dependencies.fetchImage ?? ((url: string) => fetch(url)); + const rasterizeSvg = dependencies.rasterizeSvg ?? rasterizeSvgToTransparentWebp; + + return async (contents: AgentContent[]) => + Promise.all( + contents.map(async (content): Promise => { + if (content.type !== "image" || !content.imageUrl.trim()) return content; + + const hint = imageFormatHint(content); + if (hint === "raster") return content; + + const imageUrl = content.imageUrl.trim(); + let preparedUrl = preparedUrls.get(imageUrl); + if (!preparedUrl) { + preparedUrl = prepareModelImageUrl(content, hint, fetchImage, rasterizeSvg, dependencies.uploadImage).catch((error) => { + preparedUrls.delete(imageUrl); + throw error; + }); + preparedUrls.set(imageUrl, preparedUrl); + } + + const nextImageUrl = await preparedUrl; + return nextImageUrl === content.imageUrl ? content : { ...content, imageUrl: nextImageUrl }; + }) + ); +} + +export async function rasterizeSvgToTransparentWebp(source: Blob, fileName: string) { + const svgSource = source.type.toLowerCase() === svgMimeType ? source : new Blob([source], { type: svgMimeType }); + const svgText = await svgSource.text(); + const documentNode = new DOMParser().parseFromString(svgText, svgMimeType); + if (documentNode.querySelector("parsererror") || documentNode.documentElement.localName.toLowerCase() !== "svg") { + throw new Error("The selected SVG reference is invalid."); + } + + const objectUrl = URL.createObjectURL(svgSource); + try { + const image = await loadSvgImage(objectUrl); + const aspectRatio = svgAspectRatio(documentNode.documentElement, image); + const width = aspectRatio >= 1 ? modelReferenceMaxEdge : Math.max(1, Math.round(modelReferenceMaxEdge * aspectRatio)); + const height = aspectRatio >= 1 ? Math.max(1, Math.round(modelReferenceMaxEdge / aspectRatio)) : modelReferenceMaxEdge; + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + if (!context) throw new Error("Unable to create a canvas for the SVG reference."); + context.clearRect(0, 0, width, height); + context.drawImage(image, 0, 0, width, height); + + const webp = await canvasToBlob(canvas, "image/webp", 0.95); + if (webp.type.toLowerCase() !== "image/webp") { + throw new Error("This browser cannot encode WebP reference images."); + } + return new File([webp], modelWebpFileName(fileName), { type: "image/webp", lastModified: Date.now() }); + } finally { + URL.revokeObjectURL(objectUrl); + } +} + +async function prepareModelImageUrl( + content: ImageContent, + hint: ImageFormatHint, + fetchImage: (url: string) => Promise, + rasterizeSvg: (source: Blob, fileName: string) => Promise, + uploadImage: (file: File) => Promise +) { + let response: Response; + try { + response = await fetchImage(content.imageUrl.trim()); + } catch (error) { + if (hint === "unknown") return content.imageUrl; + throw error; + } + if (!response.ok) { + if (hint === "unknown") return content.imageUrl; + throw new Error(`Unable to load SVG reference (${response.status}).`); + } + + const source = await response.blob(); + if (hint === "unknown" && !(await blobIsSvg(source, response.headers.get("content-type")))) { + return content.imageUrl; + } + + const webp = await rasterizeSvg(source, referenceFileName(content)); + const uploadedUrl = (await uploadImage(webp)).trim(); + if (!uploadedUrl) throw new Error("The converted WebP reference did not return an asset URL."); + return uploadedUrl; +} + +function imageFormatHint(content: ImageContent): ImageFormatHint { + const name = content.imageName?.trim() || ""; + const url = content.imageUrl.trim(); + if (svgExtensionPattern.test(name) || svgExtensionPattern.test(url) || /^data:image\/svg\+xml[;,]/i.test(url)) return "svg"; + if (rasterExtensionPattern.test(name) || rasterExtensionPattern.test(url) || rasterDataUrlPattern.test(url)) return "raster"; + return "unknown"; +} + +async function blobIsSvg(source: Blob, contentType: string | null) { + const normalizedType = (contentType || source.type).split(";", 1)[0]?.trim().toLowerCase(); + if (normalizedType === svgMimeType) return true; + if (normalizedType?.startsWith("image/") && normalizedType !== svgMimeType) return false; + const prefix = await source.slice(0, 4096).text(); + return /)/i.test(prefix); +} + +function referenceFileName(content: ImageContent) { + const preferred = content.imageName?.trim(); + if (preferred) return preferred; + try { + const path = new URL(content.imageUrl, "http://canvas.local").pathname; + return decodeURIComponent(path.split("/").pop() || "reference.svg"); + } catch { + return "reference.svg"; + } +} + +function modelWebpFileName(fileName: string) { + const safeName = fileName.replace(/[\\/]/g, "_").trim() || "reference.svg"; + const baseName = safeName.replace(/\.svg$/i, ""); + return `${baseName}.model.webp`; +} + +function svgAspectRatio(root: Element, image: HTMLImageElement) { + const viewBox = root + .getAttribute("viewBox") + ?.trim() + .split(/[\s,]+/) + .map(Number); + if (viewBox?.length === 4 && Number.isFinite(viewBox[2]) && Number.isFinite(viewBox[3]) && viewBox[2] > 0 && viewBox[3] > 0) { + return viewBox[2] / viewBox[3]; + } + + const width = Number.parseFloat(root.getAttribute("width") || ""); + const height = Number.parseFloat(root.getAttribute("height") || ""); + if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) return width / height; + if (image.naturalWidth > 0 && image.naturalHeight > 0) return image.naturalWidth / image.naturalHeight; + return 1; +} + +function loadSvgImage(source: string) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.decoding = "async"; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error("Unable to render the selected SVG reference.")); + image.src = source; + }); +} + +function canvasToBlob(canvas: HTMLCanvasElement, type: string, quality: number) { + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (blob) resolve(blob); + else reject(new Error("Unable to encode the SVG reference as WebP.")); + }, type, quality); + }); +} diff --git a/frontend/src/ui/lib/promptImageReferences.ts b/frontend/src/ui/lib/promptImageReferences.ts index 13d654c..a30cf86 100644 --- a/frontend/src/ui/lib/promptImageReferences.ts +++ b/frontend/src/ui/lib/promptImageReferences.ts @@ -24,7 +24,7 @@ type ReferenceMatch = { }; const imageTokenPattern = /\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]/g; -const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[（(]([^）)\n]+)[）)])?\s*[:：]\s*(https?:\/\/[^\s\]]+)/gi; +const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[（(]([^\n]*?)[）)])?\s*[:：]\s*(https?:\/\/[^\s\]]+)/gi; export function parsePromptImageReferences(content: string): PromptImageReferencePart[] { const matches = collectReferenceMatches(content); diff --git a/frontend/src/ui/pages/CanvasWorkspace/index.tsx b/frontend/src/ui/pages/CanvasWorkspace/index.tsx index 7bb013e..0c2cb90 100644 --- a/frontend/src/ui/pages/CanvasWorkspace/index.tsx +++ b/frontend/src/ui/pages/CanvasWorkspace/index.tsx @@ -39,6 +39,7 @@ import { SharePopover } from "@/ui/components/SharePopover"; import { autoImageAdjustments, defaultImageAdjustments, parseImageAdjustments, serializeImageAdjustments } from "@/ui/lib/imageAdjustments"; import { friendlyProjectFailureMessage } from "@/ui/lib/friendlyAgentErrors"; import { canvasImageUrl } from "@/ui/lib/imageDelivery"; +import { createModelAgentContentPreparer } from "@/ui/lib/modelImageReferences"; import { visiblePromptText } from "@/ui/lib/promptImageReferences"; import { isImageGeneratorThreadMessage, visibleAgentMessages } from "./canvas/agentMessageFilters"; import { CanvasColorPopover } from "./canvas/CanvasColorPopover"; @@ -90,6 +91,7 @@ const defaultCanvasBackgroundColor = "#F7F7F4"; const canvasBackgroundSwatches = ["#F5F5F5", "#000000", "#FFFFFF", "#00F016", "#A855F7", "#E9D5FF"]; const canvasSidePanelWidth = 268; const newAgentConversationId = "__new_agent_conversation__"; +const prepareModelAgentContents = createModelAgentContentPreparer({ uploadImage: designGateway.uploadImage }); const canvasWheelGuardSelector = [ ".ui-popover-content", ".ui-dropdown-menu-content", @@ -2769,10 +2771,11 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess if (dirty) { await saveCanvasSnapshot({ quiet: true }); } + const modelContents = await prepareModelAgentContents(contents); startAgentSocket( project.id, { - messages: [{ role: "user", contents }], + messages: [{ role: "user", contents: modelContents }], mode: source === "image-generator" ? "image-generator" : activeMode === "image" ? "image" : "design", threadId, threadIdType: 9, diff --git a/frontend/tests/modelImageReferences.test.mjs b/frontend/tests/modelImageReferences.test.mjs new file mode 100644 index 0000000..b43acc0 --- /dev/null +++ b/frontend/tests/modelImageReferences.test.mjs @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createModelAgentContentPreparer } from "../src/ui/lib/modelImageReferences.ts"; + +test("replaces only the submitted URL for named SVG references", async () => { + const calls = { fetch: 0, rasterize: 0, upload: 0 }; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => { + calls.fetch += 1; + return new Response('', { headers: { "content-type": "image/svg+xml" } }); + }, + rasterizeSvg: async (_source, fileName) => { + calls.rasterize += 1; + assert.equal(fileName, "drawing (DXF).svg"); + return { name: "drawing (DXF).model.webp", type: "image/webp" }; + }, + uploadImage: async (file) => { + calls.upload += 1; + assert.equal(file.type, "image/webp"); + return "https://assets.example.com/model/drawing.webp"; + } + }); + const contents = [ + { type: "image", imageUrl: "https://assets.example.com/source/opaque-id", imageName: "drawing (DXF).svg" }, + { type: "text", text: "Use this plan", textSource: "input" } + ]; + + const prepared = await prepare(contents); + + assert.equal(prepared[0].imageUrl, "https://assets.example.com/model/drawing.webp"); + assert.equal(prepared[0].imageName, "drawing (DXF).svg"); + assert.equal(contents[0].imageUrl, "https://assets.example.com/source/opaque-id"); + assert.deepEqual(calls, { fetch: 1, rasterize: 1, upload: 1 }); +}); + +test("keeps supported raster references unchanged without downloading them", async () => { + let called = false; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => { + called = true; + throw new Error("unexpected fetch"); + }, + rasterizeSvg: async () => { + throw new Error("unexpected rasterization"); + }, + uploadImage: async () => { + throw new Error("unexpected upload"); + } + }); + const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.webp", imageName: "reference.webp" }]; + + const prepared = await prepare(contents); + + assert.strictEqual(prepared[0], contents[0]); + assert.equal(called, false); +}); + +test("detects SVG references from response MIME when names have no extension", async () => { + let rasterized = 0; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => new Response("", { headers: { "content-type": "image/svg+xml; charset=utf-8" } }), + rasterizeSvg: async () => { + rasterized += 1; + return { name: "reference.model.webp", type: "image/webp" }; + }, + uploadImage: async () => "https://assets.example.com/reference.model.webp" + }); + + const prepared = await prepare([{ type: "image", imageUrl: "https://assets.example.com/opaque-id" }]); + + assert.equal(prepared[0].imageUrl, "https://assets.example.com/reference.model.webp"); + assert.equal(rasterized, 1); +}); + +test("caches uploaded WebP derivatives by source URL", async () => { + let uploads = 0; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => new Response("", { headers: { "content-type": "image/svg+xml" } }), + rasterizeSvg: async () => ({ name: "reference.model.webp", type: "image/webp" }), + uploadImage: async () => { + uploads += 1; + return "https://assets.example.com/reference.model.webp"; + } + }); + const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.svg", imageName: "reference.svg" }]; + + await Promise.all([prepare(contents), prepare(contents)]); + + assert.equal(uploads, 1); +}); diff --git a/frontend/tests/promptImageReferences.test.mjs b/frontend/tests/promptImageReferences.test.mjs new file mode 100644 index 0000000..b9dfea7 --- /dev/null +++ b/frontend/tests/promptImageReferences.test.mjs @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { parsePromptImageReferences } from "../src/ui/lib/promptImageReferences.ts"; + +test("parses CAD-derived SVG reference directives with nested parentheses", () => { + const url = "http://localhost:19000/canvas-assets/drawing.svg"; + const parts = parsePromptImageReferences(`Reference image (06-5070 R3 2026-05-05(1) (DXF).svg): ${url}\nMake it pink`); + + assert.deepEqual(parts, [ + { + type: "reference", + reference: { + id: parts[0]?.type === "reference" ? parts[0].reference.id : "", + index: undefined, + name: "06-5070 R3 2026-05-05(1) (DXF).svg", + url + } + }, + { type: "text", text: "\nMake it pink" } + ]); +}); + +test("keeps Chinese CAD reference directives on the attachment path", () => { + const url = "https://assets.example.com/plan.svg"; + const parts = parsePromptImageReferences(`参考图 1（花语江南 (DWG).svg）：${url}\n继续设计`); + + assert.equal(parts[0]?.type, "reference"); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.index : undefined, 1); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "花语江南 (DWG).svg"); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.url : "", url); +}); + +test("continues to parse raster reference directives", () => { + const parts = parsePromptImageReferences("Reference image 2 (sample.webp): https://assets.example.com/sample.webp"); + + assert.equal(parts[0]?.type, "reference"); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "sample.webp"); +});