diff --git a/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx b/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx index 0517e9f..e75790e 100644 --- a/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx +++ b/frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx @@ -161,7 +161,8 @@ export function ImageGeneratorFloatingComposer({ .filter((reference) => reference.publicUrl.trim()) .map((reference): AgentContent => ({ type: "image", - imageUrl: reference.publicUrl + imageUrl: reference.publicUrl, + imageName: reference.name })); const contents = [...referenceContents, ...promptContents]; onSubmit(contents, resetComposer); diff --git a/frontend/src/ui/lib/modelImageReferences.ts b/frontend/src/ui/lib/modelImageReferences.ts new file mode 100644 index 0000000..51130b1 --- /dev/null +++ b/frontend/src/ui/lib/modelImageReferences.ts @@ -0,0 +1,177 @@ +import type { AgentContent } from "@/domain/design"; + +type ImageContent = Extract; + +type ModelImageReferenceDependencies = { + uploadImage: (file: File) => Promise; + fetchImage?: (url: string) => Promise; + rasterizeSvg?: (source: Blob, fileName: string) => Promise; +}; + +type ImageFormatHint = "svg" | "raster" | "unknown"; + +const modelReferenceMaxEdge = 2048; +const svgMimeType = "image/svg+xml"; +const svgExtensionPattern = /\.svg(?:$|[?#])/i; +const rasterExtensionPattern = /\.(?:png|jpe?g|gif|webp)(?:$|[?#])/i; +const rasterDataUrlPattern = /^data:image\/(?:png|jpeg|gif|webp)[;,]/i; + +export function createModelAgentContentPreparer(dependencies: ModelImageReferenceDependencies) { + const preparedUrls = new Map>(); + const fetchImage = dependencies.fetchImage ?? ((url: string) => fetch(url)); + const rasterizeSvg = dependencies.rasterizeSvg ?? rasterizeSvgToTransparentWebp; + + return async (contents: AgentContent[]) => + Promise.all( + contents.map(async (content): Promise => { + if (content.type !== "image" || !content.imageUrl.trim()) return content; + + const hint = imageFormatHint(content); + if (hint === "raster") return content; + + const imageUrl = content.imageUrl.trim(); + let preparedUrl = preparedUrls.get(imageUrl); + if (!preparedUrl) { + preparedUrl = prepareModelImageUrl(content, hint, fetchImage, rasterizeSvg, dependencies.uploadImage).catch((error) => { + preparedUrls.delete(imageUrl); + throw error; + }); + preparedUrls.set(imageUrl, preparedUrl); + } + + const nextImageUrl = await preparedUrl; + return nextImageUrl === content.imageUrl ? content : { ...content, imageUrl: nextImageUrl }; + }) + ); +} + +export async function rasterizeSvgToTransparentWebp(source: Blob, fileName: string) { + const svgSource = source.type.toLowerCase() === svgMimeType ? source : new Blob([source], { type: svgMimeType }); + const svgText = await svgSource.text(); + const documentNode = new DOMParser().parseFromString(svgText, svgMimeType); + if (documentNode.querySelector("parsererror") || documentNode.documentElement.localName.toLowerCase() !== "svg") { + throw new Error("The selected SVG reference is invalid."); + } + + const objectUrl = URL.createObjectURL(svgSource); + try { + const image = await loadSvgImage(objectUrl); + const aspectRatio = svgAspectRatio(documentNode.documentElement, image); + const width = aspectRatio >= 1 ? modelReferenceMaxEdge : Math.max(1, Math.round(modelReferenceMaxEdge * aspectRatio)); + const height = aspectRatio >= 1 ? Math.max(1, Math.round(modelReferenceMaxEdge / aspectRatio)) : modelReferenceMaxEdge; + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + if (!context) throw new Error("Unable to create a canvas for the SVG reference."); + context.clearRect(0, 0, width, height); + context.drawImage(image, 0, 0, width, height); + + const webp = await canvasToBlob(canvas, "image/webp", 0.95); + if (webp.type.toLowerCase() !== "image/webp") { + throw new Error("This browser cannot encode WebP reference images."); + } + return new File([webp], modelWebpFileName(fileName), { type: "image/webp", lastModified: Date.now() }); + } finally { + URL.revokeObjectURL(objectUrl); + } +} + +async function prepareModelImageUrl( + content: ImageContent, + hint: ImageFormatHint, + fetchImage: (url: string) => Promise, + rasterizeSvg: (source: Blob, fileName: string) => Promise, + uploadImage: (file: File) => Promise +) { + let response: Response; + try { + response = await fetchImage(content.imageUrl.trim()); + } catch (error) { + if (hint === "unknown") return content.imageUrl; + throw error; + } + if (!response.ok) { + if (hint === "unknown") return content.imageUrl; + throw new Error(`Unable to load SVG reference (${response.status}).`); + } + + const source = await response.blob(); + if (hint === "unknown" && !(await blobIsSvg(source, response.headers.get("content-type")))) { + return content.imageUrl; + } + + const webp = await rasterizeSvg(source, referenceFileName(content)); + const uploadedUrl = (await uploadImage(webp)).trim(); + if (!uploadedUrl) throw new Error("The converted WebP reference did not return an asset URL."); + return uploadedUrl; +} + +function imageFormatHint(content: ImageContent): ImageFormatHint { + const name = content.imageName?.trim() || ""; + const url = content.imageUrl.trim(); + if (svgExtensionPattern.test(name) || svgExtensionPattern.test(url) || /^data:image\/svg\+xml[;,]/i.test(url)) return "svg"; + if (rasterExtensionPattern.test(name) || rasterExtensionPattern.test(url) || rasterDataUrlPattern.test(url)) return "raster"; + return "unknown"; +} + +async function blobIsSvg(source: Blob, contentType: string | null) { + const normalizedType = (contentType || source.type).split(";", 1)[0]?.trim().toLowerCase(); + if (normalizedType === svgMimeType) return true; + if (normalizedType?.startsWith("image/") && normalizedType !== svgMimeType) return false; + const prefix = await source.slice(0, 4096).text(); + return /)/i.test(prefix); +} + +function referenceFileName(content: ImageContent) { + const preferred = content.imageName?.trim(); + if (preferred) return preferred; + try { + const path = new URL(content.imageUrl, "http://canvas.local").pathname; + return decodeURIComponent(path.split("/").pop() || "reference.svg"); + } catch { + return "reference.svg"; + } +} + +function modelWebpFileName(fileName: string) { + const safeName = fileName.replace(/[\\/]/g, "_").trim() || "reference.svg"; + const baseName = safeName.replace(/\.svg$/i, ""); + return `${baseName}.model.webp`; +} + +function svgAspectRatio(root: Element, image: HTMLImageElement) { + const viewBox = root + .getAttribute("viewBox") + ?.trim() + .split(/[\s,]+/) + .map(Number); + if (viewBox?.length === 4 && Number.isFinite(viewBox[2]) && Number.isFinite(viewBox[3]) && viewBox[2] > 0 && viewBox[3] > 0) { + return viewBox[2] / viewBox[3]; + } + + const width = Number.parseFloat(root.getAttribute("width") || ""); + const height = Number.parseFloat(root.getAttribute("height") || ""); + if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) return width / height; + if (image.naturalWidth > 0 && image.naturalHeight > 0) return image.naturalWidth / image.naturalHeight; + return 1; +} + +function loadSvgImage(source: string) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.decoding = "async"; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error("Unable to render the selected SVG reference.")); + image.src = source; + }); +} + +function canvasToBlob(canvas: HTMLCanvasElement, type: string, quality: number) { + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (blob) resolve(blob); + else reject(new Error("Unable to encode the SVG reference as WebP.")); + }, type, quality); + }); +} diff --git a/frontend/src/ui/lib/promptImageReferences.ts b/frontend/src/ui/lib/promptImageReferences.ts index 13d654c..a30cf86 100644 --- a/frontend/src/ui/lib/promptImageReferences.ts +++ b/frontend/src/ui/lib/promptImageReferences.ts @@ -24,7 +24,7 @@ type ReferenceMatch = { }; const imageTokenPattern = /\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]/g; -const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[((]([^))\n]+)[))])?\s*[::]\s*(https?:\/\/[^\s\]]+)/gi; +const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[((]([^\n]*?)[))])?\s*[::]\s*(https?:\/\/[^\s\]]+)/gi; export function parsePromptImageReferences(content: string): PromptImageReferencePart[] { const matches = collectReferenceMatches(content); diff --git a/frontend/src/ui/pages/CanvasWorkspace/index.tsx b/frontend/src/ui/pages/CanvasWorkspace/index.tsx index 7bb013e..0c2cb90 100644 --- a/frontend/src/ui/pages/CanvasWorkspace/index.tsx +++ b/frontend/src/ui/pages/CanvasWorkspace/index.tsx @@ -39,6 +39,7 @@ import { SharePopover } from "@/ui/components/SharePopover"; import { autoImageAdjustments, defaultImageAdjustments, parseImageAdjustments, serializeImageAdjustments } from "@/ui/lib/imageAdjustments"; import { friendlyProjectFailureMessage } from "@/ui/lib/friendlyAgentErrors"; import { canvasImageUrl } from "@/ui/lib/imageDelivery"; +import { createModelAgentContentPreparer } from "@/ui/lib/modelImageReferences"; import { visiblePromptText } from "@/ui/lib/promptImageReferences"; import { isImageGeneratorThreadMessage, visibleAgentMessages } from "./canvas/agentMessageFilters"; import { CanvasColorPopover } from "./canvas/CanvasColorPopover"; @@ -90,6 +91,7 @@ const defaultCanvasBackgroundColor = "#F7F7F4"; const canvasBackgroundSwatches = ["#F5F5F5", "#000000", "#FFFFFF", "#00F016", "#A855F7", "#E9D5FF"]; const canvasSidePanelWidth = 268; const newAgentConversationId = "__new_agent_conversation__"; +const prepareModelAgentContents = createModelAgentContentPreparer({ uploadImage: designGateway.uploadImage }); const canvasWheelGuardSelector = [ ".ui-popover-content", ".ui-dropdown-menu-content", @@ -2769,10 +2771,11 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess if (dirty) { await saveCanvasSnapshot({ quiet: true }); } + const modelContents = await prepareModelAgentContents(contents); startAgentSocket( project.id, { - messages: [{ role: "user", contents }], + messages: [{ role: "user", contents: modelContents }], mode: source === "image-generator" ? "image-generator" : activeMode === "image" ? "image" : "design", threadId, threadIdType: 9, diff --git a/frontend/tests/modelImageReferences.test.mjs b/frontend/tests/modelImageReferences.test.mjs new file mode 100644 index 0000000..b43acc0 --- /dev/null +++ b/frontend/tests/modelImageReferences.test.mjs @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createModelAgentContentPreparer } from "../src/ui/lib/modelImageReferences.ts"; + +test("replaces only the submitted URL for named SVG references", async () => { + const calls = { fetch: 0, rasterize: 0, upload: 0 }; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => { + calls.fetch += 1; + return new Response('', { headers: { "content-type": "image/svg+xml" } }); + }, + rasterizeSvg: async (_source, fileName) => { + calls.rasterize += 1; + assert.equal(fileName, "drawing (DXF).svg"); + return { name: "drawing (DXF).model.webp", type: "image/webp" }; + }, + uploadImage: async (file) => { + calls.upload += 1; + assert.equal(file.type, "image/webp"); + return "https://assets.example.com/model/drawing.webp"; + } + }); + const contents = [ + { type: "image", imageUrl: "https://assets.example.com/source/opaque-id", imageName: "drawing (DXF).svg" }, + { type: "text", text: "Use this plan", textSource: "input" } + ]; + + const prepared = await prepare(contents); + + assert.equal(prepared[0].imageUrl, "https://assets.example.com/model/drawing.webp"); + assert.equal(prepared[0].imageName, "drawing (DXF).svg"); + assert.equal(contents[0].imageUrl, "https://assets.example.com/source/opaque-id"); + assert.deepEqual(calls, { fetch: 1, rasterize: 1, upload: 1 }); +}); + +test("keeps supported raster references unchanged without downloading them", async () => { + let called = false; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => { + called = true; + throw new Error("unexpected fetch"); + }, + rasterizeSvg: async () => { + throw new Error("unexpected rasterization"); + }, + uploadImage: async () => { + throw new Error("unexpected upload"); + } + }); + const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.webp", imageName: "reference.webp" }]; + + const prepared = await prepare(contents); + + assert.strictEqual(prepared[0], contents[0]); + assert.equal(called, false); +}); + +test("detects SVG references from response MIME when names have no extension", async () => { + let rasterized = 0; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => new Response("", { headers: { "content-type": "image/svg+xml; charset=utf-8" } }), + rasterizeSvg: async () => { + rasterized += 1; + return { name: "reference.model.webp", type: "image/webp" }; + }, + uploadImage: async () => "https://assets.example.com/reference.model.webp" + }); + + const prepared = await prepare([{ type: "image", imageUrl: "https://assets.example.com/opaque-id" }]); + + assert.equal(prepared[0].imageUrl, "https://assets.example.com/reference.model.webp"); + assert.equal(rasterized, 1); +}); + +test("caches uploaded WebP derivatives by source URL", async () => { + let uploads = 0; + const prepare = createModelAgentContentPreparer({ + fetchImage: async () => new Response("", { headers: { "content-type": "image/svg+xml" } }), + rasterizeSvg: async () => ({ name: "reference.model.webp", type: "image/webp" }), + uploadImage: async () => { + uploads += 1; + return "https://assets.example.com/reference.model.webp"; + } + }); + const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.svg", imageName: "reference.svg" }]; + + await Promise.all([prepare(contents), prepare(contents)]); + + assert.equal(uploads, 1); +}); diff --git a/frontend/tests/promptImageReferences.test.mjs b/frontend/tests/promptImageReferences.test.mjs new file mode 100644 index 0000000..b9dfea7 --- /dev/null +++ b/frontend/tests/promptImageReferences.test.mjs @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { parsePromptImageReferences } from "../src/ui/lib/promptImageReferences.ts"; + +test("parses CAD-derived SVG reference directives with nested parentheses", () => { + const url = "http://localhost:19000/canvas-assets/drawing.svg"; + const parts = parsePromptImageReferences(`Reference image (06-5070 R3 2026-05-05(1) (DXF).svg): ${url}\nMake it pink`); + + assert.deepEqual(parts, [ + { + type: "reference", + reference: { + id: parts[0]?.type === "reference" ? parts[0].reference.id : "", + index: undefined, + name: "06-5070 R3 2026-05-05(1) (DXF).svg", + url + } + }, + { type: "text", text: "\nMake it pink" } + ]); +}); + +test("keeps Chinese CAD reference directives on the attachment path", () => { + const url = "https://assets.example.com/plan.svg"; + const parts = parsePromptImageReferences(`参考图 1(花语江南 (DWG).svg):${url}\n继续设计`); + + assert.equal(parts[0]?.type, "reference"); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.index : undefined, 1); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "花语江南 (DWG).svg"); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.url : "", url); +}); + +test("continues to parse raster reference directives", () => { + const parts = parsePromptImageReferences("Reference image 2 (sample.webp): https://assets.example.com/sample.webp"); + + assert.equal(parts[0]?.type, "reference"); + assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "sample.webp"); +});