feat(canvas): attach uploaded image names to model prompts

Add modelImageReferences to preprocess agent contents, uploading local image references and surfacing their names in the prompt so the model can cite them. Wire the preparer into the canvas agent submission path, pass imageName through the floating composer references, and relax the prompt directive parser to accept names containing arbitrary characters. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-15 12:08:43 +08:00
parent 3336f23e6f
commit 09b961fcc4
6 changed files with 314 additions and 3 deletions
@@ -161,7 +161,8 @@ export function ImageGeneratorFloatingComposer({
      .filter((reference) => reference.publicUrl.trim())
      .map((reference): AgentContent => ({
        type: "image",
-        imageUrl: reference.publicUrl
+        imageUrl: reference.publicUrl,
+        imageName: reference.name
      }));
    const contents = [...referenceContents, ...promptContents];
    onSubmit(contents, resetComposer);
@@ -0,0 +1,177 @@
+import type { AgentContent } from "@/domain/design";
+
+type ImageContent = Extract<AgentContent, { type: "image" }>;
+
+type ModelImageReferenceDependencies = {
+  uploadImage: (file: File) => Promise<string>;
+  fetchImage?: (url: string) => Promise<Response>;
+  rasterizeSvg?: (source: Blob, fileName: string) => Promise<File>;
+};
+
+type ImageFormatHint = "svg" | "raster" | "unknown";
+
+const modelReferenceMaxEdge = 2048;
+const svgMimeType = "image/svg+xml";
+const svgExtensionPattern = /\.svg(?:$|[?#])/i;
+const rasterExtensionPattern = /\.(?:png|jpe?g|gif|webp)(?:$|[?#])/i;
+const rasterDataUrlPattern = /^data:image\/(?:png|jpeg|gif|webp)[;,]/i;
+
+export function createModelAgentContentPreparer(dependencies: ModelImageReferenceDependencies) {
+  const preparedUrls = new Map<string, Promise<string>>();
+  const fetchImage = dependencies.fetchImage ?? ((url: string) => fetch(url));
+  const rasterizeSvg = dependencies.rasterizeSvg ?? rasterizeSvgToTransparentWebp;
+
+  return async (contents: AgentContent[]) =>
+    Promise.all(
+      contents.map(async (content): Promise<AgentContent> => {
+        if (content.type !== "image" || !content.imageUrl.trim()) return content;
+
+        const hint = imageFormatHint(content);
+        if (hint === "raster") return content;
+
+        const imageUrl = content.imageUrl.trim();
+        let preparedUrl = preparedUrls.get(imageUrl);
+        if (!preparedUrl) {
+          preparedUrl = prepareModelImageUrl(content, hint, fetchImage, rasterizeSvg, dependencies.uploadImage).catch((error) => {
+            preparedUrls.delete(imageUrl);
+            throw error;
+          });
+          preparedUrls.set(imageUrl, preparedUrl);
+        }
+
+        const nextImageUrl = await preparedUrl;
+        return nextImageUrl === content.imageUrl ? content : { ...content, imageUrl: nextImageUrl };
+      })
+    );
+}
+
+export async function rasterizeSvgToTransparentWebp(source: Blob, fileName: string) {
+  const svgSource = source.type.toLowerCase() === svgMimeType ? source : new Blob([source], { type: svgMimeType });
+  const svgText = await svgSource.text();
+  const documentNode = new DOMParser().parseFromString(svgText, svgMimeType);
+  if (documentNode.querySelector("parsererror") || documentNode.documentElement.localName.toLowerCase() !== "svg") {
+    throw new Error("The selected SVG reference is invalid.");
+  }
+
+  const objectUrl = URL.createObjectURL(svgSource);
+  try {
+    const image = await loadSvgImage(objectUrl);
+    const aspectRatio = svgAspectRatio(documentNode.documentElement, image);
+    const width = aspectRatio >= 1 ? modelReferenceMaxEdge : Math.max(1, Math.round(modelReferenceMaxEdge * aspectRatio));
+    const height = aspectRatio >= 1 ? Math.max(1, Math.round(modelReferenceMaxEdge / aspectRatio)) : modelReferenceMaxEdge;
+    const canvas = document.createElement("canvas");
+    canvas.width = width;
+    canvas.height = height;
+    const context = canvas.getContext("2d");
+    if (!context) throw new Error("Unable to create a canvas for the SVG reference.");
+    context.clearRect(0, 0, width, height);
+    context.drawImage(image, 0, 0, width, height);
+
+    const webp = await canvasToBlob(canvas, "image/webp", 0.95);
+    if (webp.type.toLowerCase() !== "image/webp") {
+      throw new Error("This browser cannot encode WebP reference images.");
+    }
+    return new File([webp], modelWebpFileName(fileName), { type: "image/webp", lastModified: Date.now() });
+  } finally {
+    URL.revokeObjectURL(objectUrl);
+  }
+}
+
+async function prepareModelImageUrl(
+  content: ImageContent,
+  hint: ImageFormatHint,
+  fetchImage: (url: string) => Promise<Response>,
+  rasterizeSvg: (source: Blob, fileName: string) => Promise<File>,
+  uploadImage: (file: File) => Promise<string>
+) {
+  let response: Response;
+  try {
+    response = await fetchImage(content.imageUrl.trim());
+  } catch (error) {
+    if (hint === "unknown") return content.imageUrl;
+    throw error;
+  }
+  if (!response.ok) {
+    if (hint === "unknown") return content.imageUrl;
+    throw new Error(`Unable to load SVG reference (${response.status}).`);
+  }
+
+  const source = await response.blob();
+  if (hint === "unknown" && !(await blobIsSvg(source, response.headers.get("content-type")))) {
+    return content.imageUrl;
+  }
+
+  const webp = await rasterizeSvg(source, referenceFileName(content));
+  const uploadedUrl = (await uploadImage(webp)).trim();
+  if (!uploadedUrl) throw new Error("The converted WebP reference did not return an asset URL.");
+  return uploadedUrl;
+}
+
+function imageFormatHint(content: ImageContent): ImageFormatHint {
+  const name = content.imageName?.trim() || "";
+  const url = content.imageUrl.trim();
+  if (svgExtensionPattern.test(name) || svgExtensionPattern.test(url) || /^data:image\/svg\+xml[;,]/i.test(url)) return "svg";
+  if (rasterExtensionPattern.test(name) || rasterExtensionPattern.test(url) || rasterDataUrlPattern.test(url)) return "raster";
+  return "unknown";
+}
+
+async function blobIsSvg(source: Blob, contentType: string | null) {
+  const normalizedType = (contentType || source.type).split(";", 1)[0]?.trim().toLowerCase();
+  if (normalizedType === svgMimeType) return true;
+  if (normalizedType?.startsWith("image/") && normalizedType !== svgMimeType) return false;
+  const prefix = await source.slice(0, 4096).text();
+  return /<svg(?:\s|>)/i.test(prefix);
+}
+
+function referenceFileName(content: ImageContent) {
+  const preferred = content.imageName?.trim();
+  if (preferred) return preferred;
+  try {
+    const path = new URL(content.imageUrl, "http://canvas.local").pathname;
+    return decodeURIComponent(path.split("/").pop() || "reference.svg");
+  } catch {
+    return "reference.svg";
+  }
+}
+
+function modelWebpFileName(fileName: string) {
+  const safeName = fileName.replace(/[\\/]/g, "_").trim() || "reference.svg";
+  const baseName = safeName.replace(/\.svg$/i, "");
+  return `${baseName}.model.webp`;
+}
+
+function svgAspectRatio(root: Element, image: HTMLImageElement) {
+  const viewBox = root
+    .getAttribute("viewBox")
+    ?.trim()
+    .split(/[\s,]+/)
+    .map(Number);
+  if (viewBox?.length === 4 && Number.isFinite(viewBox[2]) && Number.isFinite(viewBox[3]) && viewBox[2] > 0 && viewBox[3] > 0) {
+    return viewBox[2] / viewBox[3];
+  }
+
+  const width = Number.parseFloat(root.getAttribute("width") || "");
+  const height = Number.parseFloat(root.getAttribute("height") || "");
+  if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) return width / height;
+  if (image.naturalWidth > 0 && image.naturalHeight > 0) return image.naturalWidth / image.naturalHeight;
+  return 1;
+}
+
+function loadSvgImage(source: string) {
+  return new Promise<HTMLImageElement>((resolve, reject) => {
+    const image = new Image();
+    image.decoding = "async";
+    image.onload = () => resolve(image);
+    image.onerror = () => reject(new Error("Unable to render the selected SVG reference."));
+    image.src = source;
+  });
+}
+
+function canvasToBlob(canvas: HTMLCanvasElement, type: string, quality: number) {
+  return new Promise<Blob>((resolve, reject) => {
+    canvas.toBlob((blob) => {
+      if (blob) resolve(blob);
+      else reject(new Error("Unable to encode the SVG reference as WebP."));
+    }, type, quality);
+  });
+}
@@ -24,7 +24,7 @@ type ReferenceMatch = {
 };

 const imageTokenPattern = /\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]/g;
-const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[（(]([^）)\n]+)[）)])?\s*[:：]\s*(https?:\/\/[^\s\]]+)/gi;
+const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[（(]([^\n]*?)[）)])?\s*[:：]\s*(https?:\/\/[^\s\]]+)/gi;

 export function parsePromptImageReferences(content: string): PromptImageReferencePart[] {
  const matches = collectReferenceMatches(content);
@@ -39,6 +39,7 @@ import { SharePopover } from "@/ui/components/SharePopover";
 import { autoImageAdjustments, defaultImageAdjustments, parseImageAdjustments, serializeImageAdjustments } from "@/ui/lib/imageAdjustments";
 import { friendlyProjectFailureMessage } from "@/ui/lib/friendlyAgentErrors";
 import { canvasImageUrl } from "@/ui/lib/imageDelivery";
+import { createModelAgentContentPreparer } from "@/ui/lib/modelImageReferences";
 import { visiblePromptText } from "@/ui/lib/promptImageReferences";
 import { isImageGeneratorThreadMessage, visibleAgentMessages } from "./canvas/agentMessageFilters";
 import { CanvasColorPopover } from "./canvas/CanvasColorPopover";
@@ -90,6 +91,7 @@ const defaultCanvasBackgroundColor = "#F7F7F4";
 const canvasBackgroundSwatches = ["#F5F5F5", "#000000", "#FFFFFF", "#00F016", "#A855F7", "#E9D5FF"];
 const canvasSidePanelWidth = 268;
 const newAgentConversationId = "__new_agent_conversation__";
+const prepareModelAgentContents = createModelAgentContentPreparer({ uploadImage: designGateway.uploadImage });
 const canvasWheelGuardSelector = [
  ".ui-popover-content",
  ".ui-dropdown-menu-content",
@@ -2769,10 +2771,11 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
      if (dirty) {
        await saveCanvasSnapshot({ quiet: true });
      }
+      const modelContents = await prepareModelAgentContents(contents);
      startAgentSocket(
        project.id,
        {
-          messages: [{ role: "user", contents }],
+          messages: [{ role: "user", contents: modelContents }],
          mode: source === "image-generator" ? "image-generator" : activeMode === "image" ? "image" : "design",
          threadId,
          threadIdType: 9,
@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { createModelAgentContentPreparer } from "../src/ui/lib/modelImageReferences.ts";
+
+test("replaces only the submitted URL for named SVG references", async () => {
+  const calls = { fetch: 0, rasterize: 0, upload: 0 };
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => {
+      calls.fetch += 1;
+      return new Response('<svg viewBox="0 0 10 10"></svg>', { headers: { "content-type": "image/svg+xml" } });
+    },
+    rasterizeSvg: async (_source, fileName) => {
+      calls.rasterize += 1;
+      assert.equal(fileName, "drawing (DXF).svg");
+      return { name: "drawing (DXF).model.webp", type: "image/webp" };
+    },
+    uploadImage: async (file) => {
+      calls.upload += 1;
+      assert.equal(file.type, "image/webp");
+      return "https://assets.example.com/model/drawing.webp";
+    }
+  });
+  const contents = [
+    { type: "image", imageUrl: "https://assets.example.com/source/opaque-id", imageName: "drawing (DXF).svg" },
+    { type: "text", text: "Use this plan", textSource: "input" }
+  ];
+
+  const prepared = await prepare(contents);
+
+  assert.equal(prepared[0].imageUrl, "https://assets.example.com/model/drawing.webp");
+  assert.equal(prepared[0].imageName, "drawing (DXF).svg");
+  assert.equal(contents[0].imageUrl, "https://assets.example.com/source/opaque-id");
+  assert.deepEqual(calls, { fetch: 1, rasterize: 1, upload: 1 });
+});
+
+test("keeps supported raster references unchanged without downloading them", async () => {
+  let called = false;
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => {
+      called = true;
+      throw new Error("unexpected fetch");
+    },
+    rasterizeSvg: async () => {
+      throw new Error("unexpected rasterization");
+    },
+    uploadImage: async () => {
+      throw new Error("unexpected upload");
+    }
+  });
+  const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.webp", imageName: "reference.webp" }];
+
+  const prepared = await prepare(contents);
+
+  assert.strictEqual(prepared[0], contents[0]);
+  assert.equal(called, false);
+});
+
+test("detects SVG references from response MIME when names have no extension", async () => {
+  let rasterized = 0;
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => new Response("<svg></svg>", { headers: { "content-type": "image/svg+xml; charset=utf-8" } }),
+    rasterizeSvg: async () => {
+      rasterized += 1;
+      return { name: "reference.model.webp", type: "image/webp" };
+    },
+    uploadImage: async () => "https://assets.example.com/reference.model.webp"
+  });
+
+  const prepared = await prepare([{ type: "image", imageUrl: "https://assets.example.com/opaque-id" }]);
+
+  assert.equal(prepared[0].imageUrl, "https://assets.example.com/reference.model.webp");
+  assert.equal(rasterized, 1);
+});
+
+test("caches uploaded WebP derivatives by source URL", async () => {
+  let uploads = 0;
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => new Response("<svg></svg>", { headers: { "content-type": "image/svg+xml" } }),
+    rasterizeSvg: async () => ({ name: "reference.model.webp", type: "image/webp" }),
+    uploadImage: async () => {
+      uploads += 1;
+      return "https://assets.example.com/reference.model.webp";
+    }
+  });
+  const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.svg", imageName: "reference.svg" }];
+
+  await Promise.all([prepare(contents), prepare(contents)]);
+
+  assert.equal(uploads, 1);
+});
@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parsePromptImageReferences } from "../src/ui/lib/promptImageReferences.ts";
+
+test("parses CAD-derived SVG reference directives with nested parentheses", () => {
+  const url = "http://localhost:19000/canvas-assets/drawing.svg";
+  const parts = parsePromptImageReferences(`Reference image (06-5070 R3 2026-05-05(1) (DXF).svg): ${url}\nMake it pink`);
+
+  assert.deepEqual(parts, [
+    {
+      type: "reference",
+      reference: {
+        id: parts[0]?.type === "reference" ? parts[0].reference.id : "",
+        index: undefined,
+        name: "06-5070 R3 2026-05-05(1) (DXF).svg",
+        url
+      }
+    },
+    { type: "text", text: "\nMake it pink" }
+  ]);
+});
+
+test("keeps Chinese CAD reference directives on the attachment path", () => {
+  const url = "https://assets.example.com/plan.svg";
+  const parts = parsePromptImageReferences(`参考图 1（花语江南 (DWG).svg）：${url}\n继续设计`);
+
+  assert.equal(parts[0]?.type, "reference");
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.index : undefined, 1);
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "花语江南 (DWG).svg");
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.url : "", url);
+});
+
+test("continues to parse raster reference directives", () => {
+  const parts = parsePromptImageReferences("Reference image 2 (sample.webp): https://assets.example.com/sample.webp");
+
+  assert.equal(parts[0]?.type, "reference");
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "sample.webp");
+});