feat(canvas): attach uploaded image names to model prompts

Add modelImageReferences to preprocess agent contents, uploading local image references and surfacing their names in the prompt so the model can cite them. Wire the preparer into the canvas agent submission path, pass imageName through the floating composer references, and relax the prompt directive parser to accept names containing arbitrary characters. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-15 12:08:43 +08:00
parent 3336f23e6f
commit 09b961fcc4
6 changed files with 314 additions and 3 deletions
@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { createModelAgentContentPreparer } from "../src/ui/lib/modelImageReferences.ts";
+
+test("replaces only the submitted URL for named SVG references", async () => {
+  const calls = { fetch: 0, rasterize: 0, upload: 0 };
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => {
+      calls.fetch += 1;
+      return new Response('<svg viewBox="0 0 10 10"></svg>', { headers: { "content-type": "image/svg+xml" } });
+    },
+    rasterizeSvg: async (_source, fileName) => {
+      calls.rasterize += 1;
+      assert.equal(fileName, "drawing (DXF).svg");
+      return { name: "drawing (DXF).model.webp", type: "image/webp" };
+    },
+    uploadImage: async (file) => {
+      calls.upload += 1;
+      assert.equal(file.type, "image/webp");
+      return "https://assets.example.com/model/drawing.webp";
+    }
+  });
+  const contents = [
+    { type: "image", imageUrl: "https://assets.example.com/source/opaque-id", imageName: "drawing (DXF).svg" },
+    { type: "text", text: "Use this plan", textSource: "input" }
+  ];
+
+  const prepared = await prepare(contents);
+
+  assert.equal(prepared[0].imageUrl, "https://assets.example.com/model/drawing.webp");
+  assert.equal(prepared[0].imageName, "drawing (DXF).svg");
+  assert.equal(contents[0].imageUrl, "https://assets.example.com/source/opaque-id");
+  assert.deepEqual(calls, { fetch: 1, rasterize: 1, upload: 1 });
+});
+
+test("keeps supported raster references unchanged without downloading them", async () => {
+  let called = false;
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => {
+      called = true;
+      throw new Error("unexpected fetch");
+    },
+    rasterizeSvg: async () => {
+      throw new Error("unexpected rasterization");
+    },
+    uploadImage: async () => {
+      throw new Error("unexpected upload");
+    }
+  });
+  const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.webp", imageName: "reference.webp" }];
+
+  const prepared = await prepare(contents);
+
+  assert.strictEqual(prepared[0], contents[0]);
+  assert.equal(called, false);
+});
+
+test("detects SVG references from response MIME when names have no extension", async () => {
+  let rasterized = 0;
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => new Response("<svg></svg>", { headers: { "content-type": "image/svg+xml; charset=utf-8" } }),
+    rasterizeSvg: async () => {
+      rasterized += 1;
+      return { name: "reference.model.webp", type: "image/webp" };
+    },
+    uploadImage: async () => "https://assets.example.com/reference.model.webp"
+  });
+
+  const prepared = await prepare([{ type: "image", imageUrl: "https://assets.example.com/opaque-id" }]);
+
+  assert.equal(prepared[0].imageUrl, "https://assets.example.com/reference.model.webp");
+  assert.equal(rasterized, 1);
+});
+
+test("caches uploaded WebP derivatives by source URL", async () => {
+  let uploads = 0;
+  const prepare = createModelAgentContentPreparer({
+    fetchImage: async () => new Response("<svg></svg>", { headers: { "content-type": "image/svg+xml" } }),
+    rasterizeSvg: async () => ({ name: "reference.model.webp", type: "image/webp" }),
+    uploadImage: async () => {
+      uploads += 1;
+      return "https://assets.example.com/reference.model.webp";
+    }
+  });
+  const contents = [{ type: "image", imageUrl: "https://assets.example.com/reference.svg", imageName: "reference.svg" }];
+
+  await Promise.all([prepare(contents), prepare(contents)]);
+
+  assert.equal(uploads, 1);
+});
@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parsePromptImageReferences } from "../src/ui/lib/promptImageReferences.ts";
+
+test("parses CAD-derived SVG reference directives with nested parentheses", () => {
+  const url = "http://localhost:19000/canvas-assets/drawing.svg";
+  const parts = parsePromptImageReferences(`Reference image (06-5070 R3 2026-05-05(1) (DXF).svg): ${url}\nMake it pink`);
+
+  assert.deepEqual(parts, [
+    {
+      type: "reference",
+      reference: {
+        id: parts[0]?.type === "reference" ? parts[0].reference.id : "",
+        index: undefined,
+        name: "06-5070 R3 2026-05-05(1) (DXF).svg",
+        url
+      }
+    },
+    { type: "text", text: "\nMake it pink" }
+  ]);
+});
+
+test("keeps Chinese CAD reference directives on the attachment path", () => {
+  const url = "https://assets.example.com/plan.svg";
+  const parts = parsePromptImageReferences(`参考图 1（花语江南 (DWG).svg）：${url}\n继续设计`);
+
+  assert.equal(parts[0]?.type, "reference");
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.index : undefined, 1);
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "花语江南 (DWG).svg");
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.url : "", url);
+});
+
+test("continues to parse raster reference directives", () => {
+  const parts = parsePromptImageReferences("Reference image 2 (sample.webp): https://assets.example.com/sample.webp");
+
+  assert.equal(parts[0]?.type, "reference");
+  assert.equal(parts[0]?.type === "reference" ? parts[0].reference.name : "", "sample.webp");
+});