root/moteva
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): attach uploaded image names to model prompts

Browse Source 
Add modelImageReferences to preprocess agent contents, uploading local
image references and surfacing their names in the prompt so the model can
cite them. Wire the preparer into the canvas agent submission path, pass
imageName through the floating composer references, and relax the prompt
directive parser to accept names containing arbitrary characters.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-07-15 12:08:43 +08:00
parent 3336f23e6f
commit 09b961fcc4
6 changed files with 314 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/ui/components/ImageGeneratorFloatingComposer/index.tsx
+2 -1
View File
@@ -161,7 +161,8 @@ export function ImageGeneratorFloatingComposer({
.filter((reference) => reference.publicUrl.trim())
.map((reference): AgentContent => ({
type: "image",
imageUrl: reference.publicUrl
imageUrl: reference.publicUrl,
imageName: reference.name
}));
const contents = [...referenceContents, ...promptContents];
onSubmit(contents, resetComposer);
frontend/src/ui/lib/modelImageReferences.ts
+177
View File
@@ -0,0 +1,177 @@
import type { AgentContent } from "@/domain/design";
type ImageContent = Extract<AgentContent, { type: "image" }>;
type ModelImageReferenceDependencies = {
uploadImage: (file: File) => Promise<string>;
fetchImage?: (url: string) => Promise<Response>;
rasterizeSvg?: (source: Blob, fileName: string) => Promise<File>;
};
type ImageFormatHint = "svg" | "raster" | "unknown";
const modelReferenceMaxEdge = 2048;
const svgMimeType = "image/svg+xml";
const svgExtensionPattern = /\.svg(?:$|[?#])/i;
const rasterExtensionPattern = /\.(?:png|jpe?g|gif|webp)(?:$|[?#])/i;
const rasterDataUrlPattern = /^data:image\/(?:png|jpeg|gif|webp)[;,]/i;
export function createModelAgentContentPreparer(dependencies: ModelImageReferenceDependencies) {
const preparedUrls = new Map<string, Promise<string>>();
const fetchImage = dependencies.fetchImage ?? ((url: string) => fetch(url));
const rasterizeSvg = dependencies.rasterizeSvg ?? rasterizeSvgToTransparentWebp;
return async (contents: AgentContent[]) =>
Promise.all(
contents.map(async (content): Promise<AgentContent> => {
if (content.type !== "image" || !content.imageUrl.trim()) return content;
const hint = imageFormatHint(content);
if (hint === "raster") return content;
const imageUrl = content.imageUrl.trim();
let preparedUrl = preparedUrls.get(imageUrl);
if (!preparedUrl) {
preparedUrl = prepareModelImageUrl(content, hint, fetchImage, rasterizeSvg, dependencies.uploadImage).catch((error) => {
preparedUrls.delete(imageUrl);
throw error;
});
preparedUrls.set(imageUrl, preparedUrl);
}
const nextImageUrl = await preparedUrl;
return nextImageUrl === content.imageUrl ? content : { ...content, imageUrl: nextImageUrl };
})
);
}
export async function rasterizeSvgToTransparentWebp(source: Blob, fileName: string) {
const svgSource = source.type.toLowerCase() === svgMimeType ? source : new Blob([source], { type: svgMimeType });
const svgText = await svgSource.text();
const documentNode = new DOMParser().parseFromString(svgText, svgMimeType);
if (documentNode.querySelector("parsererror") || documentNode.documentElement.localName.toLowerCase() !== "svg") {
throw new Error("The selected SVG reference is invalid.");
}
const objectUrl = URL.createObjectURL(svgSource);
try {
const image = await loadSvgImage(objectUrl);
const aspectRatio = svgAspectRatio(documentNode.documentElement, image);
const width = aspectRatio >= 1 ? modelReferenceMaxEdge : Math.max(1, Math.round(modelReferenceMaxEdge * aspectRatio));
const height = aspectRatio >= 1 ? Math.max(1, Math.round(modelReferenceMaxEdge / aspectRatio)) : modelReferenceMaxEdge;
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d");
if (!context) throw new Error("Unable to create a canvas for the SVG reference.");
context.clearRect(0, 0, width, height);
context.drawImage(image, 0, 0, width, height);
const webp = await canvasToBlob(canvas, "image/webp", 0.95);
if (webp.type.toLowerCase() !== "image/webp") {
throw new Error("This browser cannot encode WebP reference images.");
}
return new File([webp], modelWebpFileName(fileName), { type: "image/webp", lastModified: Date.now() });
} finally {
URL.revokeObjectURL(objectUrl);
}
}
async function prepareModelImageUrl(
content: ImageContent,
hint: ImageFormatHint,
fetchImage: (url: string) => Promise<Response>,
rasterizeSvg: (source: Blob, fileName: string) => Promise<File>,
uploadImage: (file: File) => Promise<string>
) {
let response: Response;
try {
response = await fetchImage(content.imageUrl.trim());
} catch (error) {
if (hint === "unknown") return content.imageUrl;
throw error;
}
if (!response.ok) {
if (hint === "unknown") return content.imageUrl;
throw new Error(`Unable to load SVG reference (${response.status}).`);
}
const source = await response.blob();
if (hint === "unknown" && !(await blobIsSvg(source, response.headers.get("content-type")))) {
return content.imageUrl;
}
const webp = await rasterizeSvg(source, referenceFileName(content));
const uploadedUrl = (await uploadImage(webp)).trim();
if (!uploadedUrl) throw new Error("The converted WebP reference did not return an asset URL.");
return uploadedUrl;
}
function imageFormatHint(content: ImageContent): ImageFormatHint {
const name = content.imageName?.trim() || "";
const url = content.imageUrl.trim();
if (svgExtensionPattern.test(name) || svgExtensionPattern.test(url) || /^data:image\/svg\+xml[;,]/i.test(url)) return "svg";
if (rasterExtensionPattern.test(name) || rasterExtensionPattern.test(url) || rasterDataUrlPattern.test(url)) return "raster";
return "unknown";
}
async function blobIsSvg(source: Blob, contentType: string | null) {
const normalizedType = (contentType || source.type).split(";", 1)[0]?.trim().toLowerCase();
if (normalizedType === svgMimeType) return true;
if (normalizedType?.startsWith("image/") && normalizedType !== svgMimeType) return false;
const prefix = await source.slice(0, 4096).text();
return /<svg(?:\s|>)/i.test(prefix);
}
function referenceFileName(content: ImageContent) {
const preferred = content.imageName?.trim();
if (preferred) return preferred;
try {
const path = new URL(content.imageUrl, "http://canvas.local").pathname;
return decodeURIComponent(path.split("/").pop() || "reference.svg");
} catch {
return "reference.svg";
}
}
function modelWebpFileName(fileName: string) {
const safeName = fileName.replace(/[\\/]/g, "_").trim() || "reference.svg";
const baseName = safeName.replace(/\.svg$/i, "");
return `${baseName}.model.webp`;
}
function svgAspectRatio(root: Element, image: HTMLImageElement) {
const viewBox = root
.getAttribute("viewBox")
?.trim()
.split(/[\s,]+/)
.map(Number);
if (viewBox?.length === 4 && Number.isFinite(viewBox[2]) && Number.isFinite(viewBox[3]) && viewBox[2] > 0 && viewBox[3] > 0) {
return viewBox[2] / viewBox[3];
}
const width = Number.parseFloat(root.getAttribute("width") || "");
const height = Number.parseFloat(root.getAttribute("height") || "");
if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) return width / height;
if (image.naturalWidth > 0 && image.naturalHeight > 0) return image.naturalWidth / image.naturalHeight;
return 1;
}
function loadSvgImage(source: string) {
return new Promise<HTMLImageElement>((resolve, reject) => {
const image = new Image();
image.decoding = "async";
image.onload = () => resolve(image);
image.onerror = () => reject(new Error("Unable to render the selected SVG reference."));
image.src = source;
});
}
function canvasToBlob(canvas: HTMLCanvasElement, type: string, quality: number) {
return new Promise<Blob>((resolve, reject) => {
canvas.toBlob((blob) => {
if (blob) resolve(blob);
else reject(new Error("Unable to encode the SVG reference as WebP."));
}, type, quality);
});
}
frontend/src/ui/lib/promptImageReferences.ts
+1 -1
View File
@@ -24,7 +24,7 @@ type ReferenceMatch = {
};
const imageTokenPattern = /\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]/g;
const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[（(]([^）)\n]+)[）)])?\s*[:：]\s*(https?:\/\/[^\s\]]+)/gi;
const promptDirectivePattern = /(参考图|Reference image)\s*(\d+)?\s*(?:[（(]([^\n]*?)[）)])?\s*[:：]\s*(https?:\/\/[^\s\]]+)/gi;
export function parsePromptImageReferences(content: string): PromptImageReferencePart[] {
const matches = collectReferenceMatches(content);
frontend/src/ui/pages/CanvasWorkspace/index.tsx
+4 -1
View File
@@ -39,6 +39,7 @@ import { SharePopover } from "@/ui/components/SharePopover";
import { autoImageAdjustments, defaultImageAdjustments, parseImageAdjustments, serializeImageAdjustments } from "@/ui/lib/imageAdjustments";
import { friendlyProjectFailureMessage } from "@/ui/lib/friendlyAgentErrors";
import { canvasImageUrl } from "@/ui/lib/imageDelivery";
import { createModelAgentContentPreparer } from "@/ui/lib/modelImageReferences";
import { visiblePromptText } from "@/ui/lib/promptImageReferences";
import { isImageGeneratorThreadMessage, visibleAgentMessages } from "./canvas/agentMessageFilters";
import { CanvasColorPopover } from "./canvas/CanvasColorPopover";
@@ -90,6 +91,7 @@ const defaultCanvasBackgroundColor = "#F7F7F4";
const canvasBackgroundSwatches = ["#F5F5F5", "#000000", "#FFFFFF", "#00F016", "#A855F7", "#E9D5FF"];
const canvasSidePanelWidth = 268;
const newAgentConversationId = "__new_agent_conversation__";
const prepareModelAgentContents = createModelAgentContentPreparer({ uploadImage: designGateway.uploadImage });
const canvasWheelGuardSelector = [
".ui-popover-content",
".ui-dropdown-menu-content",
@@ -2769,10 +2771,11 @@ export function CanvasWorkspace({ projectId, shareAccess = ownerWorkspaceAccess
if (dirty) {
await saveCanvasSnapshot({ quiet: true });
}
const modelContents = await prepareModelAgentContents(contents);
startAgentSocket(
project.id,
{
messages: [{ role: "user", contents }],
messages: [{ role: "user", contents: modelContents }],
mode: source === "image-generator" ? "image-generator" : activeMode === "image" ? "image" : "design",
threadId,
threadIdType: 9,