feat(image): pass reference images to GPT image model for edits

Server: forward canvas/reference images to the image model so extract,
continue, and edit requests stay faithful to the source product.
- Add ImageRequestOptions.Images and route requests with inputs to the
  /v1/images/edits endpoint, via multipart upload (default) or JSON
  image_url payload, selectable through Agent.Image.InputImageTransport.
- Extract reference URLs from prompt directives, inline @image tokens,
  and chat message image/mask contents; fold them into the idempotency
  key and prepend a reference-fidelity instruction to the prompt.

Frontend:
- Shrink CanvasToast to a compact top pill.
- Paste an internal transparent clipboard node instead of re-uploading
  when it matches the clipboard image.
- Keep agent-project polling alive on stream error while streaming.
This commit is contained in:
2026-07-08 01:19:20 +08:00
parent bb7fce9171
commit 2f5291a0f9
11 changed files with 614 additions and 63 deletions
@@ -233,8 +233,9 @@ export function useGenerationStream({
closeRef.current = null;
},
onError: (nextError) => {
if (closeAgentProjectPollRef.current) {
if (streamingRef.current === streamKey) {
closeRef.current = null;
startAgentProjectPolling(nextProjectId, threadId);
return;
}
completeChannel(channel);
@@ -243,7 +244,6 @@ export function useGenerationStream({
closeRef.current = null;
}
});
startAgentProjectPolling(nextProjectId, threadId);
},
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
);
@@ -306,8 +306,9 @@ export function useGenerationStream({
closeRef.current = null;
},
onError: (nextError) => {
if (closeAgentProjectPollRef.current) {
if (streamingRef.current === streamKey) {
closeRef.current = null;
startAgentProjectPolling(nextProjectId, payload.threadId);
return;
}
completeChannel(channel);
@@ -316,7 +317,6 @@ export function useGenerationStream({
closeRef.current = null;
}
});
startAgentProjectPolling(nextProjectId, payload.threadId);
},
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
);
@@ -1843,9 +1843,9 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
.catch(() => showCanvasToast(t("clipboardCopyFailed"), "error"));
};
const pasteNode = () => {
if (!clipboardNode) return;
const copy = cloneCanvasNode(clipboardNode, cryptoFallbackId());
const pasteNode = (sourceNode = clipboardNode) => {
if (!sourceNode) return;
const copy = cloneCanvasNode(sourceNode, cryptoFallbackId());
setNodes((current) => [...current, copy]);
selectOnlyNode(copy.id);
setDirty(true);
@@ -2145,11 +2145,25 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
if (isEditableShortcutTarget(event.target)) return;
const imageFiles = event.clipboardData ? dataTransferImageFiles(event.clipboardData, "clipboard-image") : [];
if (imageFiles.length > 0) {
event.preventDefault();
const pasteImageFiles = () => {
const pointer = lastCanvasPointerRef.current;
const anchor = pointer ? canvasPointFromClient(pointer.clientX, pointer.clientY) : canvasCenterPoint();
void uploadImagesToCanvas(imageFiles, anchor);
};
if (clipboardNode && imageFiles.length > 0 && isTransparentClipboardNode(clipboardNode)) {
event.preventDefault();
void shouldUseInternalClipboardNode(clipboardNode, imageFiles).then((useInternalNode) => {
if (useInternalNode) {
pasteNode(clipboardNode);
return;
}
pasteImageFiles();
});
return;
}
if (imageFiles.length > 0) {
event.preventDefault();
pasteImageFiles();
return;
}