feat(image): pass reference images to GPT image model for edits
Server: forward canvas/reference images to the image model so extract, continue, and edit requests stay faithful to the source product. - Add ImageRequestOptions.Images and route requests with inputs to the /v1/images/edits endpoint, via multipart upload (default) or JSON image_url payload, selectable through Agent.Image.InputImageTransport. - Extract reference URLs from prompt directives, inline @image tokens, and chat message image/mask contents; fold them into the idempotency key and prepend a reference-fidelity instruction to the prompt. Frontend: - Shrink CanvasToast to a compact top pill. - Paste an internal transparent clipboard node instead of re-uploading when it matches the clipboard image. - Keep agent-project polling alive on stream error while streaming.
This commit is contained in:
@@ -233,8 +233,9 @@ export function useGenerationStream({
|
||||
closeRef.current = null;
|
||||
},
|
||||
onError: (nextError) => {
|
||||
if (closeAgentProjectPollRef.current) {
|
||||
if (streamingRef.current === streamKey) {
|
||||
closeRef.current = null;
|
||||
startAgentProjectPolling(nextProjectId, threadId);
|
||||
return;
|
||||
}
|
||||
completeChannel(channel);
|
||||
@@ -243,7 +244,6 @@ export function useGenerationStream({
|
||||
closeRef.current = null;
|
||||
}
|
||||
});
|
||||
startAgentProjectPolling(nextProjectId, threadId);
|
||||
},
|
||||
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
|
||||
);
|
||||
@@ -306,8 +306,9 @@ export function useGenerationStream({
|
||||
closeRef.current = null;
|
||||
},
|
||||
onError: (nextError) => {
|
||||
if (closeAgentProjectPollRef.current) {
|
||||
if (streamingRef.current === streamKey) {
|
||||
closeRef.current = null;
|
||||
startAgentProjectPolling(nextProjectId, payload.threadId);
|
||||
return;
|
||||
}
|
||||
completeChannel(channel);
|
||||
@@ -316,7 +317,6 @@ export function useGenerationStream({
|
||||
closeRef.current = null;
|
||||
}
|
||||
});
|
||||
startAgentProjectPolling(nextProjectId, payload.threadId);
|
||||
},
|
||||
[appendThinkingMessage, applyProject, completeChannel, toUserMessageText, messages.generateError, messages.generateTimeout, setError, startAgentProjectPolling, startGeneratorTaskPolling]
|
||||
);
|
||||
|
||||
@@ -1843,9 +1843,9 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
|
||||
.catch(() => showCanvasToast(t("clipboardCopyFailed"), "error"));
|
||||
};
|
||||
|
||||
const pasteNode = () => {
|
||||
if (!clipboardNode) return;
|
||||
const copy = cloneCanvasNode(clipboardNode, cryptoFallbackId());
|
||||
const pasteNode = (sourceNode = clipboardNode) => {
|
||||
if (!sourceNode) return;
|
||||
const copy = cloneCanvasNode(sourceNode, cryptoFallbackId());
|
||||
setNodes((current) => [...current, copy]);
|
||||
selectOnlyNode(copy.id);
|
||||
setDirty(true);
|
||||
@@ -2145,11 +2145,25 @@ export function CanvasWorkspace({ projectId }: { projectId: string }) {
|
||||
if (isEditableShortcutTarget(event.target)) return;
|
||||
|
||||
const imageFiles = event.clipboardData ? dataTransferImageFiles(event.clipboardData, "clipboard-image") : [];
|
||||
if (imageFiles.length > 0) {
|
||||
event.preventDefault();
|
||||
const pasteImageFiles = () => {
|
||||
const pointer = lastCanvasPointerRef.current;
|
||||
const anchor = pointer ? canvasPointFromClient(pointer.clientX, pointer.clientY) : canvasCenterPoint();
|
||||
void uploadImagesToCanvas(imageFiles, anchor);
|
||||
};
|
||||
if (clipboardNode && imageFiles.length > 0 && isTransparentClipboardNode(clipboardNode)) {
|
||||
event.preventDefault();
|
||||
void shouldUseInternalClipboardNode(clipboardNode, imageFiles).then((useInternalNode) => {
|
||||
if (useInternalNode) {
|
||||
pasteNode(clipboardNode);
|
||||
return;
|
||||
}
|
||||
pasteImageFiles();
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (imageFiles.length > 0) {
|
||||
event.preventDefault();
|
||||
pasteImageFiles();
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user