Compare commits

...

7 Commits

Author SHA1 Message Date
root ada29cee12 refactor(agent): make emoji optional and sparse in agent replies
Loosen the creative agent guidance so emoji are used only as sparse
category markers rather than forced on every list item, and tighten the
frontend emoji section-break normalization to only split before short
labelled headings.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 16:18:11 +08:00
root ba4db85241 fix(logic): ignore reference directives when checking for prompt text
Strip inline image tokens and reference directive URLs before deciding
whether a prompt carries real user text, so a reference-only prompt is
still treated as empty while text around references counts.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 16:17:59 +08:00
root 68427ad91a feat(piagent): give image generator inline reference placement context
Derive an ordered map of each inline reference image to the surrounding
prompt text describing its role, and feed it into the decision, research,
task-plan, conversation and image-prompt builders. The image prompt now
carries this placement so reference roles are not swapped.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 16:17:52 +08:00
root 9622b27de9 fix(piagent): upload local and private reference images as multipart
When URL input transport is configured, the remote image API cannot fetch
localhost or private-network reference URLs. Detect such hosts and fall back
to multipart file upload so those references still reach the model.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 16:17:29 +08:00
root 13fdf464ae feat(netutil): add local and private host detection helper
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 16:17:20 +08:00
root 0eed208bbf feat(ui): add typewriter effect for streaming agent text
Add a useTypewriterText hook that reveals streamed content character by
character and reconciles when the target text changes. Apply it to
markdown content and agent step copy, with a blinking caret while
streaming.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 15:08:21 +08:00
root d7300ed0f6 feat(ui): make workspace layout responsive for narrow and mobile screens
Widen the assistant panel via a shared CSS variable, overlay it below
1180px, and add mobile breakpoints for the topbar, canvas controls,
toolbar, minimap, and bottom-sheet assistant panel. Use dvh units and
safe-area insets for mobile viewports.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 14:39:00 +08:00
15 changed files with 900 additions and 25 deletions
@@ -328,6 +328,16 @@
line-height: 19px;
}
.agent-step-caret {
width: 6px;
height: 1em;
display: inline-block;
margin-left: 2px;
background: currentColor;
vertical-align: -0.15em;
animation: agent-step-caret 1s steps(1) infinite;
}
@keyframes agent-thinking-enter {
from {
opacity: 0;
@@ -350,6 +360,12 @@
}
}
@keyframes agent-step-caret {
50% {
opacity: 0;
}
}
@keyframes agent-thinking-shimmer {
from {
background-position: 120% 0;
@@ -5,6 +5,7 @@ import { Check, CircleAlert, Copy, Download, Eye, Globe2, Heart } from "lucide-r
import type { AgentMessage, CanvasNode } from "@/domain/design";
import { useI18n } from "@/i18n/i18n";
import { MarkdownContent } from "@/ui/components/MarkdownContent";
import { useTypewriterText } from "@/ui/hooks/useTypewriterText";
import { friendlyAgentErrorMessage } from "@/ui/lib/friendlyAgentErrors";
import { canvasImageUrl, thumbnailImageUrl } from "@/ui/lib/imageDelivery";
import { parsePromptImageReferences, serializePromptImageReferences, type PromptImageReference } from "@/ui/lib/promptImageReferences";
@@ -275,11 +276,11 @@ function SkillProgress({ messages, active }: { messages: AgentMessage[]; active:
{showThinking && (
<section className={`agent-tool-group agent-thinking-card ${active ? "current" : "complete"} ${thinkingContent ? "has-content" : ""}`}>
<AgentThinkingTitle label={thinkingLabel} active={active} expanded={Boolean(thinkingContent)} />
{thinkingContent && (
{currentStep && thinkingContent && (
<div className="agent-tool-secondary">
<div className="agent-tool-line" aria-hidden="true" />
<div className="agent-tool-content agent-step-copy">
<p>{thinkingContent}</p>
<AgentStepCopy key={currentStep.id} content={thinkingContent} streaming={active} />
</div>
</div>
)}
@@ -289,6 +290,16 @@ function SkillProgress({ messages, active }: { messages: AgentMessage[]; active:
);
}
function AgentStepCopy({ content, streaming }: { content: string; streaming: boolean }) {
const visibleContent = useTypewriterText(content, streaming);
return (
<p>
{visibleContent}
{streaming && <span className="agent-step-caret" aria-hidden="true" />}
</p>
);
}
function AgentThinkingTitle({ label, active, expanded }: { label: string; active: boolean; expanded: boolean }) {
return (
<div className={`agent-thinking-title ${active ? "is-thinking" : "is-complete"} ${expanded ? "is-expanded" : ""}`} data-testid="agent-chat-session-title">
@@ -1,6 +1,7 @@
"use client";
import { Fragment, type ReactNode } from "react";
import { useTypewriterText } from "@/ui/hooks/useTypewriterText";
import "./index.css";
type MarkdownBlock =
@@ -9,7 +10,8 @@ type MarkdownBlock =
| { type: "ol"; items: string[] };
export function MarkdownContent({ content, streaming = false }: { content: string; streaming?: boolean }) {
const blocks = parseMarkdownBlocks(content);
const visibleContent = useTypewriterText(content, streaming);
const blocks = parseMarkdownBlocks(normalizeEmojiSectionBreaks(visibleContent));
return (
<div className={`markdown-content ${streaming ? "streaming" : ""}`}>
{blocks.map((block, index) => {
@@ -38,6 +40,10 @@ export function MarkdownContent({ content, streaming = false }: { content: strin
);
}
function normalizeEmojiSectionBreaks(content: string) {
return content.replace(/([^\s\n])[ \t]*((?:[\p{Extended_Pictographic}\p{Emoji_Presentation}]\uFE0F?(?:\u200D[\p{Extended_Pictographic}\p{Emoji_Presentation}]\uFE0F?)*)+(?=[^\s\n]{1,12}[:]))/gu, "$1\n$2");
}
function parseMarkdownBlocks(content: string) {
const blocks: MarkdownBlock[] = [];
const lines = content.replace(/\r\n/g, "\n").split("\n");
+111
View File
@@ -0,0 +1,111 @@
"use client";
import { useEffect, useRef, useState } from "react";
const typewriterIntervalMs = 18;
export function useTypewriterText(text: string, active: boolean) {
const [visibleText, setVisibleText] = useState(() => (active ? "" : text));
const [settling, setSettling] = useState(false);
const targetTextRef = useRef(text);
const visibleTextRef = useRef(visibleText);
const animatedRef = useRef(active);
useEffect(() => {
visibleTextRef.current = visibleText;
}, [visibleText]);
useEffect(() => {
targetTextRef.current = text;
if (active) {
animatedRef.current = true;
setSettling(true);
setVisibleText((current) => {
const next = reconcileVisibleText(current, text);
visibleTextRef.current = next;
return next;
});
return;
}
if (!animatedRef.current) {
visibleTextRef.current = text;
setVisibleText(text);
setSettling(false);
return;
}
const next = reconcileVisibleText(visibleTextRef.current, text);
if (next !== visibleTextRef.current) {
visibleTextRef.current = next;
setVisibleText(next);
}
if (next === text) {
animatedRef.current = false;
setSettling(false);
return;
}
setSettling(true);
}, [active, text]);
useEffect(() => {
if (!active && !settling) return;
const tick = () => {
const target = targetTextRef.current;
const current = visibleTextRef.current;
if (current === target) {
if (!active) {
animatedRef.current = false;
setSettling(false);
}
return;
}
const next = nextTypewriterText(current, target);
visibleTextRef.current = next;
setVisibleText(next);
if (next === target && !active) {
animatedRef.current = false;
setSettling(false);
}
};
const timer = window.setInterval(tick, typewriterIntervalMs);
tick();
return () => window.clearInterval(timer);
}, [active, settling]);
if (!active && !settling && !animatedRef.current) return text;
return visibleText;
}
function nextTypewriterText(current: string, target: string) {
const base = reconcileVisibleText(current, target);
const nextCharacter = Array.from(target.slice(base.length))[0] ?? "";
return `${base}${nextCharacter}`;
}
function reconcileVisibleText(current: string, target: string) {
if (!current || target.startsWith(current)) return current;
if (current.startsWith(target)) return target;
return commonPrefix(current, target);
}
function commonPrefix(left: string, right: string) {
let index = 0;
while (index < left.length && index < right.length && left[index] === right[index]) {
index += 1;
}
if (index > 0 && isHighSurrogate(right.charCodeAt(index - 1))) {
index -= 1;
}
return right.slice(0, index);
}
function isHighSurrogate(code: number) {
return code >= 0xd800 && code <= 0xdbff;
}
+222 -12
View File
@@ -28,6 +28,7 @@
--blue: #2563eb;
--blue-soft: #dbeafe;
--black: #050505;
--assistant-panel-width: 600px;
--shadow-sm: 0 2px 8px rgba(17, 24, 39, 0.06);
--shadow-md: 0 12px 30px rgba(17, 24, 39, 0.08);
}
@@ -1167,6 +1168,12 @@ button:disabled {
background: #f7f7f4;
}
@supports (height: 100dvh) {
.workspace-shell {
height: 100dvh;
}
}
.workspace-topbar {
position: fixed;
inset: 0 0 auto;
@@ -1269,11 +1276,20 @@ button:disabled {
}
.workspace-main {
position: relative;
width: 100%;
height: calc(100vh - 48px);
margin-top: 48px;
min-height: 0;
overflow: hidden;
display: grid;
grid-template-columns: minmax(0, 1fr) 400px;
grid-template-columns: minmax(0, 1fr) minmax(0, var(--assistant-panel-width));
}
@supports (height: 100dvh) {
.workspace-main {
height: calc(100dvh - 48px);
}
}
.assistant-panel-collapsed .workspace-main {
@@ -1524,6 +1540,7 @@ button:disabled {
overflow: hidden;
background: var(--canvas-background, #f7f7f4);
cursor: default;
touch-action: none;
user-select: none;
}
@@ -3369,8 +3386,8 @@ button:disabled {
.assistant-panel {
position: relative;
width: 400px;
max-width: 400px;
width: var(--assistant-panel-width);
max-width: var(--assistant-panel-width);
min-width: 0;
min-height: 0;
display: flex;
@@ -3726,6 +3743,23 @@ button:disabled {
width: 348px;
}
@media (max-width: 1180px) {
.workspace-main {
grid-template-columns: minmax(0, 1fr);
}
.assistant-panel {
position: absolute;
top: 0;
right: 0;
bottom: 0;
z-index: 48;
width: min(var(--assistant-panel-width), calc(100vw - 16px));
max-width: calc(100vw - 16px);
box-shadow: -16px 0 40px rgba(17, 24, 39, 0.12);
}
}
@media (max-width: 980px) {
.home-main {
width: min(100% - 32px, 760px);
@@ -3746,14 +3780,6 @@ button:disabled {
min-height: 300px;
}
.workspace-main {
grid-template-columns: 1fr;
}
.assistant-panel {
display: none;
}
.promo-strip {
display: none;
}
@@ -3803,10 +3829,194 @@ button:disabled {
}
.workspace-topbar {
grid-template-columns: 1fr auto;
justify-content: flex-start;
padding: 0 10px;
}
.workspace-countdown {
max-width: 100%;
overflow: hidden;
}
.workspace-countdown > span {
min-width: 18px;
height: 22px;
border-radius: 7px;
font-size: 11px;
}
.workspace-countdown div {
min-width: 0;
margin-left: 6px;
overflow: hidden;
}
.workspace-countdown b {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.workspace-actions .text-button {
display: none;
}
.workspace-stage-title {
left: 6px;
right: 132px;
min-width: 0;
}
.canvas-open-account-controls {
right: 8px;
}
.canvas-collapsed-controls {
top: 6px;
right: 6px;
}
.workspace-title-name strong {
max-width: min(126px, 36vw);
}
.workspace-title-edit {
width: min(170px, 46vw);
}
.assistant-panel {
top: auto;
left: 0;
right: 0;
bottom: 0;
width: 100%;
max-width: none;
height: min(72vh, calc(100% - 8px));
border-top: 1px solid #eceff3;
border-left: 0;
border-radius: 18px 18px 0 0;
box-shadow: 0 -18px 42px rgba(17, 24, 39, 0.14);
padding-bottom: max(8px, env(safe-area-inset-bottom));
}
.assistant-header {
flex: 0 0 44px;
min-height: 44px;
}
.assistant-header-title {
height: 44px;
padding-left: 14px;
}
.assistant-header-title strong {
font-size: 13px;
}
.assistant-header-actions {
gap: 4px;
padding-right: 8px;
}
.assistant-feed {
max-width: none;
gap: 22px;
padding: 14px 12px 28px;
}
.assistant-scroll-to-end button {
width: 36px;
height: 36px;
}
.canvas-layer-panel {
width: min(var(--layer-panel-width), calc(100vw - 32px));
max-width: calc(100vw - 32px);
}
.canvas-generated-files-panel {
width: min(var(--generated-files-panel-width), calc(100vw - 24px));
max-width: calc(100vw - 24px);
}
.canvas-bottom-left,
.canvas-bottom-left.is-offset-by-files-panel,
.canvas-bottom-left.is-offset-by-layers-panel {
left: 8px;
right: 8px;
bottom: calc(62px + env(safe-area-inset-bottom));
}
.canvas-statusbar {
max-width: calc(100vw - 16px);
overflow-x: auto;
scrollbar-width: none;
}
.canvas-statusbar::-webkit-scrollbar {
display: none;
}
.canvas-minimap {
width: min(220px, calc(100vw - 16px));
height: 136px;
border-radius: 14px;
}
.workspace-toolbar,
.workspace-toolbar.is-offset-by-files-panel,
.workspace-toolbar.is-offset-by-layers-panel {
left: 8px;
right: 8px;
bottom: calc(10px + env(safe-area-inset-bottom));
width: auto;
max-width: none;
justify-content: flex-start;
overflow-x: auto;
transform: none;
scrollbar-width: none;
}
.workspace-toolbar::-webkit-scrollbar {
display: none;
}
.workspace-toolbar button {
flex: 0 0 32px;
}
.workspace-toolbar-separator {
flex: 0 0 1px;
}
.pen-tool-controls,
.pen-tool-controls.is-offset-by-files-panel,
.pen-tool-controls.is-offset-by-layers-panel {
left: 8px;
right: 8px;
bottom: calc(62px + env(safe-area-inset-bottom));
max-width: none;
overflow-x: auto;
transform: none;
}
}
@media (max-width: 480px) {
.workspace-countdown div {
display: none;
}
.workspace-stage-title {
right: 106px;
}
.canvas-conversation-button {
min-width: 46px;
padding: 0 8px;
font-size: 13px;
}
.canvas-collapsed-credit {
display: none;
}
}
@@ -0,0 +1,32 @@
package netutil
import (
"net"
"strings"
)
func IsLocalOrPrivateHost(host string) bool {
host = strings.ToLower(strings.TrimSpace(host))
if host == "" {
return false
}
if host == "localhost" || strings.HasSuffix(host, ".localhost") || host == "host.docker.internal" {
return true
}
ip := net.ParseIP(host)
return ip != nil && IsLocalOrPrivateIP(ip)
}
func IsLocalOrPrivateIP(ip net.IP) bool {
if ip == nil {
return false
}
if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
return true
}
ip4 := ip.To4()
if ip4 == nil {
return false
}
return ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127
}
@@ -0,0 +1,33 @@
package netutil
import "testing"
func TestIsLocalOrPrivateHost(t *testing.T) {
tests := []struct {
name string
host string
want bool
}{
{name: "localhost", host: "localhost", want: true},
{name: "localhost subdomain", host: "assets.localhost", want: true},
{name: "docker host", host: "host.docker.internal", want: true},
{name: "loopback ipv4", host: "127.0.0.1", want: true},
{name: "loopback ipv6", host: "::1", want: true},
{name: "private 10", host: "10.0.0.5", want: true},
{name: "private 172", host: "172.16.2.3", want: true},
{name: "private 192", host: "192.168.1.10", want: true},
{name: "link local", host: "169.254.10.20", want: true},
{name: "cgnat", host: "100.64.10.20", want: true},
{name: "unspecified", host: "0.0.0.0", want: true},
{name: "public ip", host: "8.8.8.8", want: false},
{name: "public hostname", host: "example.com", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsLocalOrPrivateHost(tt.host); got != tt.want {
t.Fatalf("IsLocalOrPrivateHost(%q) = %t, want %t", tt.host, got, tt.want)
}
})
}
}
@@ -94,8 +94,8 @@ func (c *CreativeAgent) Respond(ctx context.Context, req design.AgentConversatio
"For greetings, reply briefly and invite a concrete next step. For capability questions, answer with image-only capabilities: brand/site visuals, ecommerce product images, posters/banners/social images, reference-based generation, image edits, background removal, vectorization, mockup placement, and multi-round canvas iteration.",
"When a current project exists, connect capabilities to the canvas instead of giving a generic product brochure.",
"When the task plan indicates missing requirements or contains user response questions, ask those questions directly and stop; do not pretend generation has started.",
"Use a few purposeful emoji in visible replies to make scanning warmer: put a relevant emoji at the start of capability or next-step bullets, for example 🎨 design direction, 📦 packaging, 🖼️ canvas. Keep the tone professional and avoid empty bullet labels such as '• :'.",
"Format emoji lists as short separate lines: intro sentence, then one emoji-led item per line, for example 🎭角色设定、\\n📦系列包装、\\n🧩展示台/贴纸、\\n📣发售海报。 Do not put a hyphen after the emoji. Do not chain items like 🎨-设计...-生成..., do not start continuation lines with bare hyphens, and never end with empty choices such as 1. 2. 3.",
"Emoji are optional. Use them only when they improve scanning, especially as sparse category markers in capability or next-step replies, for example 🎨 图片生成与编辑, 🖼️ 画布迭代, 🔍 参考收集, ❌ 不在范围内.",
"When an emoji introduces a category or list item, start that emoji-led marker on its own line. Do not force every item into emoji lines, do not insert emoji inside dense prose, do not stack several emoji in one sentence, do not put a hyphen after emoji, and never end with empty choices such as 1. 2. 3.",
}, " "),
},
{
@@ -118,8 +118,8 @@ func (c *CreativeAgent) StreamRespond(ctx context.Context, req design.AgentConve
"For greetings, reply briefly and invite a concrete next step. For capability questions, answer with image-only capabilities: brand/site visuals, ecommerce product images, posters/banners/social images, reference-based generation, image edits, background removal, vectorization, mockup placement, and multi-round canvas iteration.",
"When a current project exists, connect capabilities to the canvas instead of giving a generic product brochure.",
"When the task plan indicates missing requirements or contains user response questions, ask those questions directly and stop; do not pretend generation has started.",
"Use a few purposeful emoji in visible replies to make scanning warmer: put a relevant emoji at the start of capability or next-step bullets, for example 🎨 design direction, 📦 packaging, 🖼️ canvas. Keep the tone professional and avoid empty bullet labels such as '• :'.",
"Format emoji lists as short separate lines: intro sentence, then one emoji-led item per line, for example 🎭角色设定、\\n📦系列包装、\\n🧩展示台/贴纸、\\n📣发售海报。 Do not put a hyphen after the emoji. Do not chain items like 🎨-设计...-生成..., do not start continuation lines with bare hyphens, and never end with empty choices such as 1. 2. 3.",
"Emoji are optional. Use them only when they improve scanning, especially as sparse category markers in capability or next-step replies, for example 🎨 图片生成与编辑, 🖼️ 画布迭代, 🔍 参考收集, ❌ 不在范围内.",
"When an emoji introduces a category or list item, start that emoji-led marker on its own line. Do not force every item into emoji lines, do not insert emoji inside dense prose, do not stack several emoji in one sentence, do not put a hyphen after emoji, and never end with empty choices such as 1. 2. 3.",
}, " "),
},
{
@@ -139,7 +139,7 @@ func (c *CreativeAgent) StreamPlanNarration(ctx context.Context, req design.Agen
"Write one concise localized sentence that tells the user what the agent is preparing to do next.",
"If web research results are present, mention that references have been collected and how they will guide the visual direction.",
"Do not use markdown, numbering, or generic canned wording. Use the user's language and locale automatically.",
"A single fitting emoji at the beginning is welcome when it improves the live status; do not overuse emoji.",
"Emoji are optional in live status text. Use one only when it makes a short status clearer; otherwise write plain prose.",
}, " "),
},
{
@@ -180,7 +180,7 @@ func (c *CreativeAgent) PlanAgentTask(ctx context.Context, req design.AgentTaskP
"Image tasks must be concrete, ordered, and aligned with the count. For character/design work, separate views, detail sheets, palette/style sheets, and final composite boards when requested.",
"Do not expose hidden chain-of-thought. The userFacingResponse is a short planning summary, not a long explanation.",
"If intent is chat for clarification, userFacingResponse should be the exact concise questions to ask and imageCount must be 0.",
"For userFacingResponse, use a few purposeful emoji in visible questions or summaries, especially as list markers. Use newline-separated emoji items, for example 🎭角色设定、\\n📦系列包装、\\n🧩展示台/贴纸、\\n📣发售海报。 Do not put a hyphen after the emoji, do not chain items like 🎨-设计...-生成..., do not use bare continuation hyphens, and never output empty numbered choices such as 1. 2. 3. Do not put emoji in image task titles or briefs.",
"For userFacingResponse, emoji are optional. Use them only as sparse category/list markers when they help scan multi-part questions or summaries, following the style 🎨 图片生成与编辑, 🖼️ 画布迭代, 🔍 参考收集, ❌ 不在范围内. If an emoji-led marker is used, start it on a new line; otherwise write plain prose. Do not force emoji into every item, do not chain several emoji inline, do not put a hyphen after emoji, and never output empty numbered choices such as 1. 2. 3. Do not put emoji in image task titles or briefs.",
"Use the user's language and locale automatically.",
}, " "),
},
@@ -241,7 +241,7 @@ func (c *CreativeAgent) SummarizeImage(ctx context.Context, req design.AgentImag
"Do not invent exact visual details you cannot see. Tie the response to the prompt and canvas goal.",
"When the user authorized your judgment or asked for an iteration, state the main change you attempted and what constraint it preserves. Mention a next step only when it is useful or required by the workflow.",
"Sound like a professional art director, not a template.",
"Use one fitting emoji or a short emoji-led phrase when it improves the completion message; keep it sparse.",
"Emoji are optional. Use one sparse emoji-led phrase only when it improves the completion message; otherwise write plain prose.",
}, " "),
},
{
@@ -577,6 +577,9 @@ func buildDecisionPrompt(req design.AgentDecisionRequest) string {
"",
"Input summary:",
agentMessageContentSummary(req.Messages),
"",
"Inline reference placement:",
inlineReferencePlacementContext(req.Prompt, req.Messages),
}, "\n")
}
@@ -598,6 +601,9 @@ func buildResearchDecisionPrompt(req design.AgentResearchDecisionRequest) string
req.Prompt,
agentMessageContentSummary(req.Messages),
"",
"Inline reference placement:",
inlineReferencePlacementContext(req.Prompt, req.Messages),
"",
"Mode: " + req.Mode,
}, "\n")
}
@@ -624,6 +630,9 @@ func buildAgentTaskPlanPrompt(req design.AgentTaskPlanRequest) string {
req.Prompt,
agentMessageContentSummary(req.Messages),
"",
"Inline reference placement:",
inlineReferencePlacementContext(req.Prompt, req.Messages),
"",
"Mode: " + req.Mode,
"Web search: " + searchState,
"",
@@ -648,6 +657,9 @@ func buildConversationPrompt(req design.AgentConversationRequest) string {
"Inline references:",
agentMessageContentSummary(req.Messages),
"",
"Inline reference placement:",
inlineReferencePlacementContext(req.Prompt, req.Messages),
"",
"User message:",
req.Prompt,
"",
@@ -697,6 +709,9 @@ func buildImagePromptPlannerInput(req design.AgentImagePromptRequest) string {
req.Prompt,
agentMessageContentSummary(req.Messages),
"",
"Inline reference placement:",
inlineReferencePlacementContext(req.Prompt, req.Messages),
"",
"Mode: " + req.Mode,
}, "\n")
}
@@ -284,6 +284,8 @@ func (a *DesignAgent) generatePlannedImagesIncremental(ctx context.Context, prom
func (a *DesignAgent) generateImageWithFallback(ctx context.Context, req design.AgentRequest, prompt string, fallback string, index int) (GeneratedImage, error) {
referenceImages := referenceImageURLs(req)
prompt = withReferenceImageContext(prompt, req, len(referenceImages))
fallback = withReferenceImageContext(fallback, req, len(referenceImages))
prompt = withReferenceImageInstruction(prompt, len(referenceImages))
fallback = withReferenceImageInstruction(fallback, len(referenceImages))
prompt = prepareGPTImagePrompt(prompt, imageGenerationPromptMaxRunes)
@@ -345,7 +347,7 @@ func referenceImageURLs(req design.AgentRequest) []string {
urls := make([]string, 0)
seen := make(map[string]struct{})
add := func(value string) {
value = strings.TrimSpace(strings.TrimRight(value, ",。,."))
value = sanitizeReferenceURL(value)
if value == "" {
return
}
@@ -407,6 +409,24 @@ func withReferenceImageInstruction(prompt string, imageCount int) string {
return "已随请求附带参考图;只按用户原话说明的用途使用每张图,例如商品/源图、Logo、风格、色彩、构图或编辑对象。需要编辑或延续源商品时保持其身份,不要仅凭图片内容臆测未说明的角色。\n" + prompt
}
func withReferenceImageContext(prompt string, req design.AgentRequest, imageCount int) string {
prompt = strings.TrimSpace(prompt)
if prompt == "" || imageCount <= 0 {
return prompt
}
if strings.Contains(prompt, "Inline reference placement:") || strings.Contains(prompt, "参考图位置关系") {
return prompt
}
context := inlineReferencePlacementContext(req.Prompt, req.Messages)
if context == "" || context == "none" {
return prompt
}
if prefersEnglish(prompt + "\n" + req.Prompt) {
return "Inline reference placement (attached image inputs follow this order; nearby text before or after each image defines its role, so do not swap reference roles):\n" + context + "\n\n" + prompt
}
return "参考图位置关系(附带图片按此顺序输入;每张图前后的相邻文字代表它的用途/角色,不要调换参考图角色):\n" + context + "\n\n" + prompt
}
func imagePromptTooLarge(prompt string) bool {
return utf8.RuneCountInString(strings.TrimSpace(prompt)) > imageGenerationPromptTooLargeRunes
}
@@ -476,7 +496,7 @@ var explicitPixelSizePattern = regexp.MustCompile(`(?i)([1-9]\d{2,4})\s*(?:x|×|
var explicitAspectRatioPattern = regexp.MustCompile(`(?i)(?:^|[^\d])((?:1|2|3|4|9|16)\s*:\s*(?:1|2|3|4|9|16))(?:[^\d]|$)`)
var imagePromptURLPattern = regexp.MustCompile(`https?://\S+`)
var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[(][^)\n]+[)])?\s*[:]\s*(https?://[^\s\]]+|data:image/[^\s\]]+|/[^\s\]]+)`)
var referenceImageDirectiveURLPattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[(][^)\n]+[)])?\s*[:]\s*(https?://[^\s\],。]+|data:image/[^\s\],。]+|/[^\s\],。]+)`)
var promptImageTokenURLPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\(([^)]+)\)|([^\]]+))\]`)
func fullFrameImagePrompt(prompt string) string {
@@ -260,6 +260,35 @@ func TestDesignAgentPassesPromptDirectiveReferenceImagesToImageGenerator(t *test
}
}
func TestDesignAgentPreservesInlineReferencePlacementForImageGenerator(t *testing.T) {
imageGenerator := &fakeImageGenerator{}
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
prompt := "参考图 10.jpg):https://example.com/mood.webp 为服装品牌 stillday 创建情绪板 logo 设计 参考图 2image.png):https://example.com/logo.png,包含材质、色彩、摄影姿态。"
_, err := agent.Plan(context.Background(), design.AgentRequest{
Prompt: prompt,
Mode: "image",
ImageModel: "gpt-image-2",
Messages: []design.AgentChatMessage{
{
Role: "user",
Contents: []design.AgentContent{
{Type: "text", Text: prompt, TextSource: "input"},
},
},
},
})
if err != nil {
t.Fatal(err)
}
if len(imageGenerator.opts.Images) != 2 || imageGenerator.opts.Images[0] != "https://example.com/mood.webp" || imageGenerator.opts.Images[1] != "https://example.com/logo.png" {
t.Fatalf("expected ordered reference images, got %#v", imageGenerator.opts.Images)
}
if !containsAll(imageGenerator.prompt, []string{"参考图 2", "image.png", "logo 设计"}) {
t.Fatalf("expected image prompt to carry inline placement semantics, got %s", imageGenerator.prompt)
}
}
func TestDesignAgentUsesDefaultImageSizeWithoutExplicitRequest(t *testing.T) {
imageGenerator := &fakeImageGenerator{}
agent := NewDesignAgent(fakeRunner{}, imageGenerator, fakeCreativeAgent{})
@@ -21,6 +21,7 @@ import (
"unicode/utf8"
"img_infinite_canvas/internal/domain/design"
"img_infinite_canvas/internal/infrastructure/netutil"
"github.com/zeromicro/go-zero/core/logx"
)
@@ -219,7 +220,7 @@ func (c *ImageClient) Generate(ctx context.Context, prompt string, opts ImageReq
func (c *ImageClient) imageRequestBody(ctx context.Context, model string, prompt string, size string, count int, images []string, stream bool) ([]byte, string, error) {
if len(images) > 0 {
if c.inputImageTransport == "url" {
if c.inputImageTransport == "url" && imageInputsCanUseURLTransport(images) {
return jsonImageEditBody(model, prompt, size, count, images, stream)
}
return c.multipartImageEditBody(ctx, model, prompt, size, count, images, stream)
@@ -242,6 +243,38 @@ func (c *ImageClient) imageRequestBody(ctx context.Context, model string, prompt
return body, "application/json", err
}
func imageInputsCanUseURLTransport(images []string) bool {
for _, image := range images {
if imageInputRequiresFileUpload(image) {
return false
}
}
return true
}
func imageInputRequiresFileUpload(value string) bool {
value = strings.TrimSpace(value)
if value == "" {
return false
}
if strings.HasPrefix(value, "data:image/") || strings.HasPrefix(value, "/") {
return true
}
parsed, err := url.Parse(value)
if err != nil {
return true
}
scheme := strings.ToLower(parsed.Scheme)
if scheme != "http" && scheme != "https" {
return true
}
host := strings.ToLower(parsed.Hostname())
if host == "" {
return true
}
return netutil.IsLocalOrPrivateHost(host)
}
func jsonImageEditBody(model string, prompt string, size string, count int, images []string, stream bool) ([]byte, string, error) {
requestPayload := map[string]any{
"model": model,
@@ -234,6 +234,60 @@ func TestImageClientSendsReferenceImagesByURLWhenConfigured(t *testing.T) {
}
}
func TestImageClientUploadsLocalhostReferenceImagesEvenWhenURLTransportConfigured(t *testing.T) {
var sawMultipartEdit bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/reference.png":
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write([]byte("local image bytes"))
case "/v1/images/edits":
sawMultipartEdit = true
if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
t.Errorf("expected multipart edit request for local image, got %s", r.Header.Get("Content-Type"))
}
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse multipart form: %v", err)
return
}
file, header, err := r.FormFile("image")
if err != nil {
t.Errorf("expected image file field: %v", err)
return
}
defer file.Close()
data, err := io.ReadAll(file)
if err != nil {
t.Errorf("read image file: %v", err)
return
}
if header.Filename != "reference.png" || header.Header.Get("Content-Type") != "image/png" || string(data) != "local image bytes" {
t.Errorf("unexpected uploaded image: filename=%q content_type=%q data=%q", header.Filename, header.Header.Get("Content-Type"), string(data))
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/edited.png"}]}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client := NewImageClient(ImageClientOptions{BaseURL: server.URL, APIKey: "test-key", InputImageTransport: "url"})
image, err := client.Generate(context.Background(), "edit the local image", ImageRequestOptions{
Model: "gpt-image-2",
Images: []string{server.URL + "/reference.png"},
})
if err != nil {
t.Fatal(err)
}
if image.URL != "https://example.com/edited.png" {
t.Fatalf("expected edited image, got %#v", image)
}
if !sawMultipartEdit {
t.Fatal("expected multipart image edit request")
}
}
func TestImageClientUploadsInternalReferenceImagesAsMultipart(t *testing.T) {
var sawEdit bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -0,0 +1,273 @@
package piagent
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"unicode/utf8"
"img_infinite_canvas/internal/domain/design"
)
type inlineReferenceMatch struct {
start int
end int
index int
name string
url string
}
var (
inlineReferenceDirectivePattern = regexp.MustCompile(`(?i)(参考图|reference image)\s*(\d+)?\s*(?:[(]([^)\n]+)[)])?\s*[:]\s*(https?://[^\s\],。]+|data:image/[^\s\],。]+|/[^\s\],。]+|attachment://reference-\d+)`)
inlinePromptImageTokenPattern = regexp.MustCompile(`\[@image:#(\d+):([^:\]]+):(?:\[([^\]]+)\]\(([^)]+)\)|([^\]]+))\]`)
)
func inlineReferencePlacementContext(prompt string, messages []design.AgentChatMessage) string {
messageText := agentMessageInlineReferenceText(messages)
source := prompt
if len(collectInlineReferenceMatches(messageText)) > 0 && (len(collectInlineReferenceMatches(prompt)) == 0 || utf8.RuneCountInString(messageText) >= utf8.RuneCountInString(prompt)) {
source = messageText
}
return inlineReferencePlacementContextFromText(source, prefersEnglish(prompt+"\n"+messageText))
}
func inlineReferencePlacementContextFromText(text string, english bool) string {
matches := collectInlineReferenceMatches(text)
if len(matches) == 0 {
return "none"
}
if len(matches) > 16 {
matches = matches[:16]
}
lines := make([]string, 0, len(matches)+1)
if english {
lines = append(lines, "Image input order follows the inline reference order. Nearby text before or after a reference is semantic role/context for that image.")
} else {
lines = append(lines, "图片输入顺序按参考图出现顺序;某张图前后的相邻文字是这张图的角色/用途线索。")
}
for order, match := range matches {
beforeStart := 0
if order > 0 {
beforeStart = matches[order-1].end
}
afterEnd := len(text)
if order+1 < len(matches) {
afterEnd = matches[order+1].start
}
before := compactReferenceNeighbor(text[beforeStart:match.start], true)
after := compactReferenceNeighbor(text[match.end:afterEnd], false)
if english {
lines = append(lines, englishReferencePlacementLine(order+1, match, before, after))
continue
}
lines = append(lines, chineseReferencePlacementLine(order+1, match, before, after))
}
return strings.Join(lines, "\n")
}
func agentMessageInlineReferenceText(messages []design.AgentChatMessage) string {
var builder strings.Builder
imageIndex := 0
writeText := func(text string) {
text = strings.TrimSpace(text)
if text == "" {
return
}
if builder.Len() > 0 {
builder.WriteString(" ")
}
builder.WriteString(text)
}
for _, message := range messages {
if strings.TrimSpace(message.Role) != "" && strings.ToLower(strings.TrimSpace(message.Role)) != "user" {
continue
}
for _, content := range message.Contents {
switch strings.ToLower(strings.TrimSpace(content.Type)) {
case "text":
if strings.EqualFold(strings.TrimSpace(content.TextSource), "settings") {
continue
}
writeText(stripInlineImageGeneratorDirectiveLines(content.Text))
case "image", "mask":
imageIndex++
name := compactInlineReferenceName(content.ImageName)
if name == "" {
name = "image"
}
url := sanitizeReferenceURL(content.ImageURL)
if url == "" {
url = fmt.Sprintf("attachment://reference-%d", imageIndex)
}
writeText(fmt.Sprintf("参考图 %d%s):%s", imageIndex, name, url))
}
}
}
return builder.String()
}
func collectInlineReferenceMatches(text string) []inlineReferenceMatch {
if strings.TrimSpace(text) == "" {
return nil
}
matches := make([]inlineReferenceMatch, 0)
for _, indexes := range inlinePromptImageTokenPattern.FindAllStringSubmatchIndex(text, -1) {
match := inlineReferenceMatch{start: indexes[0], end: indexes[1]}
match.index = atoiSubmatch(text, indexes, 2)
match.name = submatch(text, indexes, 4)
match.url = sanitizeReferenceURL(firstNonEmptySubmatch(text, indexes, 8, 10))
if match.url != "" {
matches = append(matches, match)
}
}
for _, indexes := range inlineReferenceDirectivePattern.FindAllStringSubmatchIndex(text, -1) {
match := inlineReferenceMatch{start: indexes[0], end: indexes[1]}
match.index = atoiSubmatch(text, indexes, 4)
match.name = submatch(text, indexes, 6)
match.url = sanitizeReferenceURL(submatch(text, indexes, 8))
if match.url != "" {
matches = append(matches, match)
}
}
sort.SliceStable(matches, func(i, j int) bool {
if matches[i].start == matches[j].start {
return matches[i].end > matches[j].end
}
return matches[i].start < matches[j].start
})
deduped := make([]inlineReferenceMatch, 0, len(matches))
for _, match := range matches {
overlaps := false
for _, existing := range deduped {
if match.start < existing.end && match.end > existing.start {
overlaps = true
break
}
}
if !overlaps {
deduped = append(deduped, match)
}
}
return deduped
}
func chineseReferencePlacementLine(order int, match inlineReferenceMatch, before string, after string) string {
parts := []string{fmt.Sprintf("- %s", referenceDisplayName(order, match, false))}
if before != "" {
parts = append(parts, fmt.Sprintf("前文“%s”", before))
}
if after != "" {
parts = append(parts, fmt.Sprintf("后文“%s”", after))
}
if len(parts) == 1 {
parts = append(parts, "无相邻文字")
}
return strings.Join(parts, "")
}
func englishReferencePlacementLine(order int, match inlineReferenceMatch, before string, after string) string {
parts := []string{fmt.Sprintf("- %s", referenceDisplayName(order, match, true))}
if before != "" {
parts = append(parts, fmt.Sprintf("previous text %q", before))
}
if after != "" {
parts = append(parts, fmt.Sprintf("next text %q", after))
}
if len(parts) == 1 {
parts = append(parts, "no adjacent text")
}
return strings.Join(parts, ": ")
}
func referenceDisplayName(order int, match inlineReferenceMatch, english bool) string {
name := compactInlineReferenceName(match.name)
if name == "" {
name = "image"
}
if english {
label := fmt.Sprintf("Reference image %d", order)
if match.index > 0 && match.index != order {
label += fmt.Sprintf(" / user label %d", match.index)
}
return fmt.Sprintf("%s (%s)", label, name)
}
label := fmt.Sprintf("参考图 %d", order)
if match.index > 0 && match.index != order {
label += fmt.Sprintf(" / 用户标注 %d", match.index)
}
return fmt.Sprintf("%s%s", label, name)
}
func compactReferenceNeighbor(value string, keepTail bool) string {
value = strings.ReplaceAll(value, "\u00a0", " ")
value = imagePromptURLPattern.ReplaceAllString(value, "[参考图]")
value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ")
value = strings.Trim(value, " \t\r\n,。,.;、")
if value == "" {
return ""
}
const maxRunes = 72
if utf8.RuneCountInString(value) <= maxRunes {
return value
}
runes := []rune(value)
if keepTail {
return "..." + string(runes[len(runes)-maxRunes:])
}
return string(runes[:maxRunes]) + "..."
}
func sanitizeReferenceURL(value string) string {
return strings.TrimSpace(strings.TrimRight(value, ",。,.;、"))
}
func compactInlineReferenceName(value string) string {
return strings.Join(strings.Fields(value), " ")
}
func stripInlineImageGeneratorDirectiveLines(text string) string {
lines := strings.Split(text, "\n")
filtered := make([]string, 0, len(lines))
for _, line := range lines {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "image generator target node:") {
continue
}
filtered = append(filtered, line)
}
return strings.TrimSpace(strings.Join(filtered, "\n"))
}
func submatch(text string, indexes []int, offset int) string {
if offset+1 >= len(indexes) || indexes[offset] < 0 || indexes[offset+1] < 0 {
return ""
}
return text[indexes[offset]:indexes[offset+1]]
}
func firstNonEmptySubmatch(text string, indexes []int, offsets ...int) string {
for _, offset := range offsets {
if value := submatch(text, indexes, offset); strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func atoiSubmatch(text string, indexes []int, offset int) int {
value := strings.TrimSpace(submatch(text, indexes, offset))
if value == "" {
return 0
}
number, err := strconv.Atoi(value)
if err != nil {
return 0
}
return number
}
@@ -2,11 +2,17 @@ package logic
import (
"fmt"
"regexp"
"strings"
"img_infinite_canvas/internal/domain/design"
)
var (
promptReferenceDirectivePattern = regexp.MustCompile(`(?i)(?:参考图|reference image)\s*\d*\s*(?:[(][^)\n]+[)])?\s*[:]\s*(?:https?://[^\s\],。]+|data:image/[^\s\],。]+|/[^\s\],。]+)`)
promptImageTokenPattern = regexp.MustCompile(`\[@image:#\d+:[^:\]]+:(?:\[[^\]]+\]\((?:[^)]+)\)|[^\]]+)\]`)
)
func validateRequiredPrompt(prompt string, allowEmpty bool) error {
if allowEmpty {
return nil
@@ -18,6 +24,8 @@ func validateRequiredPrompt(prompt string, allowEmpty bool) error {
}
func hasRequiredPromptText(prompt string) bool {
prompt = promptImageTokenPattern.ReplaceAllString(prompt, " ")
prompt = promptReferenceDirectivePattern.ReplaceAllString(prompt, " ")
for _, line := range strings.Split(prompt, "\n") {
line = strings.TrimSpace(strings.ReplaceAll(line, "\u00a0", " "))
if line == "" || isPromptReferenceDirective(line) {
@@ -0,0 +1,24 @@
package logic
import "testing"
func TestHasRequiredPromptTextAllowsInlineReferenceContext(t *testing.T) {
prompt := "参考图 10.jpg):http://localhost:19000/canvas-assets/a.webp 为服装品牌 stillday 创建情绪板 logo 设计 参考图 2image.png):http://localhost:19000/canvas-assets/b.png ,包含材质、色彩、摄影姿态。"
if !hasRequiredPromptText(prompt) {
t.Fatalf("expected inline text around reference directives to count as prompt text")
}
}
func TestHasRequiredPromptTextRejectsReferenceOnlyPrompt(t *testing.T) {
prompt := "参考图 10.jpg):http://localhost:19000/canvas-assets/a.webp\nReference image 2 (image.png): https://example.com/b.png"
if hasRequiredPromptText(prompt) {
t.Fatalf("expected reference-only prompt to be treated as empty")
}
}
func TestHasRequiredPromptTextHandlesChinesePunctuationAfterReference(t *testing.T) {
prompt := "参考图 1image.png):https://example.com/reference.png,参考它的字体做 logo。"
if !hasRequiredPromptText(prompt) {
t.Fatalf("expected text after Chinese punctuation to count as prompt text")
}
}