diff --git a/apps/desktop-client/src/main/adapters/wenxin.test.ts b/apps/desktop-client/src/main/adapters/wenxin.test.ts index 9232001..b9b5aad 100644 --- a/apps/desktop-client/src/main/adapters/wenxin.test.ts +++ b/apps/desktop-client/src/main/adapters/wenxin.test.ts @@ -3,8 +3,10 @@ import { describe, expect, it } from "vitest"; import { __wenxinTestUtils } from "./wenxin"; const { + buildSourceItem, htmlTableToMarkdown, isObservationComplete, + normalizeUrl, parseWenxinHistoryResponse, parseWenxinSSEBody, } = __wenxinTestUtils; @@ -163,6 +165,62 @@ describe("wenxin adapter helpers", () => { expect(history.citations[0]?.url).toBe("https://fallback.example.com/article"); }); + it("unwraps Wenxin reference redirect urls", () => { + expect(normalizeUrl("https://yiyan.baidu.com/eb/link?target=https%3A%2F%2Fwww.example.com%2Farticle%23ref")).toBe( + "https://www.example.com/article", + ); + + expect(buildSourceItem({ + pageUrl: "https://yiyan.baidu.com/eb/link?url=https%3A%2F%2Fsource.example.com%2Fnews%3Fid%3D1", + title: "引用来源标题", + source: "来源站点", + })).toMatchObject({ + url: "https://source.example.com/news?id=1", + title: "引用来源标题", + site_name: "来源站点", + host: "source.example.com", + }); + }); + + it("extracts web page references from Wenxin reference list fields", () => { + const history = parseWenxinHistoryResponse({ + code: 0, + msg: "success", + data: { + state: 1, + chats: [ + { + id: "bot-1", + role: "robot", + stop: 1, + modelSign: "EB45T", + message: [ + { + contentType: "text", + content: "这是答案", + }, + ], + webPages: [ + { + targetUrl: "https://yiyan.baidu.com/eb/link?target=https%3A%2F%2Fwww.reference-source.com%2Fa%23section", + title: "参考网页 A", + siteName: "reference-source.com", + }, + ], + }, + ], + }, + }, "session-3"); + + expect(history.citations).toHaveLength(1); + expect(history.searchResults).toHaveLength(1); + expect(history.citations[0]).toMatchObject({ + url: "https://www.reference-source.com/a", + title: "参考网页 A", + site_name: "reference-source.com", + }); + }); + it("does not treat error-only observations as complete", () => { expect(isObservationComplete({ sessionId: "session-1", diff --git a/apps/desktop-client/src/main/adapters/wenxin.ts b/apps/desktop-client/src/main/adapters/wenxin.ts index 5d28fe9..b26e655 100644 --- a/apps/desktop-client/src/main/adapters/wenxin.ts +++ b/apps/desktop-client/src/main/adapters/wenxin.ts @@ -29,8 +29,28 @@ const WENXIN_LOGIN_REQUIRED_PATTERN = /(请先登录|未登录|登录后|登录 const WENXIN_PLACEHOLDER_PATTERN = /^(?:正在生成中(?:\.\.\.)?|思考中(?:\.\.\.)?|内容由AI生成,仅供参考,请仔细甄别|全选)$/; const WENXIN_TABLE_HTML_PATTERN = //gi; const WENXIN_REFERENCE_TRIGGER_PATTERN = /(参考\s*\d+\s*(?:个网页|条网页信息源)|found\s*\d+\s*web pages)/i; -const WENXIN_CITATION_FIELD_PATTERN = /^(?:citations?|citationInfo|pluginCitations?|pluginCitation|reference(?:s|List|_list|Cards|_cards|Docs|_docs)?|source(?:s|List|_list|Cards|_cards)?)$/i; -const WENXIN_SEARCH_FIELD_PATTERN = /^(?:searchCitations?|search_results?|searchResultList|search_result_list|webListData|webList|browseReferences?|browse_results?|grounding(?:_results?)?)$/i; +const WENXIN_CITATION_FIELD_PATTERN = /^(?:citations?|citationInfo|pluginCitations?|pluginCitation|reference(?:s|List|_list|Cards|_cards|Docs|_docs|Info|_info)?|source(?:s|List|_list|Cards|_cards|Info|_info)?|web(?:References?|Sources?|Pages?|Docs?|Results?)|page(?:References?|Sources?))$/i; +const WENXIN_SEARCH_FIELD_PATTERN = /^(?:searchCitations?|search_results?|searchResultList|search_result_list|webListData|webList|web(?:References?|Sources?|Pages?|Docs?|Results?)|browseReferences?|browse_results?|grounding(?:_results?)?)$/i; +const WENXIN_REDIRECT_QUERY_KEYS = [ + "url", + "u", + "target", + "targetUrl", + "target_url", + "link", + "href", + "sourceUrl", + "source_url", + "originUrl", + "origin_url", + "realUrl", + "real_url", + "redirect", + "redirect_url", + "jump", + "jump_url", + "to", +]; type WenxinPageSourceLink = { url: string; @@ -165,19 +185,85 @@ function normalizeBlockText(value: string | null): string | null { .trim() || null; } +function decodeUrlCandidate(value: string): string { + let decoded = value.trim(); + for (let index = 0; index < 3; index += 1) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) { + break; + } + decoded = next; + } catch { + break; + } + } + return decoded; +} + +function coercePotentialUrl(value: string): string { + const trimmed = decodeUrlCandidate(value); + if (/^\/\//.test(trimmed)) { + return `https:${trimmed}`; + } + if (/^[a-z0-9-]+(?:\.[a-z0-9-]+)+(?:[/?#].*)?$/i.test(trimmed)) { + return `https://${trimmed}`; + } + return trimmed; +} + +function normalizeAbsoluteUrl(value: string): string | null { + try { + const url = new URL(coercePotentialUrl(value)); + url.hash = ""; + return url.toString(); + } catch { + return null; + } +} + +function extractRedirectTargetUrl(value: string): string | null { + const direct = normalizeAbsoluteUrl(value); + if (!direct) { + return null; + } + + try { + const url = new URL(direct); + for (const key of WENXIN_REDIRECT_QUERY_KEYS) { + const raw = url.searchParams.get(key); + if (!raw) { + continue; + } + const normalized = normalizeAbsoluteUrl(raw); + if (normalized) { + return normalized; + } + } + } catch { + return null; + } + + const encodedMatch = direct.match(/https?%3A%2F%2F[^&\s"'<>]+/i); + if (encodedMatch) { + return normalizeAbsoluteUrl(encodedMatch[0]); + } + + const plainMatch = direct.match(/https?:\/\/[^&\s"'<>]+/i); + if (plainMatch && plainMatch[0] !== direct) { + return normalizeAbsoluteUrl(plainMatch[0]); + } + + return null; +} + function normalizeUrl(value: unknown): string | null { const input = normalizeText(value); if (!input) { return null; } - try { - const url = new URL(input); - url.hash = ""; - return url.toString(); - } catch { - return null; - } + return extractRedirectTargetUrl(input) ?? normalizeAbsoluteUrl(input); } function isLikelyWenxinSourceUrl(url: string | null): boolean { @@ -351,6 +437,14 @@ function buildSourceItem(input: unknown): MonitoringSourceItem | null { input.url ?? input.href ?? input.link + ?? input.pageUrl + ?? input.page_url + ?? input.webUrl + ?? input.web_url + ?? input.realUrl + ?? input.real_url + ?? input.displayUrl + ?? input.display_url ?? input.originUrl ?? input.origin_url ?? input.targetUrl @@ -1100,7 +1194,7 @@ async function ensurePageOnWenxinChat(page: PlaywrightPage, signal: AbortSignal) async function readWenxinPageSnapshot(page: PlaywrightPage): Promise { return await page.evaluate( - ({ editorSelector, loginSignals, triggerPatternSource, citationFieldPatternSource, searchFieldPatternSource }) => { + ({ editorSelector, loginSignals, triggerPatternSource, citationFieldPatternSource, searchFieldPatternSource, redirectQueryKeys }) => { const isObjectRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); @@ -1116,6 +1210,73 @@ async function readWenxinPageSnapshot(page: PlaywrightPage): Promise { + let decoded = value.trim(); + for (let index = 0; index < 3; index += 1) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) { + break; + } + decoded = next; + } catch { + break; + } + } + return decoded; + }; + + const coercePotentialUrl = (value: string): string => { + const trimmed = decodeUrlCandidate(value); + if (/^\/\//.test(trimmed)) { + return `https:${trimmed}`; + } + if (/^[a-z0-9-]+(?:\.[a-z0-9-]+)+(?:[/?#].*)?$/i.test(trimmed)) { + return `https://${trimmed}`; + } + return trimmed; + }; + + const normalizeAbsoluteHref = (value: string): string | null => { + try { + const url = new URL(coercePotentialUrl(value), window.location.href); + url.hash = ""; + return url.toString(); + } catch { + return null; + } + }; + + const extractRedirectTargetHref = (value: string): string | null => { + const direct = normalizeAbsoluteHref(value); + if (!direct) { + return null; + } + + try { + const url = new URL(direct); + for (const key of redirectQueryKeys) { + const raw = url.searchParams.get(key); + if (!raw) { + continue; + } + const normalized = normalizeAbsoluteHref(raw); + if (normalized) { + return normalized; + } + } + } catch { + return null; + } + + const encodedMatch = direct.match(/https?%3A%2F%2F[^&\s"'<>]+/i); + if (encodedMatch) { + return normalizeAbsoluteHref(encodedMatch[0]); + } + + return null; + }; + const normalizeHref = (value: unknown): string | null => { if (typeof value !== "string") { return null; @@ -1126,13 +1287,7 @@ async function readWenxinPageSnapshot(page: PlaywrightPage): Promise { @@ -1169,6 +1324,14 @@ async function readWenxinPageSnapshot(page: PlaywrightPage): Promise { + if (!value) { + return null; + } + + const httpMatch = value.match(/https?:\/\/[^\s"'<>,。;、]+/i); + if (httpMatch) { + return normalizeHref(httpMatch[0]); + } + + const domainMatch = value.match(/\b[a-z0-9-]+(?:\.[a-z0-9-]+)+(?:\/[^\s"'<>,。;、]*)?/i); + return domainMatch ? normalizeHref(domainMatch[0]) : null; + }; + + const readElementUrl = (element: Element): string | null => { + const candidates: string[] = []; + for (const name of urlAttributeNames) { + const value = element.getAttribute(name); + if (value) { + candidates.push(value); + } + } + + if (element instanceof HTMLElement) { + candidates.push(...Object.values(element.dataset).filter((value): value is string => Boolean(value))); + } + + for (const candidate of candidates) { + const url = normalizeHref(candidate) ?? firstUrlFromText(candidate); + if (isLikelySourceUrl(url)) { + return url; + } + } + + const text = normalize(element.textContent ?? ""); + if (text && text.length <= 2_000) { + const url = firstUrlFromText(text); + if (isLikelySourceUrl(url)) { + return url; + } + } + + return null; + }; + + const normalizeShortText = (value: unknown, maxLength = 220): string | null => { + const text = normalize(value); + if (!text) { + return null; + } + return text.length <= maxLength ? text : null; + }; + + const extractElementSourceLink = (element: Element) => { + if (!isVisible(element)) { + return null; + } + + const anchorLink = extractAnchorLink(element); + if (anchorLink) { + return anchorLink; + } + + const url = readElementUrl(element); + if (!isLikelySourceUrl(url)) { + return null; + } + + const card = element.closest([ + "a", + "li", + "article", + "section", + "[role=\"listitem\"]", + "[class*=\"item\"]", + "[class*=\"Item\"]", + "[class*=\"card\"]", + "[class*=\"Card\"]", + "[class*=\"source\"]", + "[class*=\"Source\"]", + "[class*=\"reference\"]", + "[class*=\"Reference\"]", + "[class*=\"web\"]", + "[class*=\"Web\"]", + ].join(",")) ?? element; + + const title = normalizeShortText( + card.querySelector("[class*=\"title\"], [class*=\"Title\"], h1, h2, h3, h4")?.textContent + ?? element.getAttribute("title") + ?? element.getAttribute("aria-label") + ?? card.textContent, + ); + const siteName = normalizeShortText( + card.querySelector("[class*=\"site\"], [class*=\"Site\"], [class*=\"domain\"], [class*=\"Domain\"], [class*=\"source\"], [class*=\"Source\"]")?.textContent + ?? element.getAttribute("data-site-name") + ?? (element instanceof HTMLElement ? element.dataset.siteName : null), + 80, + ); + + return { + url, + title, + text: title ?? siteName, + siteName, + }; + }; + + const collectSourceLinksFromElement = (root: HTMLElement) => { + const candidates = [ + root, + ...Array.from(root.querySelectorAll([ + "a[href]", + "[data-href]", + "[data-url]", + "[data-link]", + "[data-target-url]", + "[data-source-url]", + "[data-origin-url]", + "[data-real-url]", + "[data-page-url]", + "[data-web-url]", + "[data-log]", + "[data-click]", + "[data-extra]", + "[data-info]", + "li", + "article", + "[role=\"listitem\"]", + "[class*=\"item\"]", + "[class*=\"Item\"]", + "[class*=\"card\"]", + "[class*=\"Card\"]", + "[class*=\"source\"]", + "[class*=\"Source\"]", + "[class*=\"reference\"]", + "[class*=\"Reference\"]", + "[class*=\"web\"]", + "[class*=\"Web\"]", + ].join(","))), + ]; + + return dedupeLinks( + candidates + .map((node) => extractElementSourceLink(node)) + .filter((node): node is NonNullable => Boolean(node)), + ); + }; + const collectVisiblePanelLinks = (triggerText: string | null) => { const panelSelectors = [ + "[role=\"dialog\"]", + "[role=\"list\"]", + "[class*=\"modal\"]", + "[class*=\"Modal\"]", + "[class*=\"dialog\"]", + "[class*=\"Dialog\"]", "[class*=\"drawer\"]", + "[class*=\"Drawer\"]", "[class*=\"panel\"]", + "[class*=\"Panel\"]", "[class*=\"popover\"]", + "[class*=\"Popover\"]", + "[class*=\"popup\"]", + "[class*=\"Popup\"]", "[class*=\"webList\"]", + "[class*=\"WebList\"]", "[class*=\"reference\"]", + "[class*=\"Reference\"]", "[class*=\"material\"]", "[class*=\"cite\"]", + "[class*=\"source\"]", + "[class*=\"Source\"]", + "[class*=\"web\"]", + "[class*=\"Web\"]", "aside", "section", "article", + "body > div", ]; + const rootLinks = new Map>(); const roots = Array.from(document.querySelectorAll(panelSelectors.join(","))) .filter((element): element is HTMLElement => isVisible(element)) .filter((element) => { const text = normalize(element.innerText) ?? ""; - const externalAnchorCount = Array.from(element.querySelectorAll("a[href]")) - .filter((node) => extractAnchorLink(node) !== null) - .length; - return externalAnchorCount > 0 + const className = String(element.className || ""); + const links = collectSourceLinksFromElement(element); + rootLinks.set(element, links); + return links.length > 0 && ( /参考|网页|信息源|引用|web pages|references/i.test(text) || Boolean(triggerText && text.includes(triggerText)) + || element.getAttribute("role") === "dialog" + || /(drawer|panel|popover|popup|modal|dialog|web|source|reference|cite)/i.test(className) ); }); const links = roots.length - ? roots.flatMap((root) => - Array.from(root.querySelectorAll("a[href]")) - .map((node) => extractAnchorLink(node)) - .filter((node): node is NonNullable => Boolean(node))) + ? roots.flatMap((root) => rootLinks.get(root) ?? collectSourceLinksFromElement(root)) : []; return dedupeLinks(links); @@ -1464,6 +1813,7 @@ async function readWenxinPageSnapshot(page: PlaywrightPage): Promise isVisible(element)) - .find((element) => { + .filter((element) => { const text = normalize(element.innerText || element.textContent || ""); return Boolean(text && triggerPattern.test(text)); }); + const candidates = rawCandidates + .map((element) => { + const target = element.closest("button, [role=\"button\"], a, [tabindex]") ?? element; + if (!(target instanceof HTMLElement) || !isVisible(target)) { + return null; + } + + const rect = target.getBoundingClientRect(); + const text = normalize(target.innerText || target.textContent || "") ?? ""; + const style = window.getComputedStyle(target); + const className = String(target.className || ""); + let score = 0; + if (target.matches("button, [role=\"button\"], a, [tabindex]")) { + score += 80; + } + if (style.cursor === "pointer" || typeof target.onclick === "function") { + score += 40; + } + if (/(reference|cite|source|web)/i.test(className)) { + score += 30; + } + if (text.length <= 80) { + score += 20; + } + score -= Math.min(rect.width * rect.height / 10_000, 50); + return { target, score }; + }) + .filter((item): item is { target: HTMLElement; score: number } => Boolean(item)); + + const seen = new Set(); + const trigger = candidates + .sort((left, right) => right.score - left.score) + .map((item) => item.target) + .find((target) => { + if (seen.has(target)) { + return false; + } + seen.add(target); + return true; + }); if (!trigger) { return false; } + trigger.scrollIntoView({ block: "center", inline: "center" }); trigger.click(); return true; }, { @@ -1518,7 +1909,8 @@ async function maybeOpenWenxinReferencePanel(page: PlaywrightPage): Promise false); if (opened) { - await page.waitForTimeout(350).catch(() => undefined); + await page.waitForTimeout(1_000).catch(() => undefined); + await page.waitForLoadState("networkidle", { timeout: 1_500 }).catch(() => undefined); } } @@ -2153,8 +2545,10 @@ export const wenxinAdapter: MonitorAdapter = { }; export const __wenxinTestUtils = { + buildSourceItem, htmlTableToMarkdown, isObservationComplete, + normalizeUrl, parseWenxinHistoryResponse, parseWenxinSSEBody, };