From 7ca79723cd30f4ed0b2a808b2fccbff8e5dda9d4 Mon Sep 17 00:00:00 2001 From: liangxu Date: Wed, 29 Apr 2026 17:11:04 +0800 Subject: [PATCH] fix(desktop/doubao): drop byteimg and bytednsdoc CDN URLs from cited sources These ByteDance asset CDNs ship images and static docs that are not real citations, so they pollute the monitoring source list. Filter them at both buildSourceItem and dedupe stages, and add unit coverage via an exported test-only helper bag. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/main/adapters/doubao.test.ts | 51 ++++++++++++++++++ .../src/main/adapters/doubao.ts | 54 ++++++++++++++++++- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 apps/desktop-client/src/main/adapters/doubao.test.ts diff --git a/apps/desktop-client/src/main/adapters/doubao.test.ts b/apps/desktop-client/src/main/adapters/doubao.test.ts new file mode 100644 index 0000000..ef1e7ed --- /dev/null +++ b/apps/desktop-client/src/main/adapters/doubao.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; + +import { __doubaoTestUtils } from "./doubao"; + +const { + buildSourceItem, + dedupeSourceItems, + isNonCitationAssetDomain, + isNonCitationAssetUrl, + parseDoubaoStreamBody, +} = __doubaoTestUtils; + +describe("doubao adapter helpers", () => { + it("filters byteimg and bytednsdoc asset CDN links from source items", () => { + expect(isNonCitationAssetDomain("p3-spider-image-sign.byteimg.com")).toBe(true); + expect(isNonCitationAssetDomain("lf3-static.bytednsdoc.com")).toBe(true); + expect(isNonCitationAssetUrl("https://p11-spider-image-sign.byteimg.com/obj/logo.png")).toBe(true); + expect(isNonCitationAssetUrl("https://m.sohu.com/a/1012750625_122649842/")).toBe(false); + + expect(buildSourceItem({ + url: "https://p3-spider-image-sign.byteimg.com/img/logo.jpeg", + title: "logo", + site_name: "byteimg.com", + })).toBeNull(); + expect(buildSourceItem({ + url: "https://example.com/article", + title: "真实来源", + })).toMatchObject({ + url: "https://example.com/article", + host: "example.com", + }); + }); + + it("dedupes and drops asset CDN links parsed from Doubao streams", () => { + const body = [ + "event: STREAM_CHUNK", + "data: {\"references\":[{\"url\":\"https://p11-spider-image-sign.byteimg.com/logo.jpeg\",\"title\":\"logo\"},{\"url\":\"https://m.sohu.com/a/1012750625_122649842/\",\"title\":\"真实来源\"}],\"text_block\":{\"text\":\"回答\"}}", + "", + "", + ].join("\n"); + + const summary = parseDoubaoStreamBody(body); + expect(summary.search_results).toHaveLength(1); + expect(summary.search_results[0]?.url).toBe("https://m.sohu.com/a/1012750625_122649842/"); + + expect(dedupeSourceItems([ + { url: "https://static.bytednsdoc.com/logo.png", normalized_url: "https://static.bytednsdoc.com/logo.png" }, + { url: "https://example.com/a", normalized_url: "https://example.com/a" }, + ])).toHaveLength(1); + }); +}); diff --git a/apps/desktop-client/src/main/adapters/doubao.ts b/apps/desktop-client/src/main/adapters/doubao.ts index bb5b989..c348092 100644 --- a/apps/desktop-client/src/main/adapters/doubao.ts +++ b/apps/desktop-client/src/main/adapters/doubao.ts @@ -11,6 +11,7 @@ const DOUBAO_CAPTURE_LIMIT = 64; const DOUBAO_CAPTURE_BODY_LIMIT = 1_200_000; const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = 20_000; const DOUBAO_STREAM_URL_FRAGMENT = "/chat/completion"; +const DOUBAO_NON_CITATION_ASSET_DOMAINS = new Set(["byteimg.com", "bytednsdoc.com"]); const SEARCH_RESULT_KEYS = new Set([ "references", @@ -393,6 +394,46 @@ function normalizeSourceUrl(value: string | null): string | null { } } +function normalizeSourceHost(value: string | null | undefined): string | null { + const input = normalizeText(value); + if (!input) { + return null; + } + try { + return new URL(/^(https?:)?\/\//i.test(input) ? input : `https://${input}`).hostname.toLowerCase(); + } catch { + return input + .replace(/^https?:\/\//i, "") + .replace(/^\/\//, "") + .split(/[/?#:]/)[0] + ?.trim() + .toLowerCase() || null; + } +} + +function isNonCitationAssetDomain(value: string | null | undefined): boolean { + const host = normalizeSourceHost(value); + if (!host) { + return false; + } + if (DOUBAO_NON_CITATION_ASSET_DOMAINS.has(host)) { + return true; + } + return Array.from(DOUBAO_NON_CITATION_ASSET_DOMAINS).some((domain) => host.endsWith(`.${domain}`)); +} + +function isNonCitationAssetUrl(value: string | null | undefined): boolean { + const url = normalizeSourceUrl(value ?? null) ?? normalizeText(value); + if (!url) { + return false; + } + try { + return isNonCitationAssetDomain(new URL(url).hostname); + } catch { + return isNonCitationAssetDomain(url); + } +} + function resolveSourceBucket(key: string): "search" | null { const normalized = key.replace(/[^a-z0-9]/gi, "").toLowerCase(); if (!normalized) { @@ -445,6 +486,9 @@ function buildSourceItem(input: unknown): MonitoringSourceItem | null { } catch { host = null; } + if (isNonCitationAssetDomain(host) || isNonCitationAssetUrl(url)) { + return null; + } return { url, @@ -462,7 +506,7 @@ function dedupeSourceItems(items: MonitoringSourceItem[]): MonitoringSourceItem[ const keyed = new Map(); for (const item of items) { const key = normalizeSourceUrl(item.normalized_url ?? item.url); - if (!key) { + if (!key || isNonCitationAssetDomain(item.host) || isNonCitationAssetUrl(key)) { continue; } @@ -2144,3 +2188,11 @@ export const doubaoAdapter: MonitorAdapter = { }; }, }; + +export const __doubaoTestUtils = { + buildSourceItem, + dedupeSourceItems, + isNonCitationAssetDomain, + isNonCitationAssetUrl, + parseDoubaoStreamBody, +};