diff --git a/apps/desktop-client/src/main/adapters/doubao.test.ts b/apps/desktop-client/src/main/adapters/doubao.test.ts new file mode 100644 index 0000000..ef1e7ed --- /dev/null +++ b/apps/desktop-client/src/main/adapters/doubao.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; + +import { __doubaoTestUtils } from "./doubao"; + +const { + buildSourceItem, + dedupeSourceItems, + isNonCitationAssetDomain, + isNonCitationAssetUrl, + parseDoubaoStreamBody, +} = __doubaoTestUtils; + +describe("doubao adapter helpers", () => { + it("filters byteimg and bytednsdoc asset CDN links from source items", () => { + expect(isNonCitationAssetDomain("p3-spider-image-sign.byteimg.com")).toBe(true); + expect(isNonCitationAssetDomain("lf3-static.bytednsdoc.com")).toBe(true); + expect(isNonCitationAssetUrl("https://p11-spider-image-sign.byteimg.com/obj/logo.png")).toBe(true); + expect(isNonCitationAssetUrl("https://m.sohu.com/a/1012750625_122649842/")).toBe(false); + + expect(buildSourceItem({ + url: "https://p3-spider-image-sign.byteimg.com/img/logo.jpeg", + title: "logo", + site_name: "byteimg.com", + })).toBeNull(); + expect(buildSourceItem({ + url: "https://example.com/article", + title: "真实来源", + })).toMatchObject({ + url: "https://example.com/article", + host: "example.com", + }); + }); + + it("dedupes and drops asset CDN links parsed from Doubao streams", () => { + const body = [ + "event: STREAM_CHUNK", + "data: {\"references\":[{\"url\":\"https://p11-spider-image-sign.byteimg.com/logo.jpeg\",\"title\":\"logo\"},{\"url\":\"https://m.sohu.com/a/1012750625_122649842/\",\"title\":\"真实来源\"}],\"text_block\":{\"text\":\"回答\"}}", + "", + "", + ].join("\n"); + + const summary = parseDoubaoStreamBody(body); + expect(summary.search_results).toHaveLength(1); + expect(summary.search_results[0]?.url).toBe("https://m.sohu.com/a/1012750625_122649842/"); + + expect(dedupeSourceItems([ + { url: "https://static.bytednsdoc.com/logo.png", normalized_url: "https://static.bytednsdoc.com/logo.png" }, + { url: "https://example.com/a", normalized_url: "https://example.com/a" }, + ])).toHaveLength(1); + }); +}); diff --git a/apps/desktop-client/src/main/adapters/doubao.ts b/apps/desktop-client/src/main/adapters/doubao.ts index bb5b989..c348092 100644 --- a/apps/desktop-client/src/main/adapters/doubao.ts +++ b/apps/desktop-client/src/main/adapters/doubao.ts @@ -11,6 +11,7 @@ const DOUBAO_CAPTURE_LIMIT = 64; const DOUBAO_CAPTURE_BODY_LIMIT = 1_200_000; const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = 20_000; const DOUBAO_STREAM_URL_FRAGMENT = "/chat/completion"; +const DOUBAO_NON_CITATION_ASSET_DOMAINS = new Set(["byteimg.com", "bytednsdoc.com"]); const SEARCH_RESULT_KEYS = new Set([ "references", @@ -393,6 +394,46 @@ function normalizeSourceUrl(value: string | null): string | null { } } +function normalizeSourceHost(value: string | null | undefined): string | null { + const input = normalizeText(value); + if (!input) { + return null; + } + try { + return new URL(/^(https?:)?\/\//i.test(input) ? input : `https://${input}`).hostname.toLowerCase(); + } catch { + return input + .replace(/^https?:\/\//i, "") + .replace(/^\/\//, "") + .split(/[/?#:]/)[0] + ?.trim() + .toLowerCase() || null; + } +} + +function isNonCitationAssetDomain(value: string | null | undefined): boolean { + const host = normalizeSourceHost(value); + if (!host) { + return false; + } + if (DOUBAO_NON_CITATION_ASSET_DOMAINS.has(host)) { + return true; + } + return Array.from(DOUBAO_NON_CITATION_ASSET_DOMAINS).some((domain) => host.endsWith(`.${domain}`)); +} + +function isNonCitationAssetUrl(value: string | null | undefined): boolean { + const url = normalizeSourceUrl(value ?? null) ?? normalizeText(value); + if (!url) { + return false; + } + try { + return isNonCitationAssetDomain(new URL(url).hostname); + } catch { + return isNonCitationAssetDomain(url); + } +} + function resolveSourceBucket(key: string): "search" | null { const normalized = key.replace(/[^a-z0-9]/gi, "").toLowerCase(); if (!normalized) { @@ -445,6 +486,9 @@ function buildSourceItem(input: unknown): MonitoringSourceItem | null { } catch { host = null; } + if (isNonCitationAssetDomain(host) || isNonCitationAssetUrl(url)) { + return null; + } return { url, @@ -462,7 +506,7 @@ function dedupeSourceItems(items: MonitoringSourceItem[]): MonitoringSourceItem[ const keyed = new Map(); for (const item of items) { const key = normalizeSourceUrl(item.normalized_url ?? item.url); - if (!key) { + if (!key || isNonCitationAssetDomain(item.host) || isNonCitationAssetUrl(key)) { continue; } @@ -2144,3 +2188,11 @@ export const doubaoAdapter: MonitorAdapter = { }; }, }; + +export const __doubaoTestUtils = { + buildSourceItem, + dedupeSourceItems, + isNonCitationAssetDomain, + isNonCitationAssetUrl, + parseDoubaoStreamBody, +};