root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(monitoring): reject succeeded results missing answers or citation sources

Browse Source 
Server callback now rejects succeeded submissions without a non-empty answer,
or whose answer contains [citation:N] markers without any sources. Desktop
adapters mirror this client-side: kimi returns unknown when sources arrive
without a final answer, and yuanbao returns unknown when the answer has
unresolved citation markers. Yuanbao also unwraps redirect URLs, harvests
citations from more DOM attributes and document URL fields, drops links
pointing back at yuanbao itself, and dedupes overlapping streaming
fragments. Runtime controller forwards only an explicit "succeeded" status
as success; anything else becomes failed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-04-28 22:07:21 +08:00
parent 453a374a49
commit a6b17bae62
6 changed files with 312 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop-client/src/main/adapters/kimi.ts
+6 -4
View File
@@ -2918,16 +2918,16 @@ export const kimiAdapter: MonitorAdapter = {
};
}
if (!answer && !searchResults.length && !citations.length) {
if (!answer) {
const timedOut = !queryResult.ok && queryResult.error === "kimi_query_timeout";
const hasReasoningOnly = Boolean(reasoning);
if (timedOut || hasReasoningOnly) {
if (timedOut || hasReasoningOnly || searchResults.length > 0 || citations.length > 0) {
const incompleteDetail = !queryResult.ok
? queryResult.detail ?? queryResult.error
: "kimi returned no final answer before completion";
: "kimi returned sources without a final answer";
return {
status: "unknown",
summary: "Kimi 超时未拿到完整答案，已回写 unknown 等待后续补采。",
summary: "Kimi 未拿到完整答案，已回写 unknown 等待后续补采。",
error: buildAdapterError(
timedOut ? queryResult.error : "kimi_incomplete_response",
incompleteDetail,
@@ -2935,6 +2935,8 @@ export const kimiAdapter: MonitorAdapter = {
page_url: finalSnapshot.url,
current_model_label: finalSnapshot.currentModelLabel,
reasoning_present: Boolean(reasoning),
citation_count: citations.length,
search_result_count: searchResults.length,
busy: finalSnapshot.busy,
},
),
apps/desktop-client/src/main/adapters/yuanbao.test.ts
+52
View File
@@ -0,0 +1,52 @@
import { describe, expect, it } from "vitest";
import { __yuanbaoTestUtils } from "./yuanbao";
const {
buildSourceItem,
extractCitationMarkerIndexes,
mergeText,
normalizeSourceUrl,
resolveCitationsFromMarkers,
} = __yuanbaoTestUtils;
describe("yuanbao adapter helpers", () => {
it("unwraps redirect URLs and reads Yuanbao document URL fields", () => {
expect(normalizeSourceUrl("https://yuanbao.tencent.com/link?target=https%3A%2F%2Fexample.com%2Fa%23ref")).toBe(
"https://example.com/a",
);
expect(buildSourceItem({
docUrl: "https://example.com/doc?from=yuanbao#source",
title: "引用文章",
siteName: "Example",
})).toMatchObject({
url: "https://example.com/doc?from=yuanbao",
normalized_url: "https://example.com/doc?from=yuanbao",
title: "引用文章",
site_name: "Example",
host: "example.com",
});
});
it("merges streaming answer fragments without duplicating a repeated prefix", () => {
const first = "通过三轮搜索，我已经掌握了合肥全屋定制市场的高性价比品牌。一定要合同中的定标准确和责任。";
const repeated = "通过三轮搜索，我已经掌握了合肥全屋定制市场的高性价比品牌。结合你的预算，我建议优先看本土老牌。";
expect(mergeText(first, repeated)).toBe(repeated);
});
it("resolves citation markers against the best available source pool", () => {
const sources = [
{ url: "https://example.com/1", normalized_url: "https://example.com/1" },
{ url: "https://example.com/2", normalized_url: "https://example.com/2" },
{ url: "https://example.com/3", normalized_url: "https://example.com/3" },
];
expect(extractCitationMarkerIndexes("正文 [citation:1] 和 [citation:3]")).toEqual([1, 3]);
expect(resolveCitationsFromMarkers("正文 [citation:1] 和 [citation:3]", [[], sources]).map((item) => item.url)).toEqual([
"https://example.com/1",
"https://example.com/3",
]);
});
});
apps/desktop-client/src/main/adapters/yuanbao.ts
+190 -15
View File
@@ -365,6 +365,25 @@ function resolveCitationsFromMarkers(
.filter((item): item is MonitoringSourceItem => item !== null);
}
function sharedBoundaryLength(left: string, right: string): number {
const maxLength = Math.min(left.length, right.length);
for (let length = maxLength; length >= 8; length -= 1) {
if (left.slice(left.length - length) === right.slice(0, length)) {
return length;
}
}
return 0;
}
function sharedPrefixLength(left: string, right: string): number {
const maxLength = Math.min(left.length, right.length);
let length = 0;
while (length < maxLength && left[length] === right[length]) {
length += 1;
}
return length;
}
function mergeText(current: string | null, nextFragment: string | null): string | null {
if (!nextFragment) {
return current;
@@ -378,6 +397,23 @@ function mergeText(current: string | null, nextFragment: string | null): string
if (nextFragment.startsWith(current)) {
return nextFragment;
}
const forwardOverlap = sharedBoundaryLength(current, nextFragment);
if (forwardOverlap > 0) {
return `${current}${nextFragment.slice(forwardOverlap)}`;
}
const reverseOverlap = sharedBoundaryLength(nextFragment, current);
if (reverseOverlap > 0) {
return `${nextFragment}${current.slice(reverseOverlap)}`;
}
const commonPrefixLength = sharedPrefixLength(current, nextFragment);
const duplicatePrefixThreshold = Math.max(16, Math.min(80, Math.floor(Math.min(current.length, nextFragment.length) * 0.45)));
if (commonPrefixLength >= duplicatePrefixThreshold) {
return answerQualityScore(nextFragment) > answerQualityScore(current) ? nextFragment : current;
}
return `${current}${nextFragment}`;
}
@@ -399,6 +435,28 @@ function normalizeSourceUrl(value: string | null): string | null {
try {
const url = new URL(input);
for (const key of [
"url",
"u",
"target",
"target_url",
"targetUrl",
"redirect",
"redirect_url",
"redirectUrl",
"jump_url",
"jumpUrl",
"link",
]) {
const nested = normalizeText(url.searchParams.get(key));
if (!nested || nested === input || !/^https?:\/\//i.test(nested)) {
continue;
}
const unwrapped = normalizeSourceUrl(nested);
if (unwrapped) {
return unwrapped;
}
}
url.hash = "";
return url.toString();
} catch {
@@ -406,6 +464,15 @@ function normalizeSourceUrl(value: string | null): string | null {
}
}
function isYuanbaoInternalUrl(value: string): boolean {
try {
const host = new URL(value).hostname.toLowerCase();
return host === "yuanbao.tencent.com" || host.endsWith(".yuanbao.tencent.com");
} catch {
return false;
}
}
function buildSourceItem(input: unknown): MonitoringSourceItem | null {
if (typeof input === "string") {
const url = normalizeSourceUrl(input);
@@ -430,9 +497,32 @@ function buildSourceItem(input: unknown): MonitoringSourceItem | null {
"jumpUrl",
"target_url",
"targetUrl",
"target",
"doc_url",
"docUrl",
"document_url",
"documentUrl",
"article_url",
"articleUrl",
"web_url",
"webUrl",
"webpage_url",
"webpageUrl",
"url_pc",
"urlPc",
"pc_url",
"pcUrl",
"mobile_url",
"mobileUrl",
"link_url",
"linkUrl",
"ori_url",
"oriUrl",
"original_url",
"originalUrl",
]),
);
if (!url) {
if (!url || isYuanbaoInternalUrl(url)) {
return null;
}
@@ -1373,25 +1463,79 @@ const yuanbaoQueryInPage = async (
const links: YuanbaoDomLink[] = [];
const seen = new Set<string>();
for (const node of Array.from(scope.querySelectorAll("a[href]"))) {
if (!(node instanceof HTMLAnchorElement) || !isVisible(node)) {
const urlAttributeNames = [
"href",
"data-href",
"data-url",
"data-link",
"data-source-url",
"data-target-url",
"data-jump-url",
"data-doc-url",
"data-article-url",
];
const extractUrls = (value: string | null): string[] => {
const normalized = normalize(value);
if (!normalized) {
return [];
}
const urls = new Set<string>();
const pushCandidate = (candidate: string) => {
try {
const decoded = decodeURIComponent(candidate);
if (/^https?:\/\//i.test(decoded)) {
urls.add(decoded);
}
} catch {
if (/^https?:\/\//i.test(candidate)) {
urls.add(candidate);
}
}
};
if (/^https?:\/\//i.test(normalized)) {
pushCandidate(normalized);
}
for (const matched of normalized.matchAll(/https?:\/\/[^\s"'<>\\]+/gi)) {
if (matched[0]) {
pushCandidate(matched[0]);
}
}
return Array.from(urls);
};
for (const node of [scope, ...Array.from(scope.querySelectorAll("*"))]) {
if (!(node instanceof HTMLElement) || !isVisible(node)) {
continue;
}
const href = normalize(node.href);
if (!href || !/^https?:\/\//i.test(href) || seen.has(href)) {
continue;
const urls = new Set<string>();
if (node instanceof HTMLAnchorElement) {
for (const url of extractUrls(node.href)) {
urls.add(url);
}
}
for (const attributeName of urlAttributeNames) {
for (const url of extractUrls(node.getAttribute(attributeName))) {
urls.add(url);
}
}
seen.add(href);
links.push({
url: href,
title: normalize(node.title) ?? normalize(node.getAttribute("aria-label")),
text:
normalize(node.innerText)
?? normalize(node.textContent)
?? normalize(node.getAttribute("aria-label")),
});
for (const url of urls) {
if (seen.has(url)) {
continue;
}
seen.add(url);
links.push({
url,
title: normalize(node.getAttribute("title")) ?? normalize(node.getAttribute("aria-label")),
text:
normalize(node.innerText)
?? normalize(node.textContent)
?? normalize(node.getAttribute("aria-label")),
});
}
}
return links;
};
@@ -1644,6 +1788,7 @@ export const yuanbaoAdapter: MonitorAdapter = {
const providerModel = resolveProviderModel(pageResult, parsed);
const conversationID = parsed.conversationID ?? extractConversationID(pageResult.url);
const requestID = parsed.requestID ?? conversationID;
const maxCitationMarkerIndex = citationMarkerIndexes.length ? Math.max(...citationMarkerIndexes) : 0;
if (pageResult.ok === false) {
const failedPageResult = pageResult;
@@ -1691,6 +1836,26 @@ export const yuanbaoAdapter: MonitorAdapter = {
};
}
if (answer && maxCitationMarkerIndex > 0 && citations.length < maxCitationMarkerIndex) {
return {
status: "unknown",
summary: "元宝回答含引用标记但未拿到完整引用来源，已回写 unknown 等待后续补采。",
error: buildAdapterError(
"yuanbao_incomplete_citations",
"yuanbao answer contains citation markers but source extraction is incomplete",
{
citation_marker_count: citationMarkerIndexes.length,
max_citation_marker_index: maxCitationMarkerIndex,
citation_count: citations.length,
search_result_count: searchResults.length,
dom_citation_count: domCitations.length,
parsed_citation_count: parsed.citations.length,
parsed_search_result_count: parsed.searchResults.length,
},
),
};
}
context.reportProgress("yuanbao.parse_result");
return {
status: "succeeded",
@@ -1724,3 +1889,13 @@ export const yuanbaoAdapter: MonitorAdapter = {
};
},
};
export const __yuanbaoTestUtils = {
buildSourceItem,
dedupeSourceItems,
extractCitationMarkerIndexes,
mergeText,
normalizeSourceUrl,
resolveCitationsFromMarkers,
selectBestAnswerText,
};
apps/desktop-client/src/main/runtime-controller.ts
+3 -2
View File
@@ -1750,9 +1750,10 @@ function buildMonitoringResultPayload(
execution: AdapterExecutionResult,
): MonitoringTaskResultPayload {
const payload = execution.payload ?? {};
const status = execution.status === "succeeded" ? "succeeded" : "failed";
return {
lease_token: leaseToken,
status: execution.status === "failed" ? "failed" : "succeeded",
status,
provider_model: asOptionalString(payload.provider_model),
provider_request_id: asOptionalString(payload.provider_request_id),
request_id: asOptionalString(payload.request_id),
@@ -1765,7 +1766,7 @@ function buildMonitoringResultPayload(
first_recommended: asOptionalBoolean(payload.first_recommended),
sentiment_label: asOptionalString(payload.sentiment_label),
matched_brand_terms: asStringArray(payload.matched_brand_terms),
error_message: execution.status === "failed" ? execution.summary : undefined,
error_message: status === "failed" ? execution.summary : undefined,
completed_at: new Date().toISOString(),
};
}
server/internal/tenant/app/monitoring_callback_service.go
+25 -1
View File
@@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"net/url"
"regexp"
"strconv"
"strings"
"time"
@@ -40,6 +41,8 @@ const (
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
)
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
type MonitoringCallbackService struct {
businessPool *pgxpool.Pool
monitoringPool *pgxpool.Pool
@@ -390,9 +393,13 @@ func (s *MonitoringCallbackService) handleTaskResultForInstallation(
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow result submission")
}
if normalizeRunStatus(req.Status) == "" {
runStatus := normalizeRunStatus(req.Status)
if runStatus == "" {
return nil, response.ErrBadRequest(40041, "invalid_status", "status must be succeeded or failed")
}
if err := validateMonitoringTaskResultSubmission(task.AIPlatformID, runStatus, req); err != nil {
return nil, err
}
receivedAt := time.Now().UTC().Round(0)
envelope := newMonitoringTaskResultEnvelope(task, installation.ID, receivedAt, req)
@@ -2851,6 +2858,23 @@ func normalizeRunStatus(value string) string {
}
}
func validateMonitoringTaskResultSubmission(platformID string, runStatus string, req MonitoringTaskResultRequest) error {
if runStatus != "succeeded" {
return nil
}
answer := strings.TrimSpace(monitoringStringValue(req.Answer))
if answer == "" {
return response.ErrBadRequest(40041, "invalid_monitoring_result_answer", "succeeded monitoring result requires a non-empty answer")
}
if monitoringCitationMarkerPattern.MatchString(answer) {
_, sourceInputs := buildMonitoringRawPayload(platformID, req)
if len(sourceInputs) == 0 {
return response.ErrBadRequest(40041, "invalid_monitoring_result_citations", "succeeded monitoring result with citation markers requires citation sources")
}
}
return nil
}
func normalizeMonitoringLeaseLimit(value int) int {
if value <= 0 {
return defaultMonitoringLeaseLimit
server/internal/tenant/app/monitoring_callback_service_test.go
+36
View File
@@ -327,6 +327,42 @@ func TestBuildMonitoringRawPayloadKimiIncludesSearchResultsInCitationInputs(t *t
}
}
func TestValidateMonitoringTaskResultSubmissionRequiresAnswerOnSuccess(t *testing.T) {
if err := validateMonitoringTaskResultSubmission("kimi", "failed", MonitoringTaskResultRequest{Status: "failed"}); err != nil {
t.Fatalf("validateMonitoringTaskResultSubmission() failed status error = %v, want nil", err)
}
answer := "有效回答"
if err := validateMonitoringTaskResultSubmission("kimi", "succeeded", MonitoringTaskResultRequest{Status: "succeeded", Answer: &answer}); err != nil {
t.Fatalf("validateMonitoringTaskResultSubmission() succeeded status with answer error = %v, want nil", err)
}
if err := validateMonitoringTaskResultSubmission("kimi", "succeeded", MonitoringTaskResultRequest{Status: "succeeded"}); err == nil {
t.Fatal("validateMonitoringTaskResultSubmission() succeeded status without answer error = nil, want error")
}
}
func TestValidateMonitoringTaskResultSubmissionRequiresSourcesForCitationMarkers(t *testing.T) {
answer := "正文 [citation:1]"
if err := validateMonitoringTaskResultSubmission("yuanbao", "succeeded", MonitoringTaskResultRequest{
Status: "succeeded",
Answer: &answer,
}); err == nil {
t.Fatal("validateMonitoringTaskResultSubmission() citation marker without sources error = nil, want error")
}
title := "引用文章"
if err := validateMonitoringTaskResultSubmission("yuanbao", "succeeded", MonitoringTaskResultRequest{
Status: "succeeded",
Answer: &answer,
Citations: []MonitoringSourceItem{
{URL: "https://example.com/source", Title: &title},
},
}); err != nil {
t.Fatalf("validateMonitoringTaskResultSubmission() citation marker with sources error = %v, want nil", err)
}
}
func TestBuildMonitoringRawPayloadQwenIncludesSearchResultsInCitationInputs(t *testing.T) {
searchTitle := "搜索网页结果"