From f06c8dfc0166c4ed598ec7880a801501f8d5efb3 Mon Sep 17 00:00:00 2001 From: liangxu Date: Mon, 11 May 2026 19:48:21 +0800 Subject: [PATCH] fix(knowledge): always retry source fetch via lightpanda on trigger status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LinkReader 500 (or other trigger_status) used to be gated by an extra allowed_domains whitelist on the tenant API side, so URLs like m.baidu.com/bh/m/detail/... fell straight through to the user with a raw "webpage parser failed" message. Drop the whitelist gate — enabled + trigger_status is enough — and surface a friendly Chinese prompt when both LinkReader and the lightpanda fallback give up. Full error chain is kept in the logger for debugging. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../tenant/app/knowledge_url_parser.go | 21 +++++-- .../tenant/app/knowledge_url_parser_test.go | 62 ++++++++++++++----- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/server/internal/tenant/app/knowledge_url_parser.go b/server/internal/tenant/app/knowledge_url_parser.go index 539c53f..76924fa 100644 --- a/server/internal/tenant/app/knowledge_url_parser.go +++ b/server/internal/tenant/app/knowledge_url_parser.go @@ -58,6 +58,21 @@ func isSourceFetchRestrictedError(err error) bool { return errors.As(err, &restricted) } +// sourceFetchExhaustedError 表示 LinkReader 和 lightpanda 兜底都失败，应返回用户友好提示。 +type sourceFetchExhaustedError struct { + URL string + Primary error + Fallback error +} + +func (e sourceFetchExhaustedError) Error() string { + return "源文章解析失败，请更换链接或确认链接可公开访问（已尝试浏览器渲染兜底）" +} + +func (e sourceFetchExhaustedError) Unwrap() error { + return e.Primary +} + type arkToolExecuteRequest struct { ActionName string `json:"action_name"` ToolName string `json:"tool_name"` @@ -96,9 +111,10 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str s.logger.Warn("knowledge browser fetch fallback failed", zap.String("url", rawURL), zap.Error(fallbackErr), + zap.NamedError("primary", err), ) } - return nil, classifySourceFetchError(rawURL, fmt.Errorf("%w; browser fetch fallback failed: %v", err, fallbackErr)) + return nil, sourceFetchExhaustedError{URL: rawURL, Primary: err, Fallback: fallbackErr} } text := normalizeKnowledgeText(item.Content) @@ -287,9 +303,6 @@ func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err erro if !browserCfg.Enabled { return false } - if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) { - return false - } status := extractHTTPStatusFromError(err) if status <= 0 { return false diff --git a/server/internal/tenant/app/knowledge_url_parser_test.go b/server/internal/tenant/app/knowledge_url_parser_test.go index dd40ae4..eda19e3 100644 --- a/server/internal/tenant/app/knowledge_url_parser_test.go +++ b/server/internal/tenant/app/knowledge_url_parser_test.go @@ -1,10 +1,13 @@ package app import ( + "errors" "fmt" "net/http" "strings" "testing" + + "github.com/geo-platform/tenant-api/internal/shared/config" ) func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) { @@ -18,22 +21,53 @@ func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) { } } -func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) { +func TestShouldTryBrowserFetchFallbackIgnoresAllowedDomains(t *testing.T) { t.Parallel() - err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf( - "webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed", - http.StatusInternalServerError, - )) + svc := &KnowledgeService{ + cfg: knowledgeRuntimeConfig{ + BrowserFetch: config.BrowserFetchConfig{ + Enabled: true, + Provider: "lightpanda", + ServiceURL: "http://browser-fetch:8082", + TriggerStatus: []int{500}, + AllowedDomains: nil, + }, + }, + } - message := err.Error() - for _, expected := range []string{ - "source website fetch restricted status=500", - "webpage parser failed", - "browser fetch fallback failed: domain is not allowed", - } { - if !strings.Contains(message, expected) { - t.Fatalf("classified error = %q, want %q", message, expected) - } + err := fmt.Errorf("webpage parser failed: status=%d body=invalid link", http.StatusInternalServerError) + if !svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) { + t.Fatalf("expected fallback to trigger on status=500 regardless of allowed_domains") + } + + svc.cfg.BrowserFetch.Enabled = false + if svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) { + t.Fatalf("expected fallback to be skipped when browser_fetch.enabled=false") + } + + svc.cfg.BrowserFetch.Enabled = true + svc.cfg.BrowserFetch.TriggerStatus = []int{502} + if svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) { + t.Fatalf("expected fallback to be skipped when status not in trigger_status") + } +} + +func TestSourceFetchExhaustedErrorIsFriendly(t *testing.T) { + t.Parallel() + + primary := fmt.Errorf("webpage parser failed: status=500 body=invalid link") + fallback := fmt.Errorf("browser fetch timeout") + err := sourceFetchExhaustedError{URL: "https://m.baidu.com/x", Primary: primary, Fallback: fallback} + + msg := err.Error() + if !strings.Contains(msg, "源文章解析失败") { + t.Fatalf("error message %q should contain friendly Chinese prompt", msg) + } + if strings.Contains(msg, "status=500") || strings.Contains(msg, "invalid link") { + t.Fatalf("error message %q should not leak technical details", msg) + } + if !errors.Is(err, primary) { + t.Fatalf("expected errors.Is(err, primary) to be true") } }