fix(knowledge): always retry source fetch via lightpanda on trigger status
Backend CI / Backend (push) Successful in 15m15s

LinkReader 500 (or other trigger_status) used to be gated by an extra
allowed_domains whitelist on the tenant API side, so URLs like
m.baidu.com/bh/m/detail/... fell straight through to the user with a raw
"webpage parser failed" message. Drop the whitelist gate — enabled +
trigger_status is enough — and surface a friendly Chinese prompt when
both LinkReader and the lightpanda fallback give up. Full error chain
is kept in the logger for debugging.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-11 19:48:21 +08:00
parent 83c5cc76d6
commit f06c8dfc01
2 changed files with 65 additions and 18 deletions
@@ -1,10 +1,13 @@
package app
import (
"errors"
"fmt"
"net/http"
"strings"
"testing"
"github.com/geo-platform/tenant-api/internal/shared/config"
)
func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
@@ -18,22 +21,53 @@ func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
}
}
func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) {
func TestShouldTryBrowserFetchFallbackIgnoresAllowedDomains(t *testing.T) {
t.Parallel()
err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf(
"webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed",
http.StatusInternalServerError,
))
svc := &KnowledgeService{
cfg: knowledgeRuntimeConfig{
BrowserFetch: config.BrowserFetchConfig{
Enabled: true,
Provider: "lightpanda",
ServiceURL: "http://browser-fetch:8082",
TriggerStatus: []int{500},
AllowedDomains: nil,
},
},
}
message := err.Error()
for _, expected := range []string{
"source website fetch restricted status=500",
"webpage parser failed",
"browser fetch fallback failed: domain is not allowed",
} {
if !strings.Contains(message, expected) {
t.Fatalf("classified error = %q, want %q", message, expected)
}
err := fmt.Errorf("webpage parser failed: status=%d body=invalid link", http.StatusInternalServerError)
if !svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) {
t.Fatalf("expected fallback to trigger on status=500 regardless of allowed_domains")
}
svc.cfg.BrowserFetch.Enabled = false
if svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) {
t.Fatalf("expected fallback to be skipped when browser_fetch.enabled=false")
}
svc.cfg.BrowserFetch.Enabled = true
svc.cfg.BrowserFetch.TriggerStatus = []int{502}
if svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) {
t.Fatalf("expected fallback to be skipped when status not in trigger_status")
}
}
func TestSourceFetchExhaustedErrorIsFriendly(t *testing.T) {
t.Parallel()
primary := fmt.Errorf("webpage parser failed: status=500 body=invalid link")
fallback := fmt.Errorf("browser fetch timeout")
err := sourceFetchExhaustedError{URL: "https://m.baidu.com/x", Primary: primary, Fallback: fallback}
msg := err.Error()
if !strings.Contains(msg, "源文章解析失败") {
t.Fatalf("error message %q should contain friendly Chinese prompt", msg)
}
if strings.Contains(msg, "status=500") || strings.Contains(msg, "invalid link") {
t.Fatalf("error message %q should not leak technical details", msg)
}
if !errors.Is(err, primary) {
t.Fatalf("expected errors.Is(err, primary) to be true")
}
}