fix(knowledge): always retry source fetch via lightpanda on trigger status
Backend CI / Backend (push) Successful in 15m15s
Backend CI / Backend (push) Successful in 15m15s
LinkReader 500 (or other trigger_status) used to be gated by an extra allowed_domains whitelist on the tenant API side, so URLs like m.baidu.com/bh/m/detail/... fell straight through to the user with a raw "webpage parser failed" message. Drop the whitelist gate — enabled + trigger_status is enough — and surface a friendly Chinese prompt when both LinkReader and the lightpanda fallback give up. Full error chain is kept in the logger for debugging. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -58,6 +58,21 @@ func isSourceFetchRestrictedError(err error) bool {
|
||||
return errors.As(err, &restricted)
|
||||
}
|
||||
|
||||
// sourceFetchExhaustedError 表示 LinkReader 和 lightpanda 兜底都失败,应返回用户友好提示。
|
||||
type sourceFetchExhaustedError struct {
|
||||
URL string
|
||||
Primary error
|
||||
Fallback error
|
||||
}
|
||||
|
||||
func (e sourceFetchExhaustedError) Error() string {
|
||||
return "源文章解析失败,请更换链接或确认链接可公开访问(已尝试浏览器渲染兜底)"
|
||||
}
|
||||
|
||||
func (e sourceFetchExhaustedError) Unwrap() error {
|
||||
return e.Primary
|
||||
}
|
||||
|
||||
type arkToolExecuteRequest struct {
|
||||
ActionName string `json:"action_name"`
|
||||
ToolName string `json:"tool_name"`
|
||||
@@ -96,9 +111,10 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
|
||||
s.logger.Warn("knowledge browser fetch fallback failed",
|
||||
zap.String("url", rawURL),
|
||||
zap.Error(fallbackErr),
|
||||
zap.NamedError("primary", err),
|
||||
)
|
||||
}
|
||||
return nil, classifySourceFetchError(rawURL, fmt.Errorf("%w; browser fetch fallback failed: %v", err, fallbackErr))
|
||||
return nil, sourceFetchExhaustedError{URL: rawURL, Primary: err, Fallback: fallbackErr}
|
||||
}
|
||||
|
||||
text := normalizeKnowledgeText(item.Content)
|
||||
@@ -287,9 +303,6 @@ func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err erro
|
||||
if !browserCfg.Enabled {
|
||||
return false
|
||||
}
|
||||
if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) {
|
||||
return false
|
||||
}
|
||||
status := extractHTTPStatusFromError(err)
|
||||
if status <= 0 {
|
||||
return false
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
|
||||
@@ -18,22 +21,53 @@ func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) {
|
||||
func TestShouldTryBrowserFetchFallbackIgnoresAllowedDomains(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf(
|
||||
"webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed",
|
||||
http.StatusInternalServerError,
|
||||
))
|
||||
svc := &KnowledgeService{
|
||||
cfg: knowledgeRuntimeConfig{
|
||||
BrowserFetch: config.BrowserFetchConfig{
|
||||
Enabled: true,
|
||||
Provider: "lightpanda",
|
||||
ServiceURL: "http://browser-fetch:8082",
|
||||
TriggerStatus: []int{500},
|
||||
AllowedDomains: nil,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
for _, expected := range []string{
|
||||
"source website fetch restricted status=500",
|
||||
"webpage parser failed",
|
||||
"browser fetch fallback failed: domain is not allowed",
|
||||
} {
|
||||
if !strings.Contains(message, expected) {
|
||||
t.Fatalf("classified error = %q, want %q", message, expected)
|
||||
}
|
||||
err := fmt.Errorf("webpage parser failed: status=%d body=invalid link", http.StatusInternalServerError)
|
||||
if !svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) {
|
||||
t.Fatalf("expected fallback to trigger on status=500 regardless of allowed_domains")
|
||||
}
|
||||
|
||||
svc.cfg.BrowserFetch.Enabled = false
|
||||
if svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) {
|
||||
t.Fatalf("expected fallback to be skipped when browser_fetch.enabled=false")
|
||||
}
|
||||
|
||||
svc.cfg.BrowserFetch.Enabled = true
|
||||
svc.cfg.BrowserFetch.TriggerStatus = []int{502}
|
||||
if svc.shouldTryBrowserFetchFallback("https://m.baidu.com/bh/m/detail/ar_x", err) {
|
||||
t.Fatalf("expected fallback to be skipped when status not in trigger_status")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceFetchExhaustedErrorIsFriendly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
primary := fmt.Errorf("webpage parser failed: status=500 body=invalid link")
|
||||
fallback := fmt.Errorf("browser fetch timeout")
|
||||
err := sourceFetchExhaustedError{URL: "https://m.baidu.com/x", Primary: primary, Fallback: fallback}
|
||||
|
||||
msg := err.Error()
|
||||
if !strings.Contains(msg, "源文章解析失败") {
|
||||
t.Fatalf("error message %q should contain friendly Chinese prompt", msg)
|
||||
}
|
||||
if strings.Contains(msg, "status=500") || strings.Contains(msg, "invalid link") {
|
||||
t.Fatalf("error message %q should not leak technical details", msg)
|
||||
}
|
||||
if !errors.Is(err, primary) {
|
||||
t.Fatalf("expected errors.Is(err, primary) to be true")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user