fix(browser-fetch): allow baidu source fallback
Deployment Config CI / Deployment Config (push) Successful in 1m35s
Backend CI / Backend (push) Successful in 17m20s

This commit is contained in:
2026-05-11 14:30:58 +08:00
parent 2113bcb601
commit c16a684add
7 changed files with 45 additions and 3 deletions
@@ -98,7 +98,7 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
zap.Error(fallbackErr),
)
}
return nil, classifySourceFetchError(rawURL, err)
return nil, classifySourceFetchError(rawURL, fmt.Errorf("%w; browser fetch fallback failed: %v", err, fallbackErr))
}
text := normalizeKnowledgeText(item.Content)
@@ -0,0 +1,39 @@
package app
import (
"fmt"
"net/http"
"strings"
"testing"
)
func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
t.Parallel()
if !browserFetchDomainAllowed("https://mbd.baidu.com/bh/m/detail/ar_10234112381636647949", []string{"baidu.com"}) {
t.Fatalf("expected baidu.com to allow mbd.baidu.com")
}
if browserFetchDomainAllowed("https://notbaidu.com/article", []string{"baidu.com"}) {
t.Fatalf("expected baidu.com not to allow notbaidu.com")
}
}
func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) {
t.Parallel()
err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf(
"webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed",
http.StatusInternalServerError,
))
message := err.Error()
for _, expected := range []string{
"source website fetch restricted status=500",
"webpage parser failed",
"browser fetch fallback failed: domain is not allowed",
} {
if !strings.Contains(message, expected) {
t.Fatalf("classified error = %q, want %q", message, expected)
}
}
}