diff --git a/deploy/config.yaml b/deploy/config.yaml index d32b81d..1a17b5c 100644 --- a/deploy/config.yaml +++ b/deploy/config.yaml @@ -227,6 +227,7 @@ browser_fetch: allowed_domains: - zhuanlan.zhihu.com - zhihu.com + - baidu.com trigger_status: - 403 - 429 diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index cfd0470..e9a0a23 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -274,7 +274,7 @@ services: restart: unless-stopped environment: BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-geo-browser-fetch-change-me} - BROWSER_FETCH_ALLOWED_DOMAINS: zhuanlan.zhihu.com,zhihu.com + BROWSER_FETCH_ALLOWED_DOMAINS: zhuanlan.zhihu.com,zhihu.com,baidu.com BROWSER_FETCH_QUEUE_SIZE: 128 BROWSER_FETCH_WORKER_CONCURRENCY: 2 BROWSER_FETCH_PER_DOMAIN_CONCURRENCY: 1 diff --git a/deploy/k3s/apps.yaml b/deploy/k3s/apps.yaml index b82eca0..87c10ec 100644 --- a/deploy/k3s/apps.yaml +++ b/deploy/k3s/apps.yaml @@ -44,7 +44,7 @@ spec: name: geo-rankly-app-secret key: BROWSER_FETCH_TOKEN - name: BROWSER_FETCH_ALLOWED_DOMAINS - value: zhuanlan.zhihu.com,zhihu.com + value: zhuanlan.zhihu.com,zhihu.com,baidu.com - name: BROWSER_FETCH_QUEUE_SIZE value: "128" - name: BROWSER_FETCH_WORKER_CONCURRENCY diff --git a/deploy/k3s/config/config.yaml b/deploy/k3s/config/config.yaml index b172eeb..7615909 100644 --- a/deploy/k3s/config/config.yaml +++ b/deploy/k3s/config/config.yaml @@ -184,6 +184,7 @@ browser_fetch: allowed_domains: - zhuanlan.zhihu.com - zhihu.com + - baidu.com trigger_status: - 403 - 429 diff --git a/server/configs/config.local.yaml.example b/server/configs/config.local.yaml.example index e407bd7..d691692 100644 --- a/server/configs/config.local.yaml.example +++ b/server/configs/config.local.yaml.example @@ -217,6 +217,7 @@ browser_fetch: allowed_domains: - zhuanlan.zhihu.com - zhihu.com + - baidu.com trigger_status: - 403 - 429 diff --git a/server/internal/tenant/app/knowledge_url_parser.go b/server/internal/tenant/app/knowledge_url_parser.go index 3930731..539c53f 100644 --- a/server/internal/tenant/app/knowledge_url_parser.go +++ b/server/internal/tenant/app/knowledge_url_parser.go @@ -98,7 +98,7 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str zap.Error(fallbackErr), ) } - return nil, classifySourceFetchError(rawURL, err) + return nil, classifySourceFetchError(rawURL, fmt.Errorf("%w; browser fetch fallback failed: %v", err, fallbackErr)) } text := normalizeKnowledgeText(item.Content) diff --git a/server/internal/tenant/app/knowledge_url_parser_test.go b/server/internal/tenant/app/knowledge_url_parser_test.go new file mode 100644 index 0000000..dd40ae4 --- /dev/null +++ b/server/internal/tenant/app/knowledge_url_parser_test.go @@ -0,0 +1,39 @@ +package app + +import ( + "fmt" + "net/http" + "strings" + "testing" +) + +func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) { + t.Parallel() + + if !browserFetchDomainAllowed("https://mbd.baidu.com/bh/m/detail/ar_10234112381636647949", []string{"baidu.com"}) { + t.Fatalf("expected baidu.com to allow mbd.baidu.com") + } + if browserFetchDomainAllowed("https://notbaidu.com/article", []string{"baidu.com"}) { + t.Fatalf("expected baidu.com not to allow notbaidu.com") + } +} + +func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) { + t.Parallel() + + err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf( + "webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed", + http.StatusInternalServerError, + )) + + message := err.Error() + for _, expected := range []string{ + "source website fetch restricted status=500", + "webpage parser failed", + "browser fetch fallback failed: domain is not allowed", + } { + if !strings.Contains(message, expected) { + t.Fatalf("classified error = %q, want %q", message, expected) + } + } +}