fix(browser-fetch): allow baidu source fallback

deploy/config.yaml

@@ -227,6 +227,7 @@ browser_fetch:
   allowed_domains:
     - zhuanlan.zhihu.com
     - zhihu.com
+    - baidu.com
   trigger_status:
     - 403
     - 429

deploy/docker-compose.yaml

@@ -274,7 +274,7 @@ services:
     restart: unless-stopped
     environment:
       BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-geo-browser-fetch-change-me}
-      BROWSER_FETCH_ALLOWED_DOMAINS: zhuanlan.zhihu.com,zhihu.com
+      BROWSER_FETCH_ALLOWED_DOMAINS: zhuanlan.zhihu.com,zhihu.com,baidu.com
       BROWSER_FETCH_QUEUE_SIZE: 128
       BROWSER_FETCH_WORKER_CONCURRENCY: 2
       BROWSER_FETCH_PER_DOMAIN_CONCURRENCY: 1

deploy/k3s/apps.yaml

@@ -44,7 +44,7 @@ spec:
                   name: geo-rankly-app-secret
                   key: BROWSER_FETCH_TOKEN
             - name: BROWSER_FETCH_ALLOWED_DOMAINS
-              value: zhuanlan.zhihu.com,zhihu.com
+              value: zhuanlan.zhihu.com,zhihu.com,baidu.com
             - name: BROWSER_FETCH_QUEUE_SIZE
               value: "128"
             - name: BROWSER_FETCH_WORKER_CONCURRENCY

deploy/k3s/config/config.yaml

@@ -184,6 +184,7 @@ browser_fetch:
   allowed_domains:
     - zhuanlan.zhihu.com
     - zhihu.com
+    - baidu.com
   trigger_status:
     - 403
     - 429

server/configs/config.local.yaml.example

@@ -217,6 +217,7 @@ browser_fetch:
   allowed_domains:
     - zhuanlan.zhihu.com
     - zhihu.com
+    - baidu.com
   trigger_status:
     - 403
     - 429

server/internal/tenant/app/knowledge_url_parser.go

@@ -98,7 +98,7 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
 				zap.Error(fallbackErr),
 			)
 		}
-		return nil, classifySourceFetchError(rawURL, err)
+		return nil, classifySourceFetchError(rawURL, fmt.Errorf("%w; browser fetch fallback failed: %v", err, fallbackErr))
 	}
 
 	text := normalizeKnowledgeText(item.Content)

server/internal/tenant/app/knowledge_url_parser_test.go

@@ -0,0 +1,39 @@
+package app
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
+	t.Parallel()
+
+	if !browserFetchDomainAllowed("https://mbd.baidu.com/bh/m/detail/ar_10234112381636647949", []string{"baidu.com"}) {
+		t.Fatalf("expected baidu.com to allow mbd.baidu.com")
+	}
+	if browserFetchDomainAllowed("https://notbaidu.com/article", []string{"baidu.com"}) {
+		t.Fatalf("expected baidu.com not to allow notbaidu.com")
+	}
+}
+
+func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) {
+	t.Parallel()
+
+	err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf(
+		"webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed",
+		http.StatusInternalServerError,
+	))
+
+	message := err.Error()
+	for _, expected := range []string{
+		"source website fetch restricted status=500",
+		"webpage parser failed",
+		"browser fetch fallback failed: domain is not allowed",
+	} {
+		if !strings.Contains(message, expected) {
+			t.Fatalf("classified error = %q, want %q", message, expected)
+		}
+	}
+}