feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
@@ -557,6 +557,93 @@ generation: {}
 	}
 }

+func TestLoadAppliesBrowserFetchDefaultsAndOverrides(t *testing.T) {
+	configPath := writeTestConfig(t, `
+browser_fetch:
+  enabled: true
+  service_url: " http://browser-fetch:8082/ "
+  token: " token "
+  allowed_domains:
+    - " zhuanlan.zhihu.com "
+    - ""
+  trigger_status:
+    - 403
+    - 403
+    - 500
+`)
+
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	if !cfg.BrowserFetch.Enabled {
+		t.Fatalf("expected browser fetch enabled")
+	}
+	if cfg.BrowserFetch.Provider != "lightpanda" {
+		t.Fatalf("expected default provider lightpanda, got %q", cfg.BrowserFetch.Provider)
+	}
+	if cfg.BrowserFetch.ServiceURL != "http://browser-fetch:8082" {
+		t.Fatalf("expected normalized service url, got %q", cfg.BrowserFetch.ServiceURL)
+	}
+	if cfg.BrowserFetch.Token != "token" {
+		t.Fatalf("expected normalized token, got %q", cfg.BrowserFetch.Token)
+	}
+	if cfg.BrowserFetch.QueueSize != 64 ||
+		cfg.BrowserFetch.WorkerConcurrency != 1 ||
+		cfg.BrowserFetch.RequestTimeout != 45*time.Second ||
+		cfg.BrowserFetch.CacheTTL != 6*time.Hour {
+		t.Fatalf("unexpected browser fetch defaults: %#v", cfg.BrowserFetch)
+	}
+	if len(cfg.BrowserFetch.AllowedDomains) != 1 || cfg.BrowserFetch.AllowedDomains[0] != "zhuanlan.zhihu.com" {
+		t.Fatalf("unexpected allowed domains: %#v", cfg.BrowserFetch.AllowedDomains)
+	}
+	if len(cfg.BrowserFetch.TriggerStatus) != 2 ||
+		cfg.BrowserFetch.TriggerStatus[0] != 403 ||
+		cfg.BrowserFetch.TriggerStatus[1] != 500 {
+		t.Fatalf("unexpected trigger status: %#v", cfg.BrowserFetch.TriggerStatus)
+	}
+}
+
+func TestLoadAppliesBrowserFetchEnvOverrides(t *testing.T) {
+	t.Setenv("BROWSER_FETCH_ENABLED", "true")
+	t.Setenv("BROWSER_FETCH_PROVIDER", "lightpanda_service")
+	t.Setenv("BROWSER_FETCH_SERVICE_URL", "http://env-browser-fetch:8082/")
+	t.Setenv("BROWSER_FETCH_TOKEN", "env-token")
+	t.Setenv("BROWSER_FETCH_QUEUE_SIZE", "11")
+	t.Setenv("BROWSER_FETCH_WORKER_CONCURRENCY", "3")
+	t.Setenv("BROWSER_FETCH_REQUEST_TIMEOUT", "12s")
+	t.Setenv("BROWSER_FETCH_CACHE_TTL", "30m")
+	t.Setenv("BROWSER_FETCH_ALLOWED_DOMAINS", "zhihu.com,example.com")
+	t.Setenv("BROWSER_FETCH_TRIGGER_STATUS", "403,429,broken,503")
+
+	configPath := writeTestConfig(t, `
+browser_fetch: {}
+`)
+
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	if !cfg.BrowserFetch.Enabled ||
+		cfg.BrowserFetch.Provider != "lightpanda_service" ||
+		cfg.BrowserFetch.ServiceURL != "http://env-browser-fetch:8082" ||
+		cfg.BrowserFetch.Token != "env-token" ||
+		cfg.BrowserFetch.QueueSize != 11 ||
+		cfg.BrowserFetch.WorkerConcurrency != 3 ||
+		cfg.BrowserFetch.RequestTimeout != 12*time.Second ||
+		cfg.BrowserFetch.CacheTTL != 30*time.Minute {
+		t.Fatalf("unexpected browser fetch env config: %#v", cfg.BrowserFetch)
+	}
+	if got := cfg.BrowserFetch.AllowedDomains; len(got) != 2 || got[0] != "zhihu.com" || got[1] != "example.com" {
+		t.Fatalf("unexpected env allowed domains: %#v", got)
+	}
+	if got := cfg.BrowserFetch.TriggerStatus; len(got) != 3 || got[0] != 403 || got[1] != 429 || got[2] != 503 {
+		t.Fatalf("unexpected env trigger status: %#v", got)
+	}
+}
+
 func TestStoreReloadsConfigAndKeepsPreviousOnInvalidYAML(t *testing.T) {
 	configPath := writeTestConfig(t, `
 jwt: