feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).

server/internal/shared/config/config.go

@@ -38,6 +38,7 @@ type Config struct {
 	Compliance         ComplianceConfig         `mapstructure:"compliance"`
 	Retrieval          RetrievalConfig          `mapstructure:"retrieval"`
 	Generation         GenerationConfig         `mapstructure:"generation"`
+	BrowserFetch       BrowserFetchConfig       `mapstructure:"browser_fetch"`
 }
 
 type ServerConfig struct {
@@ -348,6 +349,19 @@ type GenerationConfig struct {
 	TaskStateCheckLookback  time.Duration `mapstructure:"task_state_check_lookback"`
 }
 
+type BrowserFetchConfig struct {
+	Enabled           bool          `mapstructure:"enabled"`
+	Provider          string        `mapstructure:"provider"`
+	ServiceURL        string        `mapstructure:"service_url"`
+	Token             string        `mapstructure:"token"`
+	QueueSize         int           `mapstructure:"queue_size"`
+	WorkerConcurrency int           `mapstructure:"worker_concurrency"`
+	RequestTimeout    time.Duration `mapstructure:"request_timeout"`
+	CacheTTL          time.Duration `mapstructure:"cache_ttl"`
+	AllowedDomains    []string      `mapstructure:"allowed_domains"`
+	TriggerStatus     []int         `mapstructure:"trigger_status"`
+}
+
 func Load(configPath string) (*Config, error) {
 	cfg, _, err := loadWithFiles(configPath)
 	return cfg, err
@@ -382,6 +396,7 @@ func loadWithFiles(configPath string) (*Config, []string, error) {
 	normalizeBrandLibraryConfig(&cfg.BrandLibrary)
 	NormalizeComplianceConfig(&cfg.Compliance)
 	NormalizeGenerationConfig(&cfg.Generation)
+	NormalizeBrowserFetchConfig(&cfg.BrowserFetch)
 
 	files := []string{configFile}
 	if localConfigFile != "" {
@@ -511,6 +526,42 @@ func compactStringSlice(values []string) []string {
 	return out
 }
 
+func compactIntSlice(values []int) []int {
+	if values == nil {
+		return nil
+	}
+	out := make([]int, 0, len(values))
+	seen := make(map[int]struct{}, len(values))
+	for _, value := range values {
+		if value <= 0 {
+			continue
+		}
+		if _, exists := seen[value]; exists {
+			continue
+		}
+		seen[value] = struct{}{}
+		out = append(out, value)
+	}
+	return out
+}
+
+func parseIntList(value string) []int {
+	parts := strings.Split(value, ",")
+	out := make([]int, 0, len(parts))
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		n, err := strconv.Atoi(part)
+		if err != nil {
+			continue
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
 func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
 	if cfg == nil {
 		return
@@ -643,6 +694,35 @@ func NormalizeGenerationConfig(cfg *GenerationConfig) {
 	}
 }
 
+func NormalizeBrowserFetchConfig(cfg *BrowserFetchConfig) {
+	if cfg == nil {
+		return
+	}
+	cfg.Provider = strings.ToLower(strings.TrimSpace(cfg.Provider))
+	if cfg.Provider == "" {
+		cfg.Provider = "lightpanda"
+	}
+	cfg.ServiceURL = strings.TrimRight(strings.TrimSpace(cfg.ServiceURL), "/")
+	cfg.Token = strings.TrimSpace(cfg.Token)
+	if cfg.QueueSize <= 0 {
+		cfg.QueueSize = 64
+	}
+	if cfg.WorkerConcurrency <= 0 {
+		cfg.WorkerConcurrency = 1
+	}
+	if cfg.RequestTimeout <= 0 {
+		cfg.RequestTimeout = 45 * time.Second
+	}
+	if cfg.CacheTTL <= 0 {
+		cfg.CacheTTL = 6 * time.Hour
+	}
+	cfg.AllowedDomains = compactStringSlice(cfg.AllowedDomains)
+	cfg.TriggerStatus = compactIntSlice(cfg.TriggerStatus)
+	if len(cfg.TriggerStatus) == 0 {
+		cfg.TriggerStatus = []int{403, 429, 500, 502, 503, 504}
+	}
+}
+
 func NormalizeComplianceConfig(cfg *ComplianceConfig) {
 	if cfg == nil {
 		return
@@ -759,6 +839,36 @@ func applyEnvOverrides(cfg *Config) {
 	if d, ok := lookupDurationEnv("GENERATION_TASK_STATE_CHECK_LOOKBACK"); ok {
 		cfg.Generation.TaskStateCheckLookback = d
 	}
+	if enabled, ok := lookupBoolEnv("BROWSER_FETCH_ENABLED"); ok {
+		cfg.BrowserFetch.Enabled = enabled
+	}
+	if provider, ok := lookupNonEmptyEnv("BROWSER_FETCH_PROVIDER"); ok {
+		cfg.BrowserFetch.Provider = provider
+	}
+	if serviceURL, ok := lookupFirstNonEmptyEnv("BROWSER_FETCH_SERVICE_URL", "LIGHTPANDA_BROWSER_FETCH_URL"); ok {
+		cfg.BrowserFetch.ServiceURL = serviceURL
+	}
+	if token, ok := lookupNonEmptyEnv("BROWSER_FETCH_TOKEN"); ok {
+		cfg.BrowserFetch.Token = token
+	}
+	if n, ok := lookupIntEnv("BROWSER_FETCH_QUEUE_SIZE"); ok {
+		cfg.BrowserFetch.QueueSize = n
+	}
+	if n, ok := lookupIntEnv("BROWSER_FETCH_WORKER_CONCURRENCY"); ok {
+		cfg.BrowserFetch.WorkerConcurrency = n
+	}
+	if d, ok := lookupDurationEnv("BROWSER_FETCH_REQUEST_TIMEOUT"); ok {
+		cfg.BrowserFetch.RequestTimeout = d
+	}
+	if d, ok := lookupDurationEnv("BROWSER_FETCH_CACHE_TTL"); ok {
+		cfg.BrowserFetch.CacheTTL = d
+	}
+	if domains, ok := lookupNonEmptyEnv("BROWSER_FETCH_ALLOWED_DOMAINS"); ok {
+		cfg.BrowserFetch.AllowedDomains = strings.Split(domains, ",")
+	}
+	if statuses, ok := lookupNonEmptyEnv("BROWSER_FETCH_TRIGGER_STATUS"); ok {
+		cfg.BrowserFetch.TriggerStatus = parseIntList(statuses)
+	}
 	if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
 		cfg.Qdrant.APIKey = apiKey
 	}