feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).
This commit is contained in:
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
@@ -26,6 +26,7 @@ type knowledgeRuntimeConfig struct {
ArkBaseURL string
ArkAPIKey string
URLMarkdownModel string
BrowserFetch config.BrowserFetchConfig
}
type generationStreamDisabledConfigProvider struct {
@@ -103,3 +104,14 @@ func knowledgeRuntimeFromConfig(retrievalCfg config.RetrievalConfig, llmCfg conf
URLMarkdownModel: model,
}
}
func knowledgeRuntimeFromFullConfig(cfg *config.Config) knowledgeRuntimeConfig {
if cfg == nil {
return knowledgeRuntimeFromConfig(config.RetrievalConfig{}, config.LLMConfig{})
}
runtime := knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
browserFetch := cfg.BrowserFetch
config.NormalizeBrowserFetchConfig(&browserFetch)
runtime.BrowserFetch = browserFetch
return runtime
}