feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -26,6 +26,7 @@ type knowledgeRuntimeConfig struct {
|
||||
ArkBaseURL string
|
||||
ArkAPIKey string
|
||||
URLMarkdownModel string
|
||||
BrowserFetch config.BrowserFetchConfig
|
||||
}
|
||||
|
||||
type generationStreamDisabledConfigProvider struct {
|
||||
@@ -103,3 +104,14 @@ func knowledgeRuntimeFromConfig(retrievalCfg config.RetrievalConfig, llmCfg conf
|
||||
URLMarkdownModel: model,
|
||||
}
|
||||
}
|
||||
|
||||
func knowledgeRuntimeFromFullConfig(cfg *config.Config) knowledgeRuntimeConfig {
|
||||
if cfg == nil {
|
||||
return knowledgeRuntimeFromConfig(config.RetrievalConfig{}, config.LLMConfig{})
|
||||
}
|
||||
runtime := knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
|
||||
browserFetch := cfg.BrowserFetch
|
||||
config.NormalizeBrowserFetchConfig(&browserFetch)
|
||||
runtime.BrowserFetch = browserFetch
|
||||
return runtime
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user