feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -49,6 +49,8 @@ type KnowledgeService struct {
|
||||
jobs chan knowledgeParseJob
|
||||
cleanupMu sync.Mutex
|
||||
cleanupInFlight map[string]struct{}
|
||||
browserFetchMu sync.Mutex
|
||||
browserFetchQ *browserFetchQueue
|
||||
}
|
||||
|
||||
type knowledgeParseJob struct {
|
||||
@@ -199,7 +201,7 @@ func (s *KnowledgeService) WithConfigProvider(provider runtimeConfigProvider) *K
|
||||
func (s *KnowledgeService) runtimeConfig() knowledgeRuntimeConfig {
|
||||
if s != nil && s.configProvider != nil {
|
||||
if cfg := s.configProvider.Current(); cfg != nil {
|
||||
return knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
|
||||
return knowledgeRuntimeFromFullConfig(cfg)
|
||||
}
|
||||
}
|
||||
if s == nil {
|
||||
|
||||
Reference in New Issue
Block a user