feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).
This commit is contained in:
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
+20
View File
@@ -204,3 +204,23 @@ generation:
worker_concurrency: 8
stream_enabled: false
article_timeout: 8m
browser_fetch:
enabled: false
provider: lightpanda
service_url: http://localhost:8082
token: dev-browser-fetch-token
queue_size: 64
worker_concurrency: 2
request_timeout: 45s
cache_ttl: 6h
allowed_domains:
- zhuanlan.zhihu.com
- zhihu.com
trigger_status:
- 403
- 429
- 500
- 502
- 503
- 504
+20
View File
@@ -216,4 +216,24 @@ generation:
task_recovery_batch_size: 100
task_queued_stale_after: 2m
task_max_attempts: 3
browser_fetch:
enabled: false
provider: lightpanda
service_url: http://localhost:8082
token: dev-browser-fetch-token
queue_size: 64
worker_concurrency: 2
request_timeout: 45s
cache_ttl: 6h
allowed_domains:
- zhuanlan.zhihu.com
- zhihu.com
trigger_status:
- 403
- 429
- 500
- 502
- 503
- 504