feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -16,6 +16,7 @@ x-app-env: &app-env
|
||||
JWT_ACCESS_TTL: ${JWT_ACCESS_TTL:-15m}
|
||||
JWT_REFRESH_TTL: ${JWT_REFRESH_TTL:-720h}
|
||||
SCHEDULER_INTERNAL_METRICS_TOKEN: ${SCHEDULER_INTERNAL_METRICS_TOKEN:-}
|
||||
BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-}
|
||||
OBJECT_STORAGE_ACCESS_KEY: ${MINIO_ROOT_USER:-minioadmin}
|
||||
OBJECT_STORAGE_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin}
|
||||
|
||||
@@ -261,6 +262,33 @@ services:
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
|
||||
browser-fetch:
|
||||
image: ${IMAGE_REGISTRY:-ghcr.io/your-org/geo-rankly}/browser-fetch:${IMAGE_TAG:-latest}
|
||||
container_name: geo-browser-fetch
|
||||
build:
|
||||
context: ../server
|
||||
dockerfile: Dockerfile
|
||||
target: browser-fetch
|
||||
args:
|
||||
SERVICE: browser-fetch
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-geo-browser-fetch-change-me}
|
||||
BROWSER_FETCH_ALLOWED_DOMAINS: zhuanlan.zhihu.com,zhihu.com
|
||||
BROWSER_FETCH_QUEUE_SIZE: 128
|
||||
BROWSER_FETCH_WORKER_CONCURRENCY: 2
|
||||
BROWSER_FETCH_PER_DOMAIN_CONCURRENCY: 1
|
||||
BROWSER_FETCH_REQUEST_TIMEOUT: 45s
|
||||
BROWSER_FETCH_CACHE_TTL: 6h
|
||||
LIGHTPANDA_DRIVER: fetch
|
||||
LIGHTPANDA_FETCH_ARGS: --dump markdown
|
||||
healthcheck:
|
||||
test: ["/app/service", "-healthcheck"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
|
||||
worker-generate:
|
||||
image: ${IMAGE_REGISTRY:-ghcr.io/your-org/geo-rankly}/worker-generate:${IMAGE_TAG:-latest}
|
||||
container_name: geo-worker-generate
|
||||
|
||||
Reference in New Issue
Block a user