feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).

deploy/k3s/README.md

@@ -4,7 +4,7 @@
 
 - infra: `postgres`, `monitoring-postgres`, `pgbouncer`, `monitoring-pgbouncer`, `rabbitmq`, `redis`, `qdrant`, `minio`
 - jobs: `migrate`, `minio-init`
-- apps: `tenant-api`, `worker-generate`, `kol-assist-worker`, `scheduler`, `ops-api`, `frontend`, `ops-web`
+- apps: `tenant-api`, `browser-fetch`, `worker-generate`, `kol-assist-worker`, `scheduler`, `ops-api`, `frontend`, `ops-web`
 - ingress: `geo-rankly.local` -> `frontend`, `ops.geo-rankly.local` -> `ops-web`
 
 ## 1. 准备镜像
@@ -34,6 +34,7 @@ sudo k3s ctr images import images.tar
 - `OPS_JWT_SECRET`
 - `OPS_DEFAULT_ADMIN_PASSWORD`
 - `SCHEDULER_INTERNAL_METRICS_TOKEN`
+- `BROWSER_FETCH_TOKEN`
 - `POSTGRES_PASSWORD` / `RABBITMQ_DEFAULT_PASS` / `MINIO_ROOT_*`
 - `config/config.yaml` 里的 LLM / SiliconFlow API key
 - `config.local.yaml` 里对应的数据库、RabbitMQ、MinIO、JWT 配置
@@ -131,6 +132,7 @@ kubectl -n geo-rankly port-forward svc/minio 9001:9001
 kubectl -n geo-rankly get all
 kubectl -n geo-rankly describe pod <pod-name>
 kubectl -n geo-rankly logs deploy/tenant-api -f
+kubectl -n geo-rankly logs deploy/browser-fetch -f
 kubectl -n geo-rankly logs deploy/scheduler -f
 kubectl -n geo-rankly logs deploy/worker-generate -f
 kubectl -n geo-rankly logs deploy/ops-api -f

deploy/k3s/apps.yaml

@@ -1,5 +1,89 @@
 apiVersion: v1
 kind: Service
+metadata:
+  name: browser-fetch
+  labels:
+    app.kubernetes.io/name: browser-fetch
+spec:
+  selector:
+    app.kubernetes.io/name: browser-fetch
+  ports:
+    - name: http
+      port: 8082
+      targetPort: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: browser-fetch
+  labels:
+    app.kubernetes.io/name: browser-fetch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: browser-fetch
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: browser-fetch
+    spec:
+      containers:
+        - name: browser-fetch
+          image: geo-rankly/browser-fetch:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8082
+          env:
+            - name: TZ
+              value: Asia/Shanghai
+            - name: BROWSER_FETCH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: geo-rankly-app-secret
+                  key: BROWSER_FETCH_TOKEN
+            - name: BROWSER_FETCH_ALLOWED_DOMAINS
+              value: zhuanlan.zhihu.com,zhihu.com
+            - name: BROWSER_FETCH_QUEUE_SIZE
+              value: "128"
+            - name: BROWSER_FETCH_WORKER_CONCURRENCY
+              value: "2"
+            - name: BROWSER_FETCH_PER_DOMAIN_CONCURRENCY
+              value: "1"
+            - name: BROWSER_FETCH_REQUEST_TIMEOUT
+              value: 45s
+            - name: BROWSER_FETCH_CACHE_TTL
+              value: 6h
+            - name: LIGHTPANDA_DRIVER
+              value: fetch
+            - name: LIGHTPANDA_FETCH_ARGS
+              value: --dump markdown
+          readinessProbe:
+            exec:
+              command:
+                - /app/service
+                - -healthcheck
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet:
+              path: /health/live
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            timeoutSeconds: 3
+          resources:
+            requests:
+              cpu: 500m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 2Gi
+---
+apiVersion: v1
+kind: Service
 metadata:
   name: tenant-api
   labels:
@@ -60,6 +144,11 @@ spec:
                 secretKeyRef:
                   name: geo-rankly-app-secret
                   key: MINIO_ROOT_PASSWORD
+            - name: BROWSER_FETCH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: geo-rankly-app-secret
+                  key: BROWSER_FETCH_TOKEN
             - name: SCHEDULER_INTERNAL_METRICS_TOKEN
               valueFrom:
                 secretKeyRef:
@@ -152,6 +241,11 @@ spec:
                 secretKeyRef:
                   name: geo-rankly-app-secret
                   key: MINIO_ROOT_PASSWORD
+            - name: BROWSER_FETCH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: geo-rankly-app-secret
+                  key: BROWSER_FETCH_TOKEN
           resources:
             requests:
               cpu: 200m

deploy/k3s/config/config.yaml

@@ -129,8 +129,8 @@ llm:
   provider: ark
   base_url: https://ark.cn-beijing.volces.com/api/v3
   api_key: "7fb6c66b-129c-4935-9617-709236c4205a"
-  model: doubao-seed-2-0-lite-260215
-  knowledge_url_model: doubao-seed-2-0-mini-260215
+  model: doubao-seed-2-0-pro-260215 # doubao-seed-2-0-lite-260215   #
+  knowledge_url_model: doubao-seed-2-0-mini-260215 #doubao-seed-2-0-mini-260215
   timeout: 2m
   max_output_tokens: 16000
   temperature: 0.7
@@ -172,3 +172,22 @@ generation:
   task_recovery_batch_size: 100
   task_queued_stale_after: 2m
   task_max_attempts: 3
+
+browser_fetch:
+  enabled: true
+  provider: lightpanda
+  service_url: http://browser-fetch:8082
+  queue_size: 64
+  worker_concurrency: 2
+  request_timeout: 45s
+  cache_ttl: 6h
+  allowed_domains:
+    - zhuanlan.zhihu.com
+    - zhihu.com
+  trigger_status:
+    - 403
+    - 429
+    - 500
+    - 502
+    - 503
+    - 504

deploy/k3s/kustomization.yaml

@@ -37,6 +37,9 @@ images:
   - name: geo-rankly/tenant-api
     newName: geo-rankly/tenant-api
     newTag: latest
+  - name: geo-rankly/browser-fetch
+    newName: geo-rankly/browser-fetch
+    newTag: latest
   - name: geo-rankly/worker-generate
     newName: geo-rankly/worker-generate
     newTag: latest

deploy/k3s/secrets.yaml

@@ -12,6 +12,7 @@ stringData:
   OPS_JWT_SECRET: geo-ops-change-me-in-production
   OPS_DEFAULT_ADMIN_PASSWORD: Aa123456
   SCHEDULER_INTERNAL_METRICS_TOKEN: geo-metrics-token
+  BROWSER_FETCH_TOKEN: geo-browser-fetch-change-me
   config.local.yaml: |
     database:
       password: ${POSTGRES_PASSWORD:geo_dev}
@@ -31,3 +32,6 @@ stringData:
 
     scheduler:
       internal_metrics_token: "geo-metrics-token"
+
+    browser_fetch:
+      token: ${BROWSER_FETCH_TOKEN:geo-browser-fetch-change-me}