feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).
This commit is contained in:
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
+94
View File
@@ -1,5 +1,89 @@
apiVersion: v1
kind: Service
metadata:
name: browser-fetch
labels:
app.kubernetes.io/name: browser-fetch
spec:
selector:
app.kubernetes.io/name: browser-fetch
ports:
- name: http
port: 8082
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: browser-fetch
labels:
app.kubernetes.io/name: browser-fetch
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: browser-fetch
template:
metadata:
labels:
app.kubernetes.io/name: browser-fetch
spec:
containers:
- name: browser-fetch
image: geo-rankly/browser-fetch:latest
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8082
env:
- name: TZ
value: Asia/Shanghai
- name: BROWSER_FETCH_TOKEN
valueFrom:
secretKeyRef:
name: geo-rankly-app-secret
key: BROWSER_FETCH_TOKEN
- name: BROWSER_FETCH_ALLOWED_DOMAINS
value: zhuanlan.zhihu.com,zhihu.com
- name: BROWSER_FETCH_QUEUE_SIZE
value: "128"
- name: BROWSER_FETCH_WORKER_CONCURRENCY
value: "2"
- name: BROWSER_FETCH_PER_DOMAIN_CONCURRENCY
value: "1"
- name: BROWSER_FETCH_REQUEST_TIMEOUT
value: 45s
- name: BROWSER_FETCH_CACHE_TTL
value: 6h
- name: LIGHTPANDA_DRIVER
value: fetch
- name: LIGHTPANDA_FETCH_ARGS
value: --dump markdown
readinessProbe:
exec:
command:
- /app/service
- -healthcheck
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 3
livenessProbe:
httpGet:
path: /health/live
port: http
initialDelaySeconds: 15
periodSeconds: 20
timeoutSeconds: 3
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
---
apiVersion: v1
kind: Service
metadata:
name: tenant-api
labels:
@@ -60,6 +144,11 @@ spec:
secretKeyRef:
name: geo-rankly-app-secret
key: MINIO_ROOT_PASSWORD
- name: BROWSER_FETCH_TOKEN
valueFrom:
secretKeyRef:
name: geo-rankly-app-secret
key: BROWSER_FETCH_TOKEN
- name: SCHEDULER_INTERNAL_METRICS_TOKEN
valueFrom:
secretKeyRef:
@@ -152,6 +241,11 @@ spec:
secretKeyRef:
name: geo-rankly-app-secret
key: MINIO_ROOT_PASSWORD
- name: BROWSER_FETCH_TOKEN
valueFrom:
secretKeyRef:
name: geo-rankly-app-secret
key: BROWSER_FETCH_TOKEN
resources:
requests:
cpu: 200m