feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -1,5 +1,89 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: browser-fetch
|
||||
labels:
|
||||
app.kubernetes.io/name: browser-fetch
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: browser-fetch
|
||||
ports:
|
||||
- name: http
|
||||
port: 8082
|
||||
targetPort: http
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: browser-fetch
|
||||
labels:
|
||||
app.kubernetes.io/name: browser-fetch
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: browser-fetch
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: browser-fetch
|
||||
spec:
|
||||
containers:
|
||||
- name: browser-fetch
|
||||
image: geo-rankly/browser-fetch:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8082
|
||||
env:
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: BROWSER_FETCH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: geo-rankly-app-secret
|
||||
key: BROWSER_FETCH_TOKEN
|
||||
- name: BROWSER_FETCH_ALLOWED_DOMAINS
|
||||
value: zhuanlan.zhihu.com,zhihu.com
|
||||
- name: BROWSER_FETCH_QUEUE_SIZE
|
||||
value: "128"
|
||||
- name: BROWSER_FETCH_WORKER_CONCURRENCY
|
||||
value: "2"
|
||||
- name: BROWSER_FETCH_PER_DOMAIN_CONCURRENCY
|
||||
value: "1"
|
||||
- name: BROWSER_FETCH_REQUEST_TIMEOUT
|
||||
value: 45s
|
||||
- name: BROWSER_FETCH_CACHE_TTL
|
||||
value: 6h
|
||||
- name: LIGHTPANDA_DRIVER
|
||||
value: fetch
|
||||
- name: LIGHTPANDA_FETCH_ARGS
|
||||
value: --dump markdown
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /app/service
|
||||
- -healthcheck
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health/live
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: tenant-api
|
||||
labels:
|
||||
@@ -60,6 +144,11 @@ spec:
|
||||
secretKeyRef:
|
||||
name: geo-rankly-app-secret
|
||||
key: MINIO_ROOT_PASSWORD
|
||||
- name: BROWSER_FETCH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: geo-rankly-app-secret
|
||||
key: BROWSER_FETCH_TOKEN
|
||||
- name: SCHEDULER_INTERNAL_METRICS_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -152,6 +241,11 @@ spec:
|
||||
secretKeyRef:
|
||||
name: geo-rankly-app-secret
|
||||
key: MINIO_ROOT_PASSWORD
|
||||
- name: BROWSER_FETCH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: geo-rankly-app-secret
|
||||
key: BROWSER_FETCH_TOKEN
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
|
||||
Reference in New Issue
Block a user