root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Browse Source 
Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).
This commit is contained in:
liangxu
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/.env.example
+1
View File
@@ -14,6 +14,7 @@ JWT_SECRET=geo-web-Aa123456-change-me-in-production
OPS_JWT_SECRET=geo-ops-Aa123456-change-me-in-production
OPS_DEFAULT_ADMIN_PASSWORD=Aa123456
SCHEDULER_INTERNAL_METRICS_TOKEN=geo-scheduler-Aa123456-change-me-in-production
BROWSER_FETCH_TOKEN=geo-browser-fetch-Aa123456-change-me-in-production
POSTGRES_PASSWORD=geo_dev
# ─── Optional: MinIO credentials ─────────────────────────────────────────────
deploy/config.yaml
+19 -1
View File
@@ -215,4 +215,22 @@ generation:
task_recovery_batch_size: 100
task_queued_stale_after: 2m
task_max_attempts: 3
browser_fetch:
enabled: true
provider: lightpanda
service_url: http://browser-fetch:8082
queue_size: 64
worker_concurrency: 2
request_timeout: 45s
cache_ttl: 6h
allowed_domains:
- zhuanlan.zhihu.com
- zhihu.com
trigger_status:
- 403
- 429
- 500
- 502
- 503
- 504
deploy/docker-compose.offline.yaml
+5
View File
@@ -18,6 +18,11 @@ services:
pull_policy: never
build: !reset null
browser-fetch:
image: geo-rankly/browser-fetch:${IMAGE_TAG:-latest}
pull_policy: never
build: !reset null
worker-generate:
image: geo-rankly/worker-generate:${IMAGE_TAG:-latest}
pull_policy: never
deploy/docker-compose.yaml
+28
View File
@@ -16,6 +16,7 @@ x-app-env: &app-env
JWT_ACCESS_TTL: ${JWT_ACCESS_TTL:-15m}
JWT_REFRESH_TTL: ${JWT_REFRESH_TTL:-720h}
SCHEDULER_INTERNAL_METRICS_TOKEN: ${SCHEDULER_INTERNAL_METRICS_TOKEN:-}
BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-}
OBJECT_STORAGE_ACCESS_KEY: ${MINIO_ROOT_USER:-minioadmin}
OBJECT_STORAGE_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin}
@@ -261,6 +262,33 @@ services:
retries: 5
start_period: 10s
browser-fetch:
image: ${IMAGE_REGISTRY:-ghcr.io/your-org/geo-rankly}/browser-fetch:${IMAGE_TAG:-latest}
container_name: geo-browser-fetch
build:
context: ../server
dockerfile: Dockerfile
target: browser-fetch
args:
SERVICE: browser-fetch
restart: unless-stopped
environment:
BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-geo-browser-fetch-change-me}
BROWSER_FETCH_ALLOWED_DOMAINS: zhuanlan.zhihu.com,zhihu.com
BROWSER_FETCH_QUEUE_SIZE: 128
BROWSER_FETCH_WORKER_CONCURRENCY: 2
BROWSER_FETCH_PER_DOMAIN_CONCURRENCY: 1
BROWSER_FETCH_REQUEST_TIMEOUT: 45s
BROWSER_FETCH_CACHE_TTL: 6h
LIGHTPANDA_DRIVER: fetch
LIGHTPANDA_FETCH_ARGS: --dump markdown
healthcheck:
test: ["/app/service", "-healthcheck"]
interval: 10s
timeout: 5s
retries: 5
start_period: 10s
worker-generate:
image: ${IMAGE_REGISTRY:-ghcr.io/your-org/geo-rankly}/worker-generate:${IMAGE_TAG:-latest}
container_name: geo-worker-generate
deploy/k3s/README.md
+3 -1
View File
@@ -4,7 +4,7 @@
- infra: `postgres`, `monitoring-postgres`, `pgbouncer`, `monitoring-pgbouncer`, `rabbitmq`, `redis`, `qdrant`, `minio`
- jobs: `migrate`, `minio-init`
- apps: `tenant-api`, `worker-generate`, `kol-assist-worker`, `scheduler`, `ops-api`, `frontend`, `ops-web`
- apps: `tenant-api`, `browser-fetch`, `worker-generate`, `kol-assist-worker`, `scheduler`, `ops-api`, `frontend`, `ops-web`
- ingress: `geo-rankly.local` -> `frontend`, `ops.geo-rankly.local` -> `ops-web`
## 1. 准备镜像
@@ -34,6 +34,7 @@ sudo k3s ctr images import images.tar
- `OPS_JWT_SECRET`
- `OPS_DEFAULT_ADMIN_PASSWORD`
- `SCHEDULER_INTERNAL_METRICS_TOKEN`
- `BROWSER_FETCH_TOKEN`
- `POSTGRES_PASSWORD` / `RABBITMQ_DEFAULT_PASS` / `MINIO_ROOT_*`
- `config/config.yaml` 里的 LLM / SiliconFlow API key
- `config.local.yaml` 里对应的数据库、RabbitMQ、MinIO、JWT 配置
@@ -131,6 +132,7 @@ kubectl -n geo-rankly port-forward svc/minio 9001:9001
kubectl -n geo-rankly get all
kubectl -n geo-rankly describe pod <pod-name>
kubectl -n geo-rankly logs deploy/tenant-api -f
kubectl -n geo-rankly logs deploy/browser-fetch -f
kubectl -n geo-rankly logs deploy/scheduler -f
kubectl -n geo-rankly logs deploy/worker-generate -f
kubectl -n geo-rankly logs deploy/ops-api -f
deploy/k3s/apps.yaml
+94
View File
@@ -1,5 +1,89 @@
apiVersion: v1
kind: Service
metadata:
name: browser-fetch
labels:
app.kubernetes.io/name: browser-fetch
spec:
selector:
app.kubernetes.io/name: browser-fetch
ports:
- name: http
port: 8082
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: browser-fetch
labels:
app.kubernetes.io/name: browser-fetch
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: browser-fetch
template:
metadata:
labels:
app.kubernetes.io/name: browser-fetch
spec:
containers:
- name: browser-fetch
image: geo-rankly/browser-fetch:latest
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8082
env:
- name: TZ
value: Asia/Shanghai
- name: BROWSER_FETCH_TOKEN
valueFrom:
secretKeyRef:
name: geo-rankly-app-secret
key: BROWSER_FETCH_TOKEN
- name: BROWSER_FETCH_ALLOWED_DOMAINS
value: zhuanlan.zhihu.com,zhihu.com
- name: BROWSER_FETCH_QUEUE_SIZE
value: "128"
- name: BROWSER_FETCH_WORKER_CONCURRENCY
value: "2"
- name: BROWSER_FETCH_PER_DOMAIN_CONCURRENCY
value: "1"
- name: BROWSER_FETCH_REQUEST_TIMEOUT
value: 45s
- name: BROWSER_FETCH_CACHE_TTL
value: 6h
- name: LIGHTPANDA_DRIVER
value: fetch
- name: LIGHTPANDA_FETCH_ARGS
value: --dump markdown
readinessProbe:
exec:
command:
- /app/service
- -healthcheck
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 3
livenessProbe:
httpGet:
path: /health/live
port: http
initialDelaySeconds: 15
periodSeconds: 20
timeoutSeconds: 3
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
---
apiVersion: v1
kind: Service
metadata:
name: tenant-api
labels:
@@ -60,6 +144,11 @@ spec:
secretKeyRef:
name: geo-rankly-app-secret
key: MINIO_ROOT_PASSWORD
- name: BROWSER_FETCH_TOKEN
valueFrom:
secretKeyRef:
name: geo-rankly-app-secret
key: BROWSER_FETCH_TOKEN
- name: SCHEDULER_INTERNAL_METRICS_TOKEN
valueFrom:
secretKeyRef:
@@ -152,6 +241,11 @@ spec:
secretKeyRef:
name: geo-rankly-app-secret
key: MINIO_ROOT_PASSWORD
- name: BROWSER_FETCH_TOKEN
valueFrom:
secretKeyRef:
name: geo-rankly-app-secret
key: BROWSER_FETCH_TOKEN
resources:
requests:
cpu: 200m
deploy/k3s/config/config.yaml
+21 -2
View File
@@ -129,8 +129,8 @@ llm:
provider: ark
base_url: https://ark.cn-beijing.volces.com/api/v3
api_key: "7fb6c66b-129c-4935-9617-709236c4205a"
model: doubao-seed-2-0-lite-260215
knowledge_url_model: doubao-seed-2-0-mini-260215
model: doubao-seed-2-0-pro-260215 # doubao-seed-2-0-lite-260215 #
knowledge_url_model: doubao-seed-2-0-mini-260215 #doubao-seed-2-0-mini-260215
timeout: 2m
max_output_tokens: 16000
temperature: 0.7
@@ -172,3 +172,22 @@ generation:
task_recovery_batch_size: 100
task_queued_stale_after: 2m
task_max_attempts: 3
browser_fetch:
enabled: true
provider: lightpanda
service_url: http://browser-fetch:8082
queue_size: 64
worker_concurrency: 2
request_timeout: 45s
cache_ttl: 6h
allowed_domains:
- zhuanlan.zhihu.com
- zhihu.com
trigger_status:
- 403
- 429
- 500
- 502
- 503
- 504
deploy/k3s/kustomization.yaml
+3
View File
@@ -37,6 +37,9 @@ images:
- name: geo-rankly/tenant-api
newName: geo-rankly/tenant-api
newTag: latest
- name: geo-rankly/browser-fetch
newName: geo-rankly/browser-fetch
newTag: latest
- name: geo-rankly/worker-generate
newName: geo-rankly/worker-generate
newTag: latest
deploy/k3s/secrets.yaml
+4
View File
@@ -12,6 +12,7 @@ stringData:
OPS_JWT_SECRET: geo-ops-change-me-in-production
OPS_DEFAULT_ADMIN_PASSWORD: Aa123456
SCHEDULER_INTERNAL_METRICS_TOKEN: geo-metrics-token
BROWSER_FETCH_TOKEN: geo-browser-fetch-change-me
config.local.yaml: |
database:
password: ${POSTGRES_PASSWORD:geo_dev}
@@ -31,3 +32,6 @@ stringData:
scheduler:
internal_metrics_token: "geo-metrics-token"
browser_fetch:
token: ${BROWSER_FETCH_TOKEN:geo-browser-fetch-change-me}
deploy/scripts/gitea-registry-images.sh
+10 -1
View File
@@ -16,7 +16,7 @@ REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
REGISTRY_PLAIN_HTTP="${REGISTRY_PLAIN_HTTP:-true}"
REGISTRY_SCHEME="${REGISTRY_SCHEME:-}"
IMAGE_TAG="${IMAGE_TAG:-}"
IMAGE_LIST="${IMAGE_LIST:-tenant-api ops-api worker-generate kol-assist-worker scheduler frontend ops-web migrate}"
IMAGE_LIST="${IMAGE_LIST:-tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web migrate}"
DELETE_OLD_IMAGE_TAGS="${DELETE_OLD_IMAGE_TAGS:-true}"
PUSH_LATEST="${PUSH_LATEST:-true}"
FORCE_BUILD="${FORCE_BUILD:-false}"
@@ -257,6 +257,15 @@ build_image() {
-f "${REPO_ROOT}/server/Dockerfile" \
"${REPO_ROOT}/server"
;;
browser-fetch)
docker build "${platform_args[@]}" \
--target browser-fetch \
--build-arg "SERVICE=browser-fetch" \
--tag "${repo}:${IMAGE_TAG}" \
--tag "${repo}:latest" \
-f "${REPO_ROOT}/server/Dockerfile" \
"${REPO_ROOT}/server"
;;
tenant-api|worker-generate|kol-assist-worker|scheduler|ops-api)
docker build "${platform_args[@]}" \
--target runtime \
deploy/scripts/package.sh
+3 -1
View File
@@ -59,7 +59,7 @@ cd "${REPO_ROOT}"
if [[ -n "${SOURCE_IMAGE_REGISTRY}" ]]; then
echo " pulling from ${SOURCE_IMAGE_REGISTRY}"
for svc in tenant-api worker-generate kol-assist-worker ops-api scheduler migrate frontend ops-web; do
for svc in tenant-api browser-fetch worker-generate kol-assist-worker ops-api scheduler migrate frontend ops-web; do
echo " pulling ${SOURCE_IMAGE_REGISTRY}/${svc}:${IMAGE_TAG}"
docker pull --platform "${PLATFORM}" "${SOURCE_IMAGE_REGISTRY}/${svc}:${IMAGE_TAG}"
docker tag "${SOURCE_IMAGE_REGISTRY}/${svc}:${IMAGE_TAG}" "${LOCAL_PREFIX}/${svc}:${IMAGE_TAG}"
@@ -81,6 +81,7 @@ else
}
_build runtime tenant-api --build-arg SERVICE=tenant-api
_build browser-fetch browser-fetch --build-arg SERVICE=browser-fetch
_build runtime worker-generate --build-arg SERVICE=worker-generate
_build runtime kol-assist-worker --build-arg SERVICE=kol-assist-worker
_build runtime ops-api --build-arg SERVICE=ops-api
@@ -127,6 +128,7 @@ mkdir -p "${WORK_DIR}"
ALL_IMAGES=(
"${LOCAL_PREFIX}/tenant-api:${IMAGE_TAG}"
"${LOCAL_PREFIX}/browser-fetch:${IMAGE_TAG}"
"${LOCAL_PREFIX}/worker-generate:${IMAGE_TAG}"
"${LOCAL_PREFIX}/kol-assist-worker:${IMAGE_TAG}"
"${LOCAL_PREFIX}/scheduler:${IMAGE_TAG}"