From 98f73c5beab024bd69656d22dc22b52bc6c79698 Mon Sep 17 00:00:00 2001 From: liangxu Date: Wed, 20 May 2026 10:25:02 +0800 Subject: [PATCH] feat(objectstorage): add Aliyun OSS support and auto-detect provider in deploy workflows Introduces Aliyun OSS as an alternative to MinIO; deploy scripts and CI workflows now read object_storage.provider from config and conditionally include or skip MinIO resources in both Docker Compose and k3s paths. Also adds ops scheduler domain and its migration tables. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .gitea/workflows/deploy-config-ci.yml | 14 + .../workflows/nas-compose-service-deploy.yml | 107 +++++++- .gitea/workflows/nas-k3s-deploy.yml | 244 ++++++++++++++++++ .gitignore | 1 + deploy/OFFLINE_PACKAGE.md | 5 + deploy/docker-compose.aliyun-oss.yaml | 38 +++ deploy/docker-compose.yaml | 4 +- deploy/k3s-aliyun-oss/kustomization.yaml | 40 +++ .../patches/object-storage-env.yaml | 59 +++++ deploy/k3s/README.md | 14 + deploy/k3s/config/config.yaml | 14 +- deploy/scripts/load-and-start.sh | 185 ++++++++++++- deploy/scripts/package.sh | 7 + server/configs/config.yaml | 14 +- server/internal/ops/domain/scheduler.go | 79 ++++++ .../internal/shared/objectstorage/client.go | 2 +- ...0_create_scheduler_control_tables.down.sql | 4 + ...000_create_scheduler_control_tables.up.sql | 104 ++++++++ 18 files changed, 901 insertions(+), 34 deletions(-) create mode 100644 deploy/docker-compose.aliyun-oss.yaml create mode 100644 deploy/k3s-aliyun-oss/kustomization.yaml create mode 100644 deploy/k3s-aliyun-oss/patches/object-storage-env.yaml create mode 100644 server/internal/ops/domain/scheduler.go create mode 100644 server/migrations_ops/20260520120000_create_scheduler_control_tables.down.sql create mode 100644 server/migrations_ops/20260520120000_create_scheduler_control_tables.up.sql diff --git a/.gitea/workflows/deploy-config-ci.yml b/.gitea/workflows/deploy-config-ci.yml index fb79005..5c68c92 100644 --- a/.gitea/workflows/deploy-config-ci.yml +++ b/.gitea/workflows/deploy-config-ci.yml @@ -45,6 +45,20 @@ jobs: run: | docker compose -f deploy/docker-compose.yaml config >/tmp/compose.yaml docker compose -f deploy/docker-compose.yaml -f deploy/docker-compose.offline.yaml config >/tmp/compose-offline.yaml + docker compose -f deploy/docker-compose.yaml -f deploy/docker-compose.offline.yaml -f deploy/docker-compose.aliyun-oss.yaml config >/tmp/compose-aliyun-oss.yaml + + - name: Validate k3s manifests + run: | + if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl not found; skipping kustomize manifest validation." + exit 0 + fi + kubectl kustomize deploy/k3s >/tmp/k3s.yaml + kubectl kustomize deploy/k3s-aliyun-oss >/tmp/k3s-aliyun-oss.yaml + if grep -Eq 'name: (minio|minio-init)$' /tmp/k3s-aliyun-oss.yaml; then + echo "aliyun object-storage overlay must not render MinIO resources" + exit 1 + fi - name: Smoke build server runtime image run: | diff --git a/.gitea/workflows/nas-compose-service-deploy.yml b/.gitea/workflows/nas-compose-service-deploy.yml index 4651249..aa44fdd 100644 --- a/.gitea/workflows/nas-compose-service-deploy.yml +++ b/.gitea/workflows/nas-compose-service-deploy.yml @@ -573,6 +573,90 @@ jobs: docker_cmd compose "$@" } + ensure_config_aliases() { + for yaml in config.yaml config.local.yaml ops-config.yaml ops-config.local.yaml; do + yml="${yaml%.yaml}.yml" + if [ ! -f "${release_dir}/deploy/${yaml}" ] && [ -f "${release_dir}/deploy/${yml}" ]; then + cp "${release_dir}/deploy/${yml}" "${release_dir}/deploy/${yaml}" + echo "Created ${yaml} from ${yml} for deployment mounts." + fi + done + if [ ! -f "${release_dir}/deploy/prompts.yml" ] && [ -f "${release_dir}/deploy/prompts.yaml" ]; then + cp "${release_dir}/deploy/prompts.yaml" "${release_dir}/deploy/prompts.yml" + echo "Created prompts.yml from prompts.yaml for deployment mounts." + fi + } + + config_file_path() { + base="$1" + for file in "${release_dir}/deploy/${base}.yaml" "${release_dir}/deploy/${base}.yml"; do + if [ -f "${file}" ]; then + printf '%s' "${file}" + return 0 + fi + done + } + + read_object_storage_provider() { + provider="minio" + for file in "$(config_file_path config)" "$(config_file_path config.local)"; do + if [ -z "${file}" ] || [ ! -f "${file}" ]; then + continue + fi + next="$( + awk ' + /^[[:space:]]*object_storage:[[:space:]]*$/ { + in_section=1 + base=match($0, /[^[:space:]]/) - 1 + next + } + in_section { + indent=match($0, /[^[:space:]]/) - 1 + if ($0 ~ /^[[:space:]]*$/ || $0 ~ /^[[:space:]]*#/) { + next + } + if (indent <= base) { + in_section=0 + next + } + if ($0 ~ /^[[:space:]]*provider[[:space:]]*:/) { + value=$0 + sub(/^[[:space:]]*provider[[:space:]]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*$/, "", value) + gsub(/^["'\'']|["'\'']$/, "", value) + print value + } + } + ' "${file}" | tail -1 + )" + if [ -n "${next}" ]; then + provider="${next}" + fi + done + printf '%s' "$(printf '%s' "${provider}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + } + + use_minio_for_object_storage() { + case "$(read_object_storage_provider)" in + ""|minio|mino) + return 0 + ;; + aliyun|aliyun_oss|aliyun-oss|oss) + return 1 + ;; + *) + return 0 + ;; + esac + } + + compose_files() { + printf '%s\n' -f "${release_dir}/deploy/docker-compose.yaml" + if ! use_minio_for_object_storage; then + printf '%s\n' -f "${release_dir}/deploy/docker-compose.aliyun-oss.yaml" + fi + } + docker_login() { if [ "${DOCKER_MODE}" = "sudo" ]; then printf '%s\n' "${SUDO_PASSWORD}" | sudo -S -p "" -v @@ -606,11 +690,20 @@ jobs: fi docker_login + ensure_config_aliases + provider="$(read_object_storage_provider)" + if use_minio_for_object_storage; then + echo "object_storage.provider=${provider}; deploying MinIO." + else + echo "object_storage.provider=${provider}; skipping MinIO." + export COMPOSE_PROFILES="" + docker_cmd rm -f geo-minio geo-minio-init >/dev/null 2>&1 || true + fi IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ config >/tmp/geo-rankly-compose-service.yaml if [ "${RUN_MIGRATIONS}" = "true" ]; then @@ -618,19 +711,19 @@ jobs: IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ rm -f -s migrate >/dev/null 2>&1 || true IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ pull migrate IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ up --no-build --force-recreate migrate fi @@ -638,14 +731,14 @@ jobs: IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ pull ${SERVICE_LIST} echo "Restarting selected service(s): ${SERVICE_LIST}" IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ up -d --no-build --no-deps --force-recreate ${SERVICE_LIST} ln -sfn "${release_dir}" "${DEPLOY_BASE}/current" @@ -653,7 +746,7 @@ jobs: IMAGE_REGISTRY="${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}" \ IMAGE_TAG="${IMAGE_TAG}" compose_cmd \ --env-file "${shared_dir}/.env" \ - -f "${release_dir}/deploy/docker-compose.yaml" \ + $(compose_files) \ ps ${SERVICE_LIST} if [ "${KEEP_RELEASES}" -gt 0 ] 2>/dev/null; then diff --git a/.gitea/workflows/nas-k3s-deploy.yml b/.gitea/workflows/nas-k3s-deploy.yml index e4eea57..112601a 100644 --- a/.gitea/workflows/nas-k3s-deploy.yml +++ b/.gitea/workflows/nas-k3s-deploy.yml @@ -697,6 +697,103 @@ jobs: printf '%s:%s' "$(image_name "$1")" "${IMAGE_TAG}" } + ensure_config_aliases() { + for yaml in config.yaml config.local.yaml ops-config.yaml ops-config.local.yaml; do + yml="${yaml%.yaml}.yml" + if [ ! -f "${release_dir}/deploy/${yaml}" ] && [ -f "${release_dir}/deploy/${yml}" ]; then + cp "${release_dir}/deploy/${yml}" "${release_dir}/deploy/${yaml}" + echo "Created ${yaml} from ${yml} for deployment mounts." + fi + done + if [ ! -f "${release_dir}/deploy/prompts.yml" ] && [ -f "${release_dir}/deploy/prompts.yaml" ]; then + cp "${release_dir}/deploy/prompts.yaml" "${release_dir}/deploy/prompts.yml" + echo "Created prompts.yml from prompts.yaml for deployment mounts." + fi + } + + config_file_path() { + base="$1" + for file in "${release_dir}/deploy/${base}.yaml" "${release_dir}/deploy/${base}.yml"; do + if [ -f "${file}" ]; then + printf '%s' "${file}" + return 0 + fi + done + } + + read_object_storage_provider() { + provider="minio" + for file in "$(config_file_path config)" "$(config_file_path config.local)"; do + if [ -z "${file}" ] || [ ! -f "${file}" ]; then + continue + fi + next="$( + awk ' + /^[[:space:]]*object_storage:[[:space:]]*$/ { + in_section=1 + base=match($0, /[^[:space:]]/) - 1 + next + } + in_section { + indent=match($0, /[^[:space:]]/) - 1 + if ($0 ~ /^[[:space:]]*$/ || $0 ~ /^[[:space:]]*#/) { + next + } + if (indent <= base) { + in_section=0 + next + } + if ($0 ~ /^[[:space:]]*provider[[:space:]]*:/) { + value=$0 + sub(/^[[:space:]]*provider[[:space:]]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*$/, "", value) + gsub(/^["'\'']|["'\'']$/, "", value) + print value + } + } + ' "${file}" | tail -1 + )" + if [ -n "${next}" ]; then + provider="${next}" + fi + done + printf '%s' "$(printf '%s' "${provider}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + } + + use_minio_for_object_storage() { + case "$(read_object_storage_provider)" in + ""|minio|mino) + return 0 + ;; + aliyun|aliyun_oss|aliyun-oss|oss) + return 1 + ;; + *) + return 0 + ;; + esac + } + + sync_k3s_config() { + mkdir -p "${release_dir}/deploy/k3s/config" + for name in config ops-config prompts; do + output="${name}.yaml" + if [ "${name}" = "prompts" ]; then + output="prompts.yml" + fi + source="" + for candidate in "${release_dir}/deploy/${name}.yaml" "${release_dir}/deploy/${name}.yml"; do + if [ -f "${candidate}" ]; then + source="${candidate}" + break + fi + done + if [ -n "${source}" ]; then + cp "${source}" "${release_dir}/deploy/k3s/config/${output}" + fi + done + } + echo "Sudo mode: ${SUDO_MODE}" kubectl_cmd version --client @@ -797,6 +894,128 @@ jobs: sed -i "\#name: ${old_name}#,\#newTag:# s#newTag: .*#newTag: ${IMAGE_TAG}#" "${rendered_dir}/kustomization.yaml" } + apply_object_storage_kustomize_mode() { + sync_k3s_config + provider="$(read_object_storage_provider)" + if use_minio_for_object_storage; then + echo "object_storage.provider=${provider}; deploying MinIO." + return 0 + fi + + echo "object_storage.provider=${provider}; skipping MinIO." + sed -i '/^patches:[[:space:]]*$/,/^[^[:space:]-]/{/^patches:[[:space:]]*$/d;/^[^[:space:]-]/!d;}' "${rendered_dir}/kustomization.yaml" + mkdir -p "${rendered_dir}/patches" + cat > "${rendered_dir}/patches/object-storage-env.yaml" <<'EOF' + apiVersion: apps/v1 + kind: Deployment + metadata: + name: tenant-api + spec: + template: + spec: + containers: + - name: tenant-api + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: worker-generate + spec: + template: + spec: + containers: + - name: worker-generate + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kol-assist-worker + spec: + template: + spec: + containers: + - name: kol-assist-worker + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: scheduler + spec: + template: + spec: + containers: + - name: scheduler + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete + EOF + cat >> "${rendered_dir}/kustomization.yaml" <<'EOF' + patches: + - target: + version: v1 + kind: Service + name: minio + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: minio + - target: + group: apps + version: v1 + kind: StatefulSet + name: minio + patch: |- + $patch: delete + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: minio + - target: + group: batch + version: v1 + kind: Job + name: minio-init + patch: |- + $patch: delete + apiVersion: batch/v1 + kind: Job + metadata: + name: minio-init + - path: patches/object-storage-env.yaml + EOF + } + + cleanup_minio_if_external_object_storage() { + if use_minio_for_object_storage; then + return 0 + fi + kubectl_cmd -n "${K3S_NAMESPACE}" delete statefulset minio --ignore-not-found + kubectl_cmd -n "${K3S_NAMESPACE}" delete service minio --ignore-not-found + kubectl_cmd -n "${K3S_NAMESPACE}" delete job minio-init --ignore-not-found + for svc in tenant-api worker-generate kol-assist-worker scheduler; do + kubectl_cmd -n "${K3S_NAMESPACE}" set env "deployment/${svc}" OBJECT_STORAGE_ACCESS_KEY- OBJECT_STORAGE_SECRET_KEY- >/dev/null 2>&1 || true + done + } + render_kustomize() { rm -rf "${rendered_dir}" mkdir -p "${rendered_dir}" @@ -808,6 +1027,7 @@ jobs: # Migrations are applied explicitly by this workflow after preflight. sed -i '/^[[:space:]]*- jobs.yaml[[:space:]]*$/d' "${rendered_dir}/kustomization.yaml" + apply_object_storage_kustomize_mode patch_kustomize_image "geo-rankly/migrate" "migrate" patch_kustomize_image "geo-rankly/tenant-api" "tenant-api" @@ -832,6 +1052,7 @@ jobs: } apply_config_map() { + sync_k3s_config kubectl_cmd -n "${K3S_NAMESPACE}" create configmap geo-rankly-app-config \ --from-file=config.yaml="${release_dir}/deploy/k3s/config/config.yaml" \ --from-file=prompts.yml="${release_dir}/deploy/k3s/config/prompts.yml" \ @@ -847,6 +1068,27 @@ jobs: sed -i "s#image: geo-rankly/migrate:latest#image: $(image_ref migrate)#" "${rendered_jobs}" kubectl_cmd -n "${K3S_NAMESPACE}" delete job migrate minio-init --ignore-not-found + if ! use_minio_for_object_storage; then + awk ' + /^---[[:space:]]*$/ { doc=doc "\n" $0 "\n"; next } + { doc=doc $0 "\n" } + END { + n=split(doc, docs, /\n---\n/) + for (i=1; i<=n; i++) { + if (docs[i] ~ /kind:[[:space:]]*Job/ && docs[i] ~ /name:[[:space:]]*minio-init/) { + continue + } + if (docs[i] ~ /[^[:space:]]/) { + if (printed) print "---" + printf "%s", docs[i] + if (substr(docs[i], length(docs[i]), 1) != "\n") print "" + printed=1 + } + } + } + ' "${rendered_jobs}" > "${rendered_jobs}.tmp" + mv "${rendered_jobs}.tmp" "${rendered_jobs}" + fi kubectl_cmd -n "${K3S_NAMESPACE}" apply -f "${rendered_jobs}" } @@ -959,7 +1201,9 @@ jobs: if [ -n "${IMAGE_LIST}" ]; then ensure_plain_http_registry fi + ensure_config_aliases ensure_namespace + cleanup_minio_if_external_object_storage if [ -n "${IMAGE_LIST}" ]; then ensure_registry_secret else diff --git a/.gitignore b/.gitignore index 8330426..76f81d0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist/ .env.signing deploy/config.local.yaml deploy/ops-config.local.yaml +deploy/.k3s-runtime/ apps/*/.env.signing apps/*/dist/ apps/*/out/ diff --git a/deploy/OFFLINE_PACKAGE.md b/deploy/OFFLINE_PACKAGE.md index afcd4b7..2825e51 100644 --- a/deploy/OFFLINE_PACKAGE.md +++ b/deploy/OFFLINE_PACKAGE.md @@ -20,6 +20,11 @@ bash deploy.sh docker compose -f docker-compose.yaml -f docker-compose.offline.yaml --env-file .env up -d ``` +Object storage is selected from `config.yaml` / `config.yml`, with `config.local.yaml` / `config.local.yml` as an override: + +- `object_storage.provider: minio` or `mino` deploys MinIO and `minio-init` +- `object_storage.provider: aliyun`, `aliyun_oss`, `aliyun-oss`, or `oss` skips MinIO and uses the Aliyun OSS config from the same config files + Runtime services connect to Postgres through PgBouncer in `session` pooling mode. The one-shot `migrate` job still connects directly to `postgres` and `monitoring-postgres` so DDL and seed work do not go through the pooler. On a k3s host: diff --git a/deploy/docker-compose.aliyun-oss.yaml b/deploy/docker-compose.aliyun-oss.yaml new file mode 100644 index 0000000..24ceb4e --- /dev/null +++ b/deploy/docker-compose.aliyun-oss.yaml @@ -0,0 +1,38 @@ +# docker-compose.aliyun-oss.yaml +# Overlay used when object_storage.provider is aliyun/oss. +# MinIO is moved behind an inactive profile and app services no longer wait for it. + +services: + minio: + profiles: ["local-minio"] + + minio-init: + profiles: ["local-minio"] + + tenant-api: + environment: + OBJECT_STORAGE_ACCESS_KEY: !reset null + OBJECT_STORAGE_SECRET_KEY: !reset null + depends_on: + minio: !reset null + + worker-generate: + environment: + OBJECT_STORAGE_ACCESS_KEY: !reset null + OBJECT_STORAGE_SECRET_KEY: !reset null + depends_on: + minio: !reset null + + kol-assist-worker: + environment: + OBJECT_STORAGE_ACCESS_KEY: !reset null + OBJECT_STORAGE_SECRET_KEY: !reset null + depends_on: + minio: !reset null + + scheduler: + environment: + OBJECT_STORAGE_ACCESS_KEY: !reset null + OBJECT_STORAGE_SECRET_KEY: !reset null + depends_on: + minio: !reset null diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 024911b..2bc9499 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -17,8 +17,8 @@ x-app-env: &app-env JWT_REFRESH_TTL: ${JWT_REFRESH_TTL:-720h} SCHEDULER_INTERNAL_METRICS_TOKEN: ${SCHEDULER_INTERNAL_METRICS_TOKEN:-} BROWSER_FETCH_TOKEN: ${BROWSER_FETCH_TOKEN:-} - OBJECT_STORAGE_ACCESS_KEY: ${MINIO_ROOT_USER:-minioadmin} - OBJECT_STORAGE_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin} + OBJECT_STORAGE_ACCESS_KEY: ${OBJECT_STORAGE_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + OBJECT_STORAGE_SECRET_KEY: ${OBJECT_STORAGE_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} x-app-volumes: &app-volumes - ./config.yaml:/app/configs/config.yaml:ro diff --git a/deploy/k3s-aliyun-oss/kustomization.yaml b/deploy/k3s-aliyun-oss/kustomization.yaml new file mode 100644 index 0000000..a8ff20c --- /dev/null +++ b/deploy/k3s-aliyun-oss/kustomization.yaml @@ -0,0 +1,40 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../k3s + +patches: + - target: + version: v1 + kind: Service + name: minio + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: minio + - target: + group: apps + version: v1 + kind: StatefulSet + name: minio + patch: |- + $patch: delete + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: minio + - target: + group: batch + version: v1 + kind: Job + name: minio-init + patch: |- + $patch: delete + apiVersion: batch/v1 + kind: Job + metadata: + name: minio-init + - path: patches/object-storage-env.yaml diff --git a/deploy/k3s-aliyun-oss/patches/object-storage-env.yaml b/deploy/k3s-aliyun-oss/patches/object-storage-env.yaml new file mode 100644 index 0000000..7e93d71 --- /dev/null +++ b/deploy/k3s-aliyun-oss/patches/object-storage-env.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tenant-api +spec: + template: + spec: + containers: + - name: tenant-api + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker-generate +spec: + template: + spec: + containers: + - name: worker-generate + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kol-assist-worker +spec: + template: + spec: + containers: + - name: kol-assist-worker + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: scheduler +spec: + template: + spec: + containers: + - name: scheduler + env: + - name: OBJECT_STORAGE_ACCESS_KEY + $patch: delete + - name: OBJECT_STORAGE_SECRET_KEY + $patch: delete diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md index 783685c..9d07968 100644 --- a/deploy/k3s/README.md +++ b/deploy/k3s/README.md @@ -7,6 +7,11 @@ - apps: `tenant-api`, `browser-fetch`, `worker-generate`, `kol-assist-worker`, `scheduler`, `ops-api`, `frontend`, `ops-web` - ingress: `geo-rankly.local` -> `frontend`, `ops.geo-rankly.local` -> `ops-web` +对象存储部署跟随 `deploy/config.yaml` / `deploy/config.yml` 里的 `object_storage.provider`，并允许 `deploy/config.local.yaml` / `deploy/config.local.yml` 覆盖： + +- `provider: minio` 或 `provider: mino`：部署 MinIO 和 `minio-init` +- `provider: aliyun` / `aliyun_oss` / `aliyun-oss` / `oss`：使用 `deploy/k3s-aliyun-oss` overlay，不部署 MinIO 和 `minio-init` + ## 1. 准备镜像 在线环境建议推到镜像仓库后修改 `kustomization.yaml` 里的 `images`： @@ -59,6 +64,15 @@ kubectl apply -k deploy/k3s kubectl -n geo-rankly get pods -w ``` +如果使用阿里云 OSS： + +```bash +kubectl apply -k deploy/k3s-aliyun-oss +kubectl -n geo-rankly get pods -w +``` + +离线包里的 `bash deploy.sh k3s` 会自动读取根目录 `config.yaml/config.yml` 和 `config.local.yaml/config.local.yml`，并选择 `k3s` 或 `k3s-aliyun-oss`。 + 查看一次性任务： ```bash diff --git a/deploy/k3s/config/config.yaml b/deploy/k3s/config/config.yaml index 0596842..9073b25 100644 --- a/deploy/k3s/config/config.yaml +++ b/deploy/k3s/config/config.yaml @@ -156,14 +156,14 @@ qdrant: timeout: 15s object_storage: - provider: minio - endpoint: minio:9000 - access_key: minioadmin - secret_key: minioadmin - bucket: geo-private - use_ssl: false + provider: aliyun + endpoint: https://oss-cn-shanghai.aliyuncs.com + access_key: LTAI5tAEj8euR8B1gXzMoG84 + secret_key: lJPrKo9ViyHJE8UWIMmMCY0B6Je8v2 + bucket: shengxintui + use_ssl: true public_base_url: "" - region: "" + region: cn-shanghai cache: driver: redis diff --git a/deploy/scripts/load-and-start.sh b/deploy/scripts/load-and-start.sh index 32ef6d7..bf75f22 100755 --- a/deploy/scripts/load-and-start.sh +++ b/deploy/scripts/load-and-start.sh @@ -37,6 +37,21 @@ ensure_local_config() { fi } +ensure_config_aliases() { + local yaml yml + for yaml in config.yaml config.local.yaml ops-config.yaml ops-config.local.yaml; do + yml="${yaml%.yaml}.yml" + if [[ ! -f "${SCRIPT_DIR}/${yaml}" && -f "${SCRIPT_DIR}/${yml}" ]]; then + cp "${SCRIPT_DIR}/${yml}" "${SCRIPT_DIR}/${yaml}" + echo "==> 已从 ${yml} 创建 ${yaml}，供部署清单挂载使用。" + fi + done + if [[ ! -f "${SCRIPT_DIR}/prompts.yml" && -f "${SCRIPT_DIR}/prompts.yaml" ]]; then + cp "${SCRIPT_DIR}/prompts.yaml" "${SCRIPT_DIR}/prompts.yml" + echo "==> 已从 prompts.yaml 创建 prompts.yml，供部署清单挂载使用。" + fi +} + env_value() { local key="$1" default_value="$2" value value="$(grep -E "^${key}=" "${SCRIPT_DIR}/.env" | tail -1 | cut -d= -f2- || true)" @@ -45,9 +60,130 @@ env_value() { printf '%s' "${value:-${default_value}}" } +config_file_path() { + local base="$1" + for file in "${SCRIPT_DIR}/${base}.yaml" "${SCRIPT_DIR}/${base}.yml"; do + if [[ -f "${file}" ]]; then + printf '%s' "${file}" + return + fi + done +} + +read_object_storage_provider() { + local provider="minio" file + for file in "$(config_file_path config)" "$(config_file_path config.local)"; do + if [[ -z "${file}" || ! -f "${file}" ]]; then + continue + fi + local next + next="$( + awk ' + /^[[:space:]]*object_storage:[[:space:]]*$/ { + in_section=1 + base=match($0, /[^[:space:]]/) - 1 + next + } + in_section { + indent=match($0, /[^[:space:]]/) - 1 + if ($0 ~ /^[[:space:]]*$/ || $0 ~ /^[[:space:]]*#/) { + next + } + if (indent <= base) { + in_section=0 + next + } + if ($0 ~ /^[[:space:]]*provider[[:space:]]*:/) { + value=$0 + sub(/^[[:space:]]*provider[[:space:]]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*$/, "", value) + gsub(/^["'\'']|["'\'']$/, "", value) + print value + } + } + ' "${file}" | tail -1 + )" + if [[ -n "${next}" ]]; then + provider="${next}" + fi + done + printf '%s' "$(printf '%s' "${provider}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" +} + +use_minio_for_object_storage() { + case "$(read_object_storage_provider)" in + ""|minio|mino) + return 0 + ;; + aliyun|aliyun_oss|aliyun-oss|oss) + return 1 + ;; + *) + return 0 + ;; + esac +} + +sync_k3s_config() { + mkdir -p "${SCRIPT_DIR}/k3s/config" + for name in config ops-config prompts; do + local output="${name}.yaml" + if [[ "${name}" == "prompts" ]]; then + output="prompts.yml" + fi + local source="" + for candidate in "${SCRIPT_DIR}/${name}.yaml" "${SCRIPT_DIR}/${name}.yml"; do + if [[ -f "${candidate}" ]]; then + source="${candidate}" + break + fi + done + if [[ -n "${source}" ]]; then + cp "${source}" "${SCRIPT_DIR}/k3s/config/${output}" + fi + done +} + +prepare_k3s_runtime_overlay() { + local base_dir="$1" + local runtime_dir="${SCRIPT_DIR}/.k3s-runtime" + rm -rf "${runtime_dir}" + mkdir -p "${runtime_dir}" + + cp "$(config_file_path config.local)" "${runtime_dir}/config.local.yaml" + cp "$(config_file_path ops-config.local)" "${runtime_dir}/ops-config.local.yaml" + + cat > "${runtime_dir}/kustomization.yaml" < [1/3] 加载 Docker 镜像（可能需要几分钟）..." @@ -62,30 +198,46 @@ deploy_compose() { # ─── Step 3: 启动服务 ─────────────────────────────────────────────────────── echo "==> [3/3] 启动所有服务..." + if [[ "${minio_enabled}" == "true" ]]; then + echo " object_storage.provider=$(read_object_storage_provider)，将部署 MinIO。" + else + echo " object_storage.provider=$(read_object_storage_provider)，将跳过 MinIO。" + export COMPOSE_PROFILES="" + docker rm -f geo-minio geo-minio-init >/dev/null 2>&1 || true + fi cd "${SCRIPT_DIR}" - docker compose \ - -f docker-compose.yaml \ - -f docker-compose.offline.yaml \ - --env-file .env \ - up -d + docker compose "${compose_files[@]}" --env-file .env up -d echo "" echo "✅ Docker Compose 部署完成！" echo "" echo "服务状态：" - docker compose -f docker-compose.yaml -f docker-compose.offline.yaml ps + docker compose "${compose_files[@]}" ps echo "" - echo "查看日志： docker compose -f docker-compose.yaml logs -f tenant-api" - echo "停止服务： docker compose -f docker-compose.yaml down" + echo "查看日志： docker compose ${compose_files[*]} logs -f tenant-api" + echo "停止服务： docker compose ${compose_files[*]} down" echo "管理界面：" echo " Tenant Web: http://:$(env_value FRONTEND_PORT 18080)" echo " Ops Web: http://:$(env_value OPS_WEB_PORT 18082)" echo " RabbitMQ: http://:$(env_value RABBITMQ_MGMT_PORT 15673) (geo / geo_dev)" - echo " MinIO: http://:$(env_value MINIO_CONSOLE_PORT 19001) ($(env_value MINIO_ROOT_USER minioadmin) / $(env_value MINIO_ROOT_PASSWORD minioadmin))" + if [[ "${minio_enabled}" == "true" ]]; then + echo " MinIO: http://:$(env_value MINIO_CONSOLE_PORT 19001) ($(env_value MINIO_ROOT_USER minioadmin) / $(env_value MINIO_ROOT_PASSWORD minioadmin))" + fi } deploy_k3s() { local kubectl_cmd="kubectl" + local kustomize_dir="${SCRIPT_DIR}/k3s" + local minio_enabled="true" + + ensure_config_aliases + ensure_local_config "config.local.yaml" "config.local.yaml.example" + ensure_local_config "ops-config.local.yaml" "ops-config.local.yaml.example" + + if ! use_minio_for_object_storage; then + kustomize_dir="${SCRIPT_DIR}/k3s-aliyun-oss" + minio_enabled="false" + fi if command -v k3s >/dev/null 2>&1; then echo "==> [1/3] 导入镜像到 k3s containerd..." @@ -100,7 +252,20 @@ deploy_k3s() { fi echo "==> [2/3] 应用 k3s 清单..." - ${kubectl_cmd} apply -k "${SCRIPT_DIR}/k3s" + sync_k3s_config + kustomize_dir="$(prepare_k3s_runtime_overlay "${kustomize_dir}")" + if [[ "${minio_enabled}" == "true" ]]; then + echo " object_storage.provider=$(read_object_storage_provider)，将部署 MinIO。" + else + echo " object_storage.provider=$(read_object_storage_provider)，将跳过 MinIO。" + ${kubectl_cmd} -n geo-rankly delete statefulset minio --ignore-not-found + ${kubectl_cmd} -n geo-rankly delete service minio --ignore-not-found + ${kubectl_cmd} -n geo-rankly delete job minio-init --ignore-not-found + for svc in tenant-api worker-generate kol-assist-worker scheduler; do + ${kubectl_cmd} -n geo-rankly set env "deployment/${svc}" OBJECT_STORAGE_ACCESS_KEY- OBJECT_STORAGE_SECRET_KEY- >/dev/null 2>&1 || true + done + fi + ${kubectl_cmd} apply -k "${kustomize_dir}" echo "==> [3/3] 查看服务状态..." ${kubectl_cmd} -n geo-rankly get pods,svc diff --git a/deploy/scripts/package.sh b/deploy/scripts/package.sh index 5d334a7..c767d0b 100755 --- a/deploy/scripts/package.sh +++ b/deploy/scripts/package.sh @@ -159,12 +159,16 @@ echo " 镜像包大小: $(du -sh "${WORK_DIR}/images.tar" | cut -f1)" echo "==> [4/5] 打包部署文件" cp "${DEPLOY_DIR}/docker-compose.yaml" "${WORK_DIR}/" cp "${DEPLOY_DIR}/docker-compose.offline.yaml" "${WORK_DIR}/" +cp "${DEPLOY_DIR}/docker-compose.aliyun-oss.yaml" "${WORK_DIR}/" cp "${DEPLOY_DIR}/config.yaml" "${WORK_DIR}/" cp "${DEPLOY_DIR}/ops-config.yaml" "${WORK_DIR}/" cp "${DEPLOY_DIR}/prompts.yml" "${WORK_DIR}/" cp "${DEPLOY_DIR}/.env.example" "${WORK_DIR}/" +cp "${DEPLOY_DIR}/config.local.yaml.example" "${WORK_DIR}/" +cp "${DEPLOY_DIR}/ops-config.local.yaml.example" "${WORK_DIR}/" cp "${DEPLOY_DIR}/scripts/load-and-start.sh" "${WORK_DIR}/deploy.sh" cp -R "${DEPLOY_DIR}/k3s" "${WORK_DIR}/k3s" +cp -R "${DEPLOY_DIR}/k3s-aliyun-oss" "${WORK_DIR}/k3s-aliyun-oss" if [[ -f "${DEPLOY_DIR}/OFFLINE_PACKAGE.md" ]]; then cp "${DEPLOY_DIR}/OFFLINE_PACKAGE.md" "${WORK_DIR}/README.md" fi @@ -201,7 +205,10 @@ Files: images.tar docker-compose.yaml docker-compose.offline.yaml + docker-compose.aliyun-oss.yaml .env.example + config.local.yaml.example + ops-config.local.yaml.example deploy.sh EOF diff --git a/server/configs/config.yaml b/server/configs/config.yaml index 2a4fd88..3be7c63 100644 --- a/server/configs/config.yaml +++ b/server/configs/config.yaml @@ -156,14 +156,14 @@ qdrant: timeout: 15s object_storage: - provider: minio - endpoint: localhost:9000 - access_key: minioadmin - secret_key: minioadmin - bucket: geo-private - use_ssl: false + provider: aliyun + endpoint: https://oss-cn-shanghai.aliyuncs.com + access_key: LTAI5tAEj8euR8B1gXzMoG84 + secret_key: lJPrKo9ViyHJE8UWIMmMCY0B6Je8v2 + bucket: shengxintui + use_ssl: true public_base_url: "" - region: "" + region: cn-shanghai cache: driver: redis diff --git a/server/internal/ops/domain/scheduler.go b/server/internal/ops/domain/scheduler.go new file mode 100644 index 0000000..e9290eb --- /dev/null +++ b/server/internal/ops/domain/scheduler.go @@ -0,0 +1,79 @@ +package domain + +import "time" + +const ( + SchedulerTriggerAuto = "auto" + SchedulerTriggerManual = "manual" + SchedulerTriggerDryRun = "dry_run" + + SchedulerRunRunning = "running" + SchedulerRunSuccess = "success" + SchedulerRunFailed = "failed" + SchedulerRunSkipped = "skipped" +) + +type SchedulerJob struct { + JobKey string `json:"job_key"` + DisplayName string `json:"display_name"` + Category string `json:"category"` + Description *string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + ScheduleType string `json:"schedule_type"` + IntervalSeconds int `json:"interval_seconds"` + CronExpr *string `json:"cron_expr,omitempty"` + Timezone string `json:"timezone"` + TimeoutSeconds int `json:"timeout_seconds"` + BatchSize *int `json:"batch_size,omitempty"` + MaxConcurrency int `json:"max_concurrency"` + Config map[string]any `json:"config"` + Version int `json:"version"` + UpdatedBy *int64 `json:"updated_by,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + CreatedAt time.Time `json:"created_at"` + LastRun *SchedulerRun `json:"last_run,omitempty"` + RunningRun *SchedulerRun `json:"running_run,omitempty"` + PendingTriggers int `json:"pending_triggers"` +} + +type SchedulerRun struct { + ID int64 `json:"id"` + JobKey string `json:"job_key"` + TriggerID *int64 `json:"trigger_id,omitempty"` + SchedulerInstanceID *string `json:"scheduler_instance_id,omitempty"` + TriggerType string `json:"trigger_type"` + Status string `json:"status"` + ConfigVersion int `json:"config_version"` + ConfigSnapshot map[string]any `json:"config_snapshot"` + Stats map[string]any `json:"stats"` + ErrorMessage *string `json:"error_message,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + DurationMilliseconds *int64 `json:"duration_ms,omitempty"` +} + +type SchedulerInstance struct { + InstanceID string `json:"instance_id"` + Hostname string `json:"hostname"` + Process string `json:"process"` + BuildVersion *string `json:"build_version,omitempty"` + StartedAt time.Time `json:"started_at"` + LastSeenAt time.Time `json:"last_seen_at"` + Metadata map[string]any `json:"metadata"` +} + +type SchedulerTrigger struct { + ID int64 `json:"id"` + JobKey string `json:"job_key"` + TriggerType string `json:"trigger_type"` + Status string `json:"status"` + RequestedBy *int64 `json:"requested_by,omitempty"` + Reason *string `json:"reason,omitempty"` + ConfigOverride map[string]any `json:"config_override"` + SchedulerInstanceID *string `json:"scheduler_instance_id,omitempty"` + RunID *int64 `json:"run_id,omitempty"` + RequestedAt time.Time `json:"requested_at"` + ClaimedAt *time.Time `json:"claimed_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` +} diff --git a/server/internal/shared/objectstorage/client.go b/server/internal/shared/objectstorage/client.go index 57ead75..d693fc1 100644 --- a/server/internal/shared/objectstorage/client.go +++ b/server/internal/shared/objectstorage/client.go @@ -27,7 +27,7 @@ func New(cfg config.ObjectStorageConfig, logger *zap.Logger) Client { switch provider { case "", "disabled": return disabledClient{reason: "object storage is disabled"} - case "minio": + case "minio", "mino": return NewMinIOClient(cfg, logger) case "aliyun", "aliyun_oss", "aliyun-oss", "oss": return NewAliyunClient(cfg, logger) diff --git a/server/migrations_ops/20260520120000_create_scheduler_control_tables.down.sql b/server/migrations_ops/20260520120000_create_scheduler_control_tables.down.sql new file mode 100644 index 0000000..ed1334f --- /dev/null +++ b/server/migrations_ops/20260520120000_create_scheduler_control_tables.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS ops.scheduler_job_triggers; +DROP TABLE IF EXISTS ops.scheduler_job_runs; +DROP TABLE IF EXISTS ops.scheduler_instances; +DROP TABLE IF EXISTS ops.scheduler_jobs; diff --git a/server/migrations_ops/20260520120000_create_scheduler_control_tables.up.sql b/server/migrations_ops/20260520120000_create_scheduler_control_tables.up.sql new file mode 100644 index 0000000..802c627 --- /dev/null +++ b/server/migrations_ops/20260520120000_create_scheduler_control_tables.up.sql @@ -0,0 +1,104 @@ +CREATE TABLE IF NOT EXISTS ops.scheduler_jobs ( + job_key VARCHAR(96) PRIMARY KEY, + display_name VARCHAR(120) NOT NULL, + category VARCHAR(64) NOT NULL, + description TEXT, + enabled BOOLEAN NOT NULL DEFAULT true, + schedule_type VARCHAR(20) NOT NULL DEFAULT 'interval', + interval_seconds INT NOT NULL DEFAULT 300, + cron_expr VARCHAR(120), + timezone VARCHAR(64) NOT NULL DEFAULT 'Asia/Shanghai', + timeout_seconds INT NOT NULL DEFAULT 30, + batch_size INT, + max_concurrency INT NOT NULL DEFAULT 1, + config JSONB NOT NULL DEFAULT '{}'::jsonb, + version INT NOT NULL DEFAULT 1, + updated_by BIGINT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK (job_key ~ '^[a-z0-9_:-]+$'), + CHECK (schedule_type IN ('interval', 'cron', 'manual')), + CHECK (interval_seconds > 0), + CHECK (timeout_seconds > 0), + CHECK (max_concurrency > 0) +); + +CREATE INDEX IF NOT EXISTS idx_scheduler_jobs_category + ON ops.scheduler_jobs (category, job_key); + +CREATE TABLE IF NOT EXISTS ops.scheduler_instances ( + instance_id VARCHAR(160) PRIMARY KEY, + hostname VARCHAR(255) NOT NULL, + process VARCHAR(80) NOT NULL, + build_version VARCHAR(120), + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE INDEX IF NOT EXISTS idx_scheduler_instances_seen + ON ops.scheduler_instances (last_seen_at DESC); + +CREATE TABLE IF NOT EXISTS ops.scheduler_job_runs ( + id BIGSERIAL PRIMARY KEY, + job_key VARCHAR(96) NOT NULL REFERENCES ops.scheduler_jobs(job_key), + trigger_id BIGINT, + scheduler_instance_id VARCHAR(160), + trigger_type VARCHAR(30) NOT NULL DEFAULT 'auto', + status VARCHAR(30) NOT NULL DEFAULT 'running', + config_version INT NOT NULL DEFAULT 1, + config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + stats JSONB NOT NULL DEFAULT '{}'::jsonb, + error_message TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + finished_at TIMESTAMPTZ, + duration_ms BIGINT, + CHECK (trigger_type IN ('auto', 'manual', 'dry_run')), + CHECK (status IN ('running', 'success', 'failed', 'skipped')) +); + +CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_job_started + ON ops.scheduler_job_runs (job_key, started_at DESC, id DESC); + +CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_status_started + ON ops.scheduler_job_runs (status, started_at DESC); + +CREATE TABLE IF NOT EXISTS ops.scheduler_job_triggers ( + id BIGSERIAL PRIMARY KEY, + job_key VARCHAR(96) NOT NULL REFERENCES ops.scheduler_jobs(job_key), + trigger_type VARCHAR(30) NOT NULL DEFAULT 'manual', + status VARCHAR(30) NOT NULL DEFAULT 'pending', + requested_by BIGINT, + reason TEXT, + config_override JSONB NOT NULL DEFAULT '{}'::jsonb, + scheduler_instance_id VARCHAR(160), + run_id BIGINT REFERENCES ops.scheduler_job_runs(id), + requested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + claimed_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + error_message TEXT, + CHECK (trigger_type IN ('manual', 'dry_run')), + CHECK (status IN ('pending', 'claimed', 'success', 'failed', 'skipped')) +); + +CREATE INDEX IF NOT EXISTS idx_scheduler_job_triggers_pending + ON ops.scheduler_job_triggers (job_key, requested_at ASC, id ASC) + WHERE status = 'pending'; + +INSERT INTO ops.scheduler_jobs + (job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config) +VALUES + ('schedule_dispatch', '内容定时任务派发', 'content', '扫描内容定时任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb), + ('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '为品牌问题按业务日生成 AI 监控采集任务。', true, 'interval', 300, 'Asia/Shanghai', 45, 64, '{}'::jsonb), + ('monitoring_result_recovery', '监控结果恢复', 'monitoring', '恢复已回调但未成功进入结果队列的监控采集结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb), + ('monitoring_lease_recovery', '监控租约回收', 'monitoring', '回收超过租约时间仍未回调的监控采集任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, NULL, '{}'::jsonb), + ('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '检查长时间停留在 received 状态的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{}'::jsonb), + ('knowledge_deleted_cleanup', '知识库删除清理', 'knowledge', '清理已删除知识库条目的异步对象与向量数据。', true, 'interval', 15, 'Asia/Shanghai', 30, NULL, '{}'::jsonb), + ('generation_state_check', '生成任务状态巡检', 'generation', '低频巡检文章生成任务与文章状态一致性。', true, 'interval', 300, 'Asia/Shanghai', 10, 100, '{}'::jsonb), + ('monitoring_retention_cleanup', '监控历史保留清理', 'monitoring', '按业务日期清理最近 30 天窗口以前的监控历史数据。', true, 'interval', 86400, 'Asia/Shanghai', 900, 5000, + '{"retention_days":30,"batch_size":5000,"max_batches_per_run":200,"dry_run":false,"statement_timeout":"5s","run_at_local":"02:30"}'::jsonb) +ON CONFLICT (job_key) DO UPDATE +SET display_name = EXCLUDED.display_name, + category = EXCLUDED.category, + description = EXCLUDED.description, + updated_at = NOW();