root
7f08d92958
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
229 lines
8.7 KiB
Bash
Executable File
229 lines
8.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# package.sh — 在【有网络的开发机】上运行
|
||
# 构建所有镜像（支持跨平台），打包成可离线部署的压缩包
|
||
#
|
||
# 用法：
|
||
# bash package.sh [IMAGE_TAG] [PLATFORM]
|
||
#
|
||
# 示例：
|
||
# bash package.sh latest linux/amd64 # M1/M2 Mac 打包给 x86 服务器（默认）
|
||
# bash package.sh v1.0.0 linux/arm64 # 打包给 ARM 服务器（如树莓派、飞腾）
|
||
# bash package.sh latest linux/amd64,linux/arm64 # 暂不支持（save 不支持 multi-arch）
|
||
#
|
||
# 产物：
|
||
# geo-rankly-deploy-<image-tag>-<date>-<arch>.tar.gz
|
||
|
||
set -euo pipefail
|
||
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
DEPLOY_DIR="${REPO_ROOT}/deploy"
|
||
IMAGE_TAG="${1:-latest}"
|
||
PLATFORM="${2:-linux/amd64}"
|
||
LOCAL_PREFIX="${LOCAL_PREFIX:-geo-rankly}"
|
||
SOURCE_IMAGE_REGISTRY="${SOURCE_IMAGE_REGISTRY:-}"
|
||
OUTPUT_DIR="${PACKAGE_OUTPUT_DIR:-${REPO_ROOT}}"
|
||
DATE_TAG="$(date +%Y%m%d)"
|
||
# 将 linux/amd64 → amd64，用于包名
|
||
ARCH_LABEL="${PLATFORM##*/}"
|
||
SAFE_IMAGE_TAG="$(printf '%s' "${IMAGE_TAG}" | tr '/:@' '---' | tr -c 'A-Za-z0-9_.-' '-')"
|
||
PACKAGE_NAME="geo-rankly-deploy-${SAFE_IMAGE_TAG}-${DATE_TAG}-${ARCH_LABEL}"
|
||
WORK_DIR="/tmp/${PACKAGE_NAME}"
|
||
PACKAGE_PATH="${OUTPUT_DIR}/${PACKAGE_NAME}.tar.gz"
|
||
|
||
echo "==> 目标平台: ${PLATFORM} | 镜像 tag: ${IMAGE_TAG}"
|
||
echo "==> 输出目录: ${OUTPUT_DIR}"
|
||
echo ""
|
||
|
||
# ─── 确保 buildx builder 支持目标平台 ────────────────────────────────────────
|
||
echo "==> [0/5] 检查 buildx builder"
|
||
BUILDER_NAME="geo-cross-builder"
|
||
|
||
if ! docker buildx inspect "${BUILDER_NAME}" &>/dev/null; then
|
||
echo " 创建多平台 builder: ${BUILDER_NAME}"
|
||
docker buildx create \
|
||
--name "${BUILDER_NAME}" \
|
||
--driver docker-container \
|
||
--driver-opt image=moby/buildkit:latest \
|
||
--use
|
||
else
|
||
docker buildx use "${BUILDER_NAME}"
|
||
echo " 使用已有 builder: ${BUILDER_NAME}"
|
||
fi
|
||
|
||
# 确保 builder 已启动（首次使用时需要）
|
||
docker buildx inspect --bootstrap "${BUILDER_NAME}" > /dev/null
|
||
|
||
# ─── 准备自定义镜像 ────────────────────────────────────────────────────────────
|
||
echo "==> [1/5] 准备自定义镜像 (platform: ${PLATFORM}, tag: ${IMAGE_TAG})"
|
||
cd "${REPO_ROOT}"
|
||
|
||
if [[ -n "${SOURCE_IMAGE_REGISTRY}" ]]; then
|
||
echo " pulling from ${SOURCE_IMAGE_REGISTRY}"
|
||
for svc in tenant-api browser-fetch worker-generate kol-assist-worker ops-api scheduler migrate frontend ops-web; do
|
||
echo " pulling ${SOURCE_IMAGE_REGISTRY}/${svc}:${IMAGE_TAG}"
|
||
docker pull --platform "${PLATFORM}" "${SOURCE_IMAGE_REGISTRY}/${svc}:${IMAGE_TAG}"
|
||
docker tag "${SOURCE_IMAGE_REGISTRY}/${svc}:${IMAGE_TAG}" "${LOCAL_PREFIX}/${svc}:${IMAGE_TAG}"
|
||
done
|
||
else
|
||
# 注意：--load 仅支持单平台；交叉编译时 buildx 使用 QEMU 或 Go 交叉编译
|
||
_build() {
|
||
local target="$1" tag_suffix="$2"
|
||
shift 2
|
||
echo " building ${LOCAL_PREFIX}/${tag_suffix}:${IMAGE_TAG}"
|
||
docker buildx build \
|
||
--platform "${PLATFORM}" \
|
||
--target "${target}" \
|
||
"$@" \
|
||
--tag "${LOCAL_PREFIX}/${tag_suffix}:${IMAGE_TAG}" \
|
||
--load \
|
||
-f server/Dockerfile \
|
||
server/
|
||
}
|
||
|
||
_build runtime tenant-api --build-arg SERVICE=tenant-api
|
||
_build browser-fetch browser-fetch --build-arg SERVICE=browser-fetch
|
||
_build runtime worker-generate --build-arg SERVICE=worker-generate
|
||
_build runtime kol-assist-worker --build-arg SERVICE=kol-assist-worker
|
||
_build runtime ops-api --build-arg SERVICE=ops-api
|
||
_build runtime scheduler --build-arg SERVICE=scheduler
|
||
_build migrate migrate # migrate stage 不需要 SERVICE arg
|
||
|
||
echo " building ${LOCAL_PREFIX}/frontend:${IMAGE_TAG}"
|
||
docker buildx build \
|
||
--platform "${PLATFORM}" \
|
||
--tag "${LOCAL_PREFIX}/frontend:${IMAGE_TAG}" \
|
||
--load \
|
||
-f Dockerfile.frontend \
|
||
.
|
||
|
||
echo " building ${LOCAL_PREFIX}/ops-web:${IMAGE_TAG}"
|
||
docker buildx build \
|
||
--platform "${PLATFORM}" \
|
||
--tag "${LOCAL_PREFIX}/ops-web:${IMAGE_TAG}" \
|
||
--load \
|
||
-f Dockerfile.ops-web \
|
||
.
|
||
fi
|
||
|
||
# ─── 拉取基础设施镜像（指定平台） ─────────────────────────────────────────────
|
||
echo "==> [2/5] 拉取基础设施镜像 (platform: ${PLATFORM})"
|
||
INFRA_IMAGES=(
|
||
"postgres:16-alpine"
|
||
"edoburu/pgbouncer:v1.24.1-p1"
|
||
"rabbitmq:3.13-management-alpine"
|
||
"redis:7-alpine"
|
||
"qdrant/qdrant:v1.13.2"
|
||
"minio/minio:RELEASE.2025-03-12T18-04-18Z"
|
||
"minio/mc:latest"
|
||
)
|
||
for img in "${INFRA_IMAGES[@]}"; do
|
||
echo " pulling ${img}"
|
||
# --platform 确保拉取目标架构，覆盖本机可能已缓存的 arm64 版本
|
||
docker pull --platform "${PLATFORM}" "${img}"
|
||
done
|
||
|
||
# ─── 保存所有镜像 ───────────────────────────────────────────────────────────
|
||
echo "==> [3/5] 保存所有镜像到 images.tar"
|
||
mkdir -p "${WORK_DIR}"
|
||
|
||
ALL_IMAGES=(
|
||
"${LOCAL_PREFIX}/tenant-api:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/browser-fetch:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/worker-generate:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/kol-assist-worker:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/scheduler:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/ops-api:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/migrate:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/frontend:${IMAGE_TAG}"
|
||
"${LOCAL_PREFIX}/ops-web:${IMAGE_TAG}"
|
||
"postgres:16-alpine"
|
||
"edoburu/pgbouncer:v1.24.1-p1"
|
||
"rabbitmq:3.13-management-alpine"
|
||
"redis:7-alpine"
|
||
"qdrant/qdrant:v1.13.2"
|
||
"minio/minio:RELEASE.2025-03-12T18-04-18Z"
|
||
"minio/mc:latest"
|
||
)
|
||
|
||
docker save "${ALL_IMAGES[@]}" -o "${WORK_DIR}/images.tar"
|
||
|
||
# 验证镜像架构
|
||
echo " 验证镜像架构："
|
||
for img in "${LOCAL_PREFIX}/tenant-api:${IMAGE_TAG}" "postgres:16-alpine"; do
|
||
arch="$(docker inspect --format '{{.Architecture}}' "${img}" 2>/dev/null || echo unknown)"
|
||
echo " ${img} → ${arch}"
|
||
done
|
||
echo " 镜像包大小: $(du -sh "${WORK_DIR}/images.tar" | cut -f1)"
|
||
|
||
# ─── 打包部署文件 ───────────────────────────────────────────────────────────
|
||
echo "==> [4/5] 打包部署文件"
|
||
cp "${DEPLOY_DIR}/docker-compose.yaml" "${WORK_DIR}/"
|
||
cp "${DEPLOY_DIR}/docker-compose.offline.yaml" "${WORK_DIR}/"
|
||
cp "${DEPLOY_DIR}/config.yaml" "${WORK_DIR}/"
|
||
cp "${DEPLOY_DIR}/ops-config.yaml" "${WORK_DIR}/"
|
||
cp "${DEPLOY_DIR}/prompts.yml" "${WORK_DIR}/"
|
||
cp "${DEPLOY_DIR}/.env.example" "${WORK_DIR}/"
|
||
cp "${DEPLOY_DIR}/scripts/load-and-start.sh" "${WORK_DIR}/deploy.sh"
|
||
cp -R "${DEPLOY_DIR}/k3s" "${WORK_DIR}/k3s"
|
||
if [[ -f "${DEPLOY_DIR}/OFFLINE_PACKAGE.md" ]]; then
|
||
cp "${DEPLOY_DIR}/OFFLINE_PACKAGE.md" "${WORK_DIR}/README.md"
|
||
fi
|
||
if [[ -f "${WORK_DIR}/k3s/kustomization.yaml" ]]; then
|
||
awk -v tag="${IMAGE_TAG}" '
|
||
$1 == "newTag:" && $2 == "latest" { print " newTag: " tag; next }
|
||
{ print }
|
||
' "${WORK_DIR}/k3s/kustomization.yaml" > "${WORK_DIR}/k3s/kustomization.yaml.tmp"
|
||
mv "${WORK_DIR}/k3s/kustomization.yaml.tmp" "${WORK_DIR}/k3s/kustomization.yaml"
|
||
fi
|
||
chmod +x "${WORK_DIR}/deploy.sh"
|
||
|
||
# 写入元数据，供 load 脚本读取
|
||
cat > "${WORK_DIR}/.package-meta" <<EOF
|
||
IMAGE_TAG=${IMAGE_TAG}
|
||
LOCAL_PREFIX=${LOCAL_PREFIX}
|
||
PLATFORM=${PLATFORM}
|
||
BUILT_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||
BUILT_ON=$(uname -m)
|
||
EOF
|
||
|
||
cat > "${WORK_DIR}/MANIFEST.txt" <<EOF
|
||
geo-rankly offline deployment package
|
||
|
||
Image tag: ${IMAGE_TAG}
|
||
Platform: ${PLATFORM}
|
||
Local image prefix: ${LOCAL_PREFIX}
|
||
Built at: $(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
Start:
|
||
bash deploy.sh
|
||
|
||
Files:
|
||
images.tar
|
||
docker-compose.yaml
|
||
docker-compose.offline.yaml
|
||
.env.example
|
||
deploy.sh
|
||
EOF
|
||
|
||
# ─── 压缩 ──────────────────────────────────────────────────────────────────
|
||
echo "==> [5/5] 压缩打包"
|
||
mkdir -p "${OUTPUT_DIR}"
|
||
cd /tmp
|
||
tar -czf "${PACKAGE_PATH}" "${PACKAGE_NAME}/"
|
||
if command -v sha256sum >/dev/null 2>&1; then
|
||
sha256sum "${PACKAGE_PATH}" > "${PACKAGE_PATH}.sha256"
|
||
else
|
||
shasum -a 256 "${PACKAGE_PATH}" > "${PACKAGE_PATH}.sha256"
|
||
fi
|
||
rm -rf "${WORK_DIR}"
|
||
|
||
echo ""
|
||
echo "✅ 打包完成: ${PACKAGE_PATH}"
|
||
echo " 校验文件: ${PACKAGE_PATH}.sha256"
|
||
echo " 平台: ${PLATFORM} | 大小: $(du -sh "${PACKAGE_PATH}" | cut -f1)"
|
||
echo ""
|
||
echo "传输到目标服务器后执行："
|
||
echo " tar xzf ${PACKAGE_NAME}.tar.gz"
|
||
echo " cd ${PACKAGE_NAME}"
|
||
echo " bash deploy.sh"
|