7f08d92958
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
368 lines
9.5 KiB
Bash
368 lines
9.5 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
MODE="${1:-ensure}"
|
|
|
|
REGISTRY_HOST="${REGISTRY_HOST:-192.168.100.19:13000}"
|
|
REGISTRY_HOST="${REGISTRY_HOST#http://}"
|
|
REGISTRY_HOST="${REGISTRY_HOST#https://}"
|
|
REGISTRY_HOST="${REGISTRY_HOST%/}"
|
|
REGISTRY_OWNER="${REGISTRY_OWNER:-root}"
|
|
REGISTRY_IMAGE_PREFIX="${REGISTRY_IMAGE_PREFIX:-geo-rankly}"
|
|
REGISTRY_USERNAME="${REGISTRY_USERNAME:-root}"
|
|
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
|
|
REGISTRY_PLAIN_HTTP="${REGISTRY_PLAIN_HTTP:-true}"
|
|
REGISTRY_SCHEME="${REGISTRY_SCHEME:-}"
|
|
IMAGE_TAG="${IMAGE_TAG:-}"
|
|
IMAGE_LIST="${IMAGE_LIST:-tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web migrate}"
|
|
DELETE_OLD_IMAGE_TAGS="${DELETE_OLD_IMAGE_TAGS:-true}"
|
|
PUSH_LATEST="${PUSH_LATEST:-true}"
|
|
FORCE_BUILD="${FORCE_BUILD:-false}"
|
|
TARGET_PLATFORM="${TARGET_PLATFORM:-}"
|
|
|
|
case "${MODE}" in
|
|
ensure|verify) ;;
|
|
*)
|
|
echo "Usage: $0 [ensure|verify]" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if [[ -z "${IMAGE_TAG}" ]]; then
|
|
IMAGE_TAG="$(git -C "${REPO_ROOT}" rev-parse --short=8 HEAD)"
|
|
fi
|
|
|
|
if [[ -z "${REGISTRY_SCHEME}" ]]; then
|
|
if [[ "${REGISTRY_PLAIN_HTTP}" == "true" ]]; then
|
|
REGISTRY_SCHEME="http"
|
|
else
|
|
REGISTRY_SCHEME="https"
|
|
fi
|
|
fi
|
|
|
|
case "${IMAGE_TAG}" in
|
|
*[!A-Za-z0-9_.-]*|"")
|
|
echo "IMAGE_TAG may only contain letters, numbers, underscore, dot, and dash." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
for value_name in REGISTRY_HOST REGISTRY_OWNER REGISTRY_IMAGE_PREFIX REGISTRY_USERNAME; do
|
|
value="${!value_name}"
|
|
if [[ -z "${value}" ]]; then
|
|
echo "${value_name} is required." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if [[ -z "${REGISTRY_PASSWORD}" ]]; then
|
|
echo "REGISTRY_PASSWORD is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
case "${REGISTRY_PLAIN_HTTP}" in
|
|
true|false) ;;
|
|
*)
|
|
echo "REGISTRY_PLAIN_HTTP must be true or false." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
echo "curl is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${MODE}" == "ensure" ]]; then
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "docker is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${REGISTRY_PLAIN_HTTP}" == "true" ]] \
|
|
&& ! docker info 2>/dev/null \
|
|
| sed -n '/Insecure Registries:/,/Live Restore/p' \
|
|
| grep -Fq "${REGISTRY_HOST}"; then
|
|
echo "${REGISTRY_HOST} is not listed in Docker insecure registries." >&2
|
|
echo "Configure the runner Docker daemon with: {\"insecure-registries\":[\"${REGISTRY_HOST}\"]}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s\n' "${REGISTRY_PASSWORD}" \
|
|
| docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin
|
|
fi
|
|
|
|
image_name() {
|
|
local svc="$1"
|
|
printf '%s/%s/%s/%s' \
|
|
"${REGISTRY_HOST}" \
|
|
"${REGISTRY_OWNER}" \
|
|
"${REGISTRY_IMAGE_PREFIX}" \
|
|
"${svc}"
|
|
}
|
|
|
|
registry_repo_path() {
|
|
local svc="$1"
|
|
printf '%s/%s/%s' "${REGISTRY_OWNER}" "${REGISTRY_IMAGE_PREFIX}" "${svc}"
|
|
}
|
|
|
|
registry_api_base() {
|
|
local svc="$1"
|
|
printf '%s://%s/v2/%s' \
|
|
"${REGISTRY_SCHEME}" \
|
|
"${REGISTRY_HOST}" \
|
|
"$(registry_repo_path "${svc}")"
|
|
}
|
|
|
|
curl_registry() {
|
|
curl -fsS -u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" "$@"
|
|
}
|
|
|
|
manifest_digest() {
|
|
local svc="$1" tag="$2" base headers status digest
|
|
base="$(registry_api_base "${svc}")"
|
|
headers="$(mktemp)"
|
|
status="$(
|
|
curl -sS \
|
|
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
|
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
|
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
|
|
-D "${headers}" \
|
|
-o /dev/null \
|
|
-w '%{http_code}' \
|
|
"${base}/manifests/${tag}" || true
|
|
)"
|
|
|
|
case "${status}" in
|
|
200)
|
|
digest="$(awk 'BEGIN{IGNORECASE=1} /^Docker-Content-Digest:/ {gsub("\r", "", $2); print $2; exit}' "${headers}")"
|
|
rm -f "${headers}"
|
|
printf '%s' "${digest}"
|
|
;;
|
|
404)
|
|
rm -f "${headers}"
|
|
return 1
|
|
;;
|
|
401|403)
|
|
rm -f "${headers}"
|
|
echo "Registry authentication failed while checking ${svc}:${tag}." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
rm -f "${headers}"
|
|
echo "Failed to check ${svc}:${tag}; registry returned HTTP ${status}." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
tag_exists() {
|
|
local svc="$1" tag="$2"
|
|
manifest_digest "${svc}" "${tag}" >/dev/null
|
|
}
|
|
|
|
list_tags() {
|
|
local svc="$1" base json
|
|
base="$(registry_api_base "${svc}")"
|
|
json="$(curl_registry "${base}/tags/list" 2>/dev/null || true)"
|
|
if [[ -z "${json}" ]]; then
|
|
return 0
|
|
fi
|
|
printf '%s' "${json}" \
|
|
| tr -d '\n' \
|
|
| sed -n 's/.*"tags":[[]\([^]]*\)[]].*/\1/p' \
|
|
| tr ',' '\n' \
|
|
| tr -d '" ' \
|
|
| sed '/^$/d'
|
|
}
|
|
|
|
delete_manifest_digest() {
|
|
local svc="$1" digest="$2" base status
|
|
base="$(registry_api_base "${svc}")"
|
|
status="$(
|
|
curl -sS \
|
|
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
|
-X DELETE \
|
|
-o /dev/null \
|
|
-w '%{http_code}' \
|
|
"${base}/manifests/${digest}" || true
|
|
)"
|
|
|
|
case "${status}" in
|
|
200|202|404)
|
|
return 0
|
|
;;
|
|
405)
|
|
echo "Registry does not allow deleting manifests for ${svc}; skipping cleanup." >&2
|
|
return 0
|
|
;;
|
|
*)
|
|
echo "Failed to delete old manifest ${svc}@${digest}; HTTP ${status}. Continuing." >&2
|
|
return 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
cleanup_old_hash_tags() {
|
|
local svc="$1" tag digest current_digest latest_digest
|
|
|
|
if [[ "${DELETE_OLD_IMAGE_TAGS}" != "true" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
current_digest="$(manifest_digest "${svc}" "${IMAGE_TAG}" 2>/dev/null || true)"
|
|
latest_digest="$(manifest_digest "${svc}" "latest" 2>/dev/null || true)"
|
|
|
|
while IFS= read -r tag; do
|
|
[[ -n "${tag}" ]] || continue
|
|
[[ "${tag}" != "${IMAGE_TAG}" ]] || continue
|
|
[[ "${tag}" != "latest" ]] || continue
|
|
if [[ ! "${tag}" =~ ^[0-9a-f]{8}$ ]]; then
|
|
continue
|
|
fi
|
|
|
|
digest="$(manifest_digest "${svc}" "${tag}" 2>/dev/null || true)"
|
|
if [[ -z "${digest}" ]]; then
|
|
continue
|
|
fi
|
|
if [[ -n "${current_digest}" && "${digest}" == "${current_digest}" ]]; then
|
|
continue
|
|
fi
|
|
if [[ -n "${latest_digest}" && "${digest}" == "${latest_digest}" ]]; then
|
|
continue
|
|
fi
|
|
|
|
echo "Deleting old image tag ${svc}:${tag}"
|
|
delete_manifest_digest "${svc}" "${digest}"
|
|
done < <(list_tags "${svc}")
|
|
}
|
|
|
|
platform_args=()
|
|
if [[ -n "${TARGET_PLATFORM}" ]]; then
|
|
platform_args=(--platform "${TARGET_PLATFORM}")
|
|
fi
|
|
|
|
build_image() {
|
|
local svc="$1" repo
|
|
repo="$(image_name "${svc}")"
|
|
|
|
case "${svc}" in
|
|
migrate)
|
|
docker build "${platform_args[@]}" \
|
|
--target migrate \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/server/Dockerfile" \
|
|
"${REPO_ROOT}/server"
|
|
;;
|
|
browser-fetch)
|
|
docker build "${platform_args[@]}" \
|
|
--target browser-fetch \
|
|
--build-arg "SERVICE=browser-fetch" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/server/Dockerfile" \
|
|
"${REPO_ROOT}/server"
|
|
;;
|
|
tenant-api|worker-generate|kol-assist-worker|scheduler|ops-api)
|
|
docker build "${platform_args[@]}" \
|
|
--target runtime \
|
|
--build-arg "SERVICE=${svc}" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/server/Dockerfile" \
|
|
"${REPO_ROOT}/server"
|
|
;;
|
|
frontend)
|
|
docker build "${platform_args[@]}" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/Dockerfile.frontend" \
|
|
"${REPO_ROOT}"
|
|
;;
|
|
ops-web)
|
|
docker build "${platform_args[@]}" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/Dockerfile.ops-web" \
|
|
"${REPO_ROOT}"
|
|
;;
|
|
*)
|
|
echo "Unsupported image service: ${svc}" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
push_latest_from_existing_tag() {
|
|
local svc="$1" repo tag_digest latest_digest
|
|
|
|
if [[ "${PUSH_LATEST}" != "true" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
tag_digest="$(manifest_digest "${svc}" "${IMAGE_TAG}" 2>/dev/null || true)"
|
|
latest_digest="$(manifest_digest "${svc}" "latest" 2>/dev/null || true)"
|
|
if [[ -n "${tag_digest}" && "${tag_digest}" == "${latest_digest}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
repo="$(image_name "${svc}")"
|
|
echo "Updating ${svc}:latest to existing ${svc}:${IMAGE_TAG}"
|
|
docker pull "${repo}:${IMAGE_TAG}"
|
|
docker tag "${repo}:${IMAGE_TAG}" "${repo}:latest"
|
|
docker push "${repo}:latest"
|
|
}
|
|
|
|
ensure_image() {
|
|
local svc="$1" repo
|
|
repo="$(image_name "${svc}")"
|
|
|
|
if [[ "${FORCE_BUILD}" != "true" ]] && tag_exists "${svc}" "${IMAGE_TAG}"; then
|
|
echo "Image already current: ${repo}:${IMAGE_TAG}; skipping build."
|
|
push_latest_from_existing_tag "${svc}"
|
|
cleanup_old_hash_tags "${svc}"
|
|
return 0
|
|
fi
|
|
|
|
echo "Building image: ${repo}:${IMAGE_TAG}"
|
|
build_image "${svc}"
|
|
docker push "${repo}:${IMAGE_TAG}"
|
|
if [[ "${PUSH_LATEST}" == "true" ]]; then
|
|
docker push "${repo}:latest"
|
|
fi
|
|
cleanup_old_hash_tags "${svc}"
|
|
}
|
|
|
|
verify_image() {
|
|
local svc="$1" repo
|
|
repo="$(image_name "${svc}")"
|
|
|
|
if tag_exists "${svc}" "${IMAGE_TAG}"; then
|
|
echo "Image is current: ${repo}:${IMAGE_TAG}"
|
|
else
|
|
echo "Missing current image: ${repo}:${IMAGE_TAG}" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
echo "Gitea Registry: ${REGISTRY_SCHEME}://${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/<service>"
|
|
echo "Image tag: ${IMAGE_TAG}"
|
|
echo "Image list: ${IMAGE_LIST}"
|
|
|
|
missing=0
|
|
for svc in ${IMAGE_LIST}; do
|
|
if [[ "${MODE}" == "verify" ]]; then
|
|
verify_image "${svc}" || missing=1
|
|
else
|
|
ensure_image "${svc}"
|
|
fi
|
|
done
|
|
|
|
if [[ "${missing}" -ne 0 ]]; then
|
|
echo "One or more current images are missing. Run the matching CI workflow for this commit first." >&2
|
|
exit 1
|
|
fi
|