568 lines
14 KiB
Bash
568 lines
14 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
MODE="${1:-ensure}"
|
|
|
|
REGISTRY_HOST="${REGISTRY_HOST:-192.168.100.19:13000}"
|
|
REGISTRY_HOST="${REGISTRY_HOST#http://}"
|
|
REGISTRY_HOST="${REGISTRY_HOST#https://}"
|
|
REGISTRY_HOST="${REGISTRY_HOST%/}"
|
|
REGISTRY_OWNER="${REGISTRY_OWNER:-root}"
|
|
REGISTRY_IMAGE_PREFIX="${REGISTRY_IMAGE_PREFIX:-geo-rankly}"
|
|
REGISTRY_USERNAME="${REGISTRY_USERNAME:-root}"
|
|
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
|
|
REGISTRY_PLAIN_HTTP="${REGISTRY_PLAIN_HTTP:-true}"
|
|
REGISTRY_SCHEME="${REGISTRY_SCHEME:-}"
|
|
IMAGE_TAG="${IMAGE_TAG:-}"
|
|
IMAGE_LIST="${IMAGE_LIST:-tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web migrate}"
|
|
DELETE_OLD_IMAGE_TAGS="${DELETE_OLD_IMAGE_TAGS:-true}"
|
|
KEEP_IMAGE_TAGS="${KEEP_IMAGE_TAGS:-2}"
|
|
GITEA_PACKAGE_PAGE_LIMIT="${GITEA_PACKAGE_PAGE_LIMIT:-100}"
|
|
PUSH_LATEST="${PUSH_LATEST:-true}"
|
|
FORCE_BUILD="${FORCE_BUILD:-false}"
|
|
TARGET_PLATFORM="${TARGET_PLATFORM:-}"
|
|
|
|
case "${MODE}" in
|
|
ensure|verify|cleanup) ;;
|
|
*)
|
|
echo "Usage: $0 [ensure|verify|cleanup]" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if [[ -z "${IMAGE_TAG}" ]]; then
|
|
IMAGE_TAG="$(git -C "${REPO_ROOT}" rev-parse --short=8 HEAD)"
|
|
fi
|
|
|
|
if [[ -z "${REGISTRY_SCHEME}" ]]; then
|
|
if [[ "${REGISTRY_PLAIN_HTTP}" == "true" ]]; then
|
|
REGISTRY_SCHEME="http"
|
|
else
|
|
REGISTRY_SCHEME="https"
|
|
fi
|
|
fi
|
|
|
|
case "${IMAGE_TAG}" in
|
|
*[!A-Za-z0-9_.-]*|"")
|
|
echo "IMAGE_TAG may only contain letters, numbers, underscore, dot, and dash." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
for value_name in REGISTRY_HOST REGISTRY_OWNER REGISTRY_IMAGE_PREFIX REGISTRY_USERNAME; do
|
|
value="${!value_name}"
|
|
if [[ -z "${value}" ]]; then
|
|
echo "${value_name} is required." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if [[ -z "${REGISTRY_PASSWORD}" ]]; then
|
|
echo "REGISTRY_PASSWORD is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
case "${REGISTRY_PLAIN_HTTP}" in
|
|
true|false) ;;
|
|
*)
|
|
echo "REGISTRY_PLAIN_HTTP must be true or false." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
case "${KEEP_IMAGE_TAGS}" in
|
|
''|*[!0-9]*)
|
|
echo "KEEP_IMAGE_TAGS must be a non-negative integer." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
case "${GITEA_PACKAGE_PAGE_LIMIT}" in
|
|
''|*[!0-9]*)
|
|
echo "GITEA_PACKAGE_PAGE_LIMIT must be a positive integer." >&2
|
|
exit 1
|
|
;;
|
|
0)
|
|
echo "GITEA_PACKAGE_PAGE_LIMIT must be greater than 0." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
echo "curl is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v python3 >/dev/null 2>&1; then
|
|
echo "python3 is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${MODE}" == "ensure" ]]; then
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "docker is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${REGISTRY_PLAIN_HTTP}" == "true" ]] \
|
|
&& ! docker info 2>/dev/null \
|
|
| sed -n '/Insecure Registries:/,/Live Restore/p' \
|
|
| grep -Fq "${REGISTRY_HOST}"; then
|
|
echo "${REGISTRY_HOST} is not listed in Docker insecure registries." >&2
|
|
echo "Configure the runner Docker daemon with: {\"insecure-registries\":[\"${REGISTRY_HOST}\"]}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s\n' "${REGISTRY_PASSWORD}" \
|
|
| docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin
|
|
fi
|
|
|
|
image_name() {
|
|
local svc="$1"
|
|
printf '%s/%s/%s/%s' \
|
|
"${REGISTRY_HOST}" \
|
|
"${REGISTRY_OWNER}" \
|
|
"${REGISTRY_IMAGE_PREFIX}" \
|
|
"${svc}"
|
|
}
|
|
|
|
registry_repo_path() {
|
|
local svc="$1"
|
|
printf '%s/%s/%s' "${REGISTRY_OWNER}" "${REGISTRY_IMAGE_PREFIX}" "${svc}"
|
|
}
|
|
|
|
registry_api_base() {
|
|
local svc="$1"
|
|
printf '%s://%s/v2/%s' \
|
|
"${REGISTRY_SCHEME}" \
|
|
"${REGISTRY_HOST}" \
|
|
"$(registry_repo_path "${svc}")"
|
|
}
|
|
|
|
gitea_api_base() {
|
|
printf '%s://%s/api/v1' "${REGISTRY_SCHEME}" "${REGISTRY_HOST}"
|
|
}
|
|
|
|
gitea_package_base_url() {
|
|
local svc="$1" encoded_owner encoded_name
|
|
encoded_owner="$(url_encode_segment "${REGISTRY_OWNER}")"
|
|
encoded_name="$(url_encode_segment "$(package_name "${svc}")")"
|
|
printf '%s/packages/%s/container/%s' \
|
|
"$(gitea_api_base)" \
|
|
"${encoded_owner}" \
|
|
"${encoded_name}"
|
|
}
|
|
|
|
gitea_package_list_url() {
|
|
local svc="$1" page="$2" encoded_owner encoded_query
|
|
encoded_owner="$(url_encode_segment "${REGISTRY_OWNER}")"
|
|
encoded_query="$(url_encode_segment "$(package_name "${svc}")")"
|
|
printf '%s/packages/%s/?type=container&q=%s&page=%s&limit=%s' \
|
|
"$(gitea_api_base)" \
|
|
"${encoded_owner}" \
|
|
"${encoded_query}" \
|
|
"${page}" \
|
|
"${GITEA_PACKAGE_PAGE_LIMIT}"
|
|
}
|
|
|
|
package_name() {
|
|
local svc="$1"
|
|
printf '%s/%s' "${REGISTRY_IMAGE_PREFIX}" "${svc}"
|
|
}
|
|
|
|
url_encode_segment() {
|
|
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"
|
|
}
|
|
|
|
curl_registry() {
|
|
curl -fsS -u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" "$@"
|
|
}
|
|
|
|
curl_gitea_api_json() {
|
|
local url="$1" body_file status body
|
|
body_file="$(mktemp)"
|
|
status="$(
|
|
curl -sS \
|
|
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
|
-o "${body_file}" \
|
|
-w '%{http_code}' \
|
|
--path-as-is \
|
|
"${url}" || true
|
|
)"
|
|
body="$(<"${body_file}")"
|
|
rm -f "${body_file}"
|
|
|
|
case "${status}" in
|
|
200)
|
|
printf '%s' "${body}"
|
|
;;
|
|
401|403)
|
|
echo "Gitea package API authentication failed." >&2
|
|
exit 1
|
|
;;
|
|
404)
|
|
return 1
|
|
;;
|
|
*)
|
|
echo "Gitea package API request failed; HTTP ${status}. URL: ${url}" >&2
|
|
[[ -z "${body}" ]] || echo "${body}" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
manifest_digest() {
|
|
local svc="$1" tag="$2" base headers status digest
|
|
base="$(registry_api_base "${svc}")"
|
|
headers="$(mktemp)"
|
|
status="$(
|
|
curl -sS \
|
|
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
|
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
|
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
|
|
-D "${headers}" \
|
|
-o /dev/null \
|
|
-w '%{http_code}' \
|
|
"${base}/manifests/${tag}" || true
|
|
)"
|
|
|
|
case "${status}" in
|
|
200)
|
|
digest="$(awk 'BEGIN{IGNORECASE=1} /^Docker-Content-Digest:/ {gsub("\r", "", $2); print $2; exit}' "${headers}")"
|
|
rm -f "${headers}"
|
|
printf '%s' "${digest}"
|
|
;;
|
|
404)
|
|
rm -f "${headers}"
|
|
return 1
|
|
;;
|
|
401|403)
|
|
rm -f "${headers}"
|
|
echo "Registry authentication failed while checking ${svc}:${tag}." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
rm -f "${headers}"
|
|
echo "Failed to check ${svc}:${tag}; registry returned HTTP ${status}." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
tag_exists() {
|
|
local svc="$1" tag="$2"
|
|
manifest_digest "${svc}" "${tag}" >/dev/null
|
|
}
|
|
|
|
delete_manifest_reference() {
|
|
local svc="$1" reference="$2" base status
|
|
base="$(registry_api_base "${svc}")"
|
|
status="$(
|
|
curl -sS \
|
|
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
|
-X DELETE \
|
|
-o /dev/null \
|
|
-w '%{http_code}' \
|
|
"${base}/manifests/${reference}" || true
|
|
)"
|
|
|
|
case "${status}" in
|
|
200|202|404)
|
|
return 0
|
|
;;
|
|
405)
|
|
echo "Registry does not allow deleting ${svc}:${reference}; cleanup failed." >&2
|
|
return 1
|
|
;;
|
|
*)
|
|
echo "Failed to delete old image tag ${svc}:${reference}; HTTP ${status}." >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
delete_gitea_package_version() {
|
|
local svc="$1" tag="$2" encoded_tag url status
|
|
encoded_tag="$(url_encode_segment "${tag}")"
|
|
url="$(gitea_package_base_url "${svc}")/${encoded_tag}"
|
|
|
|
status="$(
|
|
curl -sS \
|
|
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
|
-X DELETE \
|
|
-o /dev/null \
|
|
-w '%{http_code}' \
|
|
--path-as-is \
|
|
"${url}" || true
|
|
)"
|
|
|
|
case "${status}" in
|
|
200|202|204)
|
|
return 0
|
|
;;
|
|
404)
|
|
return 1
|
|
;;
|
|
401|403)
|
|
echo "Gitea package API authentication failed while deleting ${svc}:${tag}." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "Gitea package API could not delete ${svc}:${tag}; HTTP ${status}." >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
delete_old_hash_tag() {
|
|
local svc="$1" tag="$2"
|
|
|
|
if delete_gitea_package_version "${svc}" "${tag}"; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Falling back to registry tag delete for ${svc}:${tag}"
|
|
delete_manifest_reference "${svc}" "${tag}"
|
|
}
|
|
|
|
list_gitea_hash_versions() {
|
|
local svc="$1" page=1 json count expected_name
|
|
expected_name="$(package_name "${svc}")"
|
|
|
|
while true; do
|
|
json="$(
|
|
curl_gitea_api_json "$(gitea_package_list_url "${svc}" "${page}")"
|
|
)" || return 1
|
|
|
|
count="$(
|
|
printf '%s' "${json}" \
|
|
| python3 -c 'import json, sys; data = json.load(sys.stdin); print(len(data) if isinstance(data, list) else 0)'
|
|
)"
|
|
|
|
printf '%s' "${json}" \
|
|
| python3 -c '
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
expected_name = sys.argv[1].lower()
|
|
data = json.load(sys.stdin)
|
|
if not isinstance(data, list):
|
|
raise SystemExit("Gitea package API response was not a list")
|
|
|
|
for item in data:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
if item.get("type") not in ("", "container"):
|
|
continue
|
|
if (item.get("name") or "").lower() != expected_name:
|
|
continue
|
|
version = item.get("version") or ""
|
|
if not re.fullmatch(r"[0-9a-f]{8}", version):
|
|
continue
|
|
created_at = item.get("created_at") or ""
|
|
print(f"{created_at}\t{version}")
|
|
' "${expected_name}"
|
|
|
|
if (( count < GITEA_PACKAGE_PAGE_LIMIT )); then
|
|
break
|
|
fi
|
|
page=$((page + 1))
|
|
done | sort -r -k1,1 -k2,2
|
|
}
|
|
|
|
cleanup_old_hash_tags() {
|
|
local svc="$1" kept=0 hash_tags_to_keep line tag versions_file keep_file
|
|
|
|
if [[ "${DELETE_OLD_IMAGE_TAGS}" != "true" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
hash_tags_to_keep="${KEEP_IMAGE_TAGS}"
|
|
if [[ "${PUSH_LATEST}" == "true" && "${hash_tags_to_keep}" -gt 0 ]]; then
|
|
hash_tags_to_keep=$((hash_tags_to_keep - 1))
|
|
fi
|
|
|
|
versions_file="$(mktemp)"
|
|
keep_file="$(mktemp)"
|
|
if ! list_gitea_hash_versions "${svc}" >"${versions_file}"; then
|
|
rm -f "${versions_file}" "${keep_file}"
|
|
echo "Failed to list Gitea package versions for ${svc}; cleanup aborted." >&2
|
|
return 1
|
|
fi
|
|
|
|
if (( hash_tags_to_keep > 0 )); then
|
|
while IFS= read -r line; do
|
|
tag="${line#*$'\t'}"
|
|
if [[ "${tag}" == "${IMAGE_TAG}" ]]; then
|
|
printf '%s\n' "${tag}" >"${keep_file}"
|
|
break
|
|
fi
|
|
done <"${versions_file}"
|
|
|
|
while IFS= read -r line; do
|
|
tag="${line#*$'\t'}"
|
|
[[ -n "${tag}" ]] || continue
|
|
if grep -Fxq "${tag}" "${keep_file}"; then
|
|
continue
|
|
fi
|
|
kept="$(wc -l <"${keep_file}" | tr -d ' ')"
|
|
if (( kept >= hash_tags_to_keep )); then
|
|
break
|
|
fi
|
|
printf '%s\n' "${tag}" >>"${keep_file}"
|
|
done <"${versions_file}"
|
|
fi
|
|
|
|
kept=0
|
|
while IFS= read -r line; do
|
|
tag="${line#*$'\t'}"
|
|
[[ -n "${tag}" ]] || continue
|
|
|
|
if grep -Fxq "${tag}" "${keep_file}"; then
|
|
kept=$((kept + 1))
|
|
echo "Keeping image tag ${svc}:${tag} (${kept}/${hash_tags_to_keep} hash tags, total target ${KEEP_IMAGE_TAGS} including latest)"
|
|
continue
|
|
fi
|
|
|
|
echo "Deleting old image tag ${svc}:${tag}"
|
|
if ! delete_old_hash_tag "${svc}" "${tag}"; then
|
|
rm -f "${versions_file}" "${keep_file}"
|
|
return 1
|
|
fi
|
|
done <"${versions_file}"
|
|
rm -f "${versions_file}" "${keep_file}"
|
|
}
|
|
|
|
platform_args=()
|
|
if [[ -n "${TARGET_PLATFORM}" ]]; then
|
|
platform_args=(--platform "${TARGET_PLATFORM}")
|
|
fi
|
|
|
|
build_image() {
|
|
local svc="$1" repo
|
|
repo="$(image_name "${svc}")"
|
|
|
|
case "${svc}" in
|
|
migrate)
|
|
docker build "${platform_args[@]}" \
|
|
--target migrate \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/server/Dockerfile" \
|
|
"${REPO_ROOT}/server"
|
|
;;
|
|
browser-fetch)
|
|
docker build "${platform_args[@]}" \
|
|
--target browser-fetch \
|
|
--build-arg "SERVICE=browser-fetch" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/server/Dockerfile" \
|
|
"${REPO_ROOT}/server"
|
|
;;
|
|
tenant-api|worker-generate|kol-assist-worker|scheduler|ops-api)
|
|
docker build "${platform_args[@]}" \
|
|
--target runtime \
|
|
--build-arg "SERVICE=${svc}" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/server/Dockerfile" \
|
|
"${REPO_ROOT}/server"
|
|
;;
|
|
frontend)
|
|
docker build "${platform_args[@]}" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/Dockerfile.frontend" \
|
|
"${REPO_ROOT}"
|
|
;;
|
|
ops-web)
|
|
docker build "${platform_args[@]}" \
|
|
--tag "${repo}:${IMAGE_TAG}" \
|
|
--tag "${repo}:latest" \
|
|
-f "${REPO_ROOT}/Dockerfile.ops-web" \
|
|
"${REPO_ROOT}"
|
|
;;
|
|
*)
|
|
echo "Unsupported image service: ${svc}" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
push_latest_from_existing_tag() {
|
|
local svc="$1" repo tag_digest latest_digest
|
|
|
|
if [[ "${PUSH_LATEST}" != "true" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
tag_digest="$(manifest_digest "${svc}" "${IMAGE_TAG}" 2>/dev/null || true)"
|
|
latest_digest="$(manifest_digest "${svc}" "latest" 2>/dev/null || true)"
|
|
if [[ -n "${tag_digest}" && "${tag_digest}" == "${latest_digest}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
repo="$(image_name "${svc}")"
|
|
echo "Updating ${svc}:latest to existing ${svc}:${IMAGE_TAG}"
|
|
docker pull "${repo}:${IMAGE_TAG}"
|
|
docker tag "${repo}:${IMAGE_TAG}" "${repo}:latest"
|
|
docker push "${repo}:latest"
|
|
}
|
|
|
|
ensure_image() {
|
|
local svc="$1" repo
|
|
repo="$(image_name "${svc}")"
|
|
|
|
if [[ "${FORCE_BUILD}" != "true" ]] && tag_exists "${svc}" "${IMAGE_TAG}"; then
|
|
echo "Image already current: ${repo}:${IMAGE_TAG}; skipping build."
|
|
push_latest_from_existing_tag "${svc}"
|
|
cleanup_old_hash_tags "${svc}"
|
|
return 0
|
|
fi
|
|
|
|
echo "Building image: ${repo}:${IMAGE_TAG}"
|
|
build_image "${svc}"
|
|
docker push "${repo}:${IMAGE_TAG}"
|
|
if [[ "${PUSH_LATEST}" == "true" ]]; then
|
|
docker push "${repo}:latest"
|
|
fi
|
|
cleanup_old_hash_tags "${svc}"
|
|
}
|
|
|
|
verify_image() {
|
|
local svc="$1" repo
|
|
repo="$(image_name "${svc}")"
|
|
|
|
if tag_exists "${svc}" "${IMAGE_TAG}"; then
|
|
echo "Image is current: ${repo}:${IMAGE_TAG}"
|
|
else
|
|
echo "Missing current image: ${repo}:${IMAGE_TAG}" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
echo "Gitea Registry: ${REGISTRY_SCHEME}://${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/<service>"
|
|
echo "Image tag: ${IMAGE_TAG}"
|
|
echo "Image list: ${IMAGE_LIST}"
|
|
echo "Keeping image versions per service: ${KEEP_IMAGE_TAGS} (latest counts as one when PUSH_LATEST=true)"
|
|
|
|
missing=0
|
|
for svc in ${IMAGE_LIST}; do
|
|
if [[ "${MODE}" == "verify" ]]; then
|
|
verify_image "${svc}" || missing=1
|
|
elif [[ "${MODE}" == "cleanup" ]]; then
|
|
cleanup_old_hash_tags "${svc}"
|
|
else
|
|
ensure_image "${svc}"
|
|
fi
|
|
done
|
|
|
|
if [[ "${missing}" -ne 0 ]]; then
|
|
echo "One or more current images are missing. Run the matching CI workflow for this commit first." >&2
|
|
exit 1
|
|
fi
|