#!/usr/bin/env bash # Deploy already-built images to a remote k3s host. # # The script can either use local Docker images or pull images from the Gitea # registry, retag them as geo-rankly/:, copy a Docker archive to # the production host, import it into k3s containerd, and roll deployments. # # Example: # IMAGE_TAG=20260531-abcdef12 \ # PULL_FROM_REGISTRY=true REGISTRY_PASSWORD=... \ # TARGET_PASSWORD=... \ # bash deploy/scripts/deploy-images-to-k3s.sh ops-api ops-web frontend set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" TARGET_HOST="${TARGET_HOST:-root@39.105.229.239}" TARGET_PORT="${TARGET_PORT:-22}" TARGET_PASSWORD="${TARGET_PASSWORD:-}" NAMESPACE="${NAMESPACE:-geo-rankly}" LOCAL_PREFIX="${LOCAL_PREFIX:-geo-rankly}" IMAGE_TAG="${IMAGE_TAG:-}" REMOTE_DIR="${REMOTE_DIR:-/opt/geo-rankly/releases}" WAIT_TIMEOUT="${WAIT_TIMEOUT:-10m}" DELETE_REMOTE_TAR="${DELETE_REMOTE_TAR:-true}" PULL_FROM_REGISTRY="${PULL_FROM_REGISTRY:-false}" REGISTRY_HOST="${REGISTRY_HOST:-192.168.100.19:13000}" REGISTRY_HOST="${REGISTRY_HOST#http://}" REGISTRY_HOST="${REGISTRY_HOST#https://}" REGISTRY_HOST="${REGISTRY_HOST%/}" REGISTRY_OWNER="${REGISTRY_OWNER:-root}" REGISTRY_IMAGE_PREFIX="${REGISTRY_IMAGE_PREFIX:-geo-rankly}" REGISTRY_USERNAME="${REGISTRY_USERNAME:-root}" REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}" REGISTRY_PLAIN_HTTP="${REGISTRY_PLAIN_HTTP:-true}" RUN_MIGRATIONS="${RUN_MIGRATIONS:-false}" ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT="${ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT:-false}" SKIP_DEPLOYMENTS="${SKIP_DEPLOYMENTS:-false}" TARGET_PLATFORM="${TARGET_PLATFORM:-}" CLEAN_LOCAL_IMAGES="${CLEAN_LOCAL_IMAGES:-false}" DEFAULT_SERVICES=( tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web ) DEPLOYABLE_SERVICES=( tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web ) MIGRATION_GATED_SERVICES=( tenant-api ops-api worker-generate kol-assist-worker scheduler ) if [[ -z "${IMAGE_TAG}" ]]; then echo "IMAGE_TAG is required. Use a unique tag, for example: IMAGE_TAG=$(git -C "${REPO_ROOT}" rev-parse --short=8 HEAD)-$(date +%Y%m%d%H%M%S)" >&2 exit 1 fi if [[ "$#" -gt 0 ]]; then SERVICES=("$@") elif [[ "${SKIP_DEPLOYMENTS}" == "true" ]]; then SERVICES=() else SERVICES=("${DEFAULT_SERVICES[@]}") fi case "${IMAGE_TAG}" in *[!A-Za-z0-9_.-]*|"") echo "IMAGE_TAG may only contain letters, numbers, underscore, dot, and dash." >&2 exit 1 ;; esac case "${TARGET_PORT}" in *[!0-9]*|"") echo "TARGET_PORT must be numeric." >&2 exit 1 ;; esac case "${TARGET_HOST}" in *[!A-Za-z0-9_.@:-]*|"") echo "TARGET_HOST must be user@host or host without spaces/shell characters." >&2 exit 1 ;; esac case "${REMOTE_DIR}" in /*) ;; *) echo "REMOTE_DIR must be an absolute path." >&2 exit 1 ;; esac case "${REMOTE_DIR}" in *[!A-Za-z0-9_./-]*) echo "REMOTE_DIR contains unsupported characters." >&2 exit 1 ;; esac case "${NAMESPACE}" in *[!A-Za-z0-9_.-]*|"") echo "NAMESPACE contains unsupported characters." >&2 exit 1 ;; esac case "${LOCAL_PREFIX}" in *[!A-Za-z0-9_./-]*|"") echo "LOCAL_PREFIX contains unsupported characters." >&2 exit 1 ;; esac case "${PULL_FROM_REGISTRY}" in true|false) ;; *) echo "PULL_FROM_REGISTRY must be true or false." >&2 exit 1 ;; esac case "${RUN_MIGRATIONS}" in true|false) ;; *) echo "RUN_MIGRATIONS must be true or false." >&2 exit 1 ;; esac case "${ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT}" in true|false) ;; *) echo "ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT must be true or false." >&2 exit 1 ;; esac case "${SKIP_DEPLOYMENTS}" in true|false) ;; *) echo "SKIP_DEPLOYMENTS must be true or false." >&2 exit 1 ;; esac case "${DELETE_REMOTE_TAR}" in true|false) ;; *) echo "DELETE_REMOTE_TAR must be true or false." >&2 exit 1 ;; esac case "${WAIT_TIMEOUT}" in *[!A-Za-z0-9_.-]*|"") echo "WAIT_TIMEOUT contains unsupported characters." >&2 exit 1 ;; esac if [[ "${#SERVICES[@]}" -gt 0 ]]; then for svc in "${SERVICES[@]}"; do supported=false for known in "${DEPLOYABLE_SERVICES[@]}"; do if [[ "${svc}" == "${known}" ]]; then supported=true break fi done if [[ "${supported}" != "true" ]]; then echo "Unsupported service ${svc}. Pass deployment/container names only." >&2 exit 1 fi done fi if [[ "${RUN_MIGRATIONS}" != "true" && "${ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT}" != "true" ]]; then for svc in "${SERVICES[@]}"; do for gated in "${MIGRATION_GATED_SERVICES[@]}"; do if [[ "${svc}" == "${gated}" ]]; then echo "Refusing to roll ${svc} without RUN_MIGRATIONS=true." >&2 echo "Use ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT=true only for a reviewed no-schema-change rollout." >&2 exit 1 fi done done fi IMAGE_SERVICES=() if [[ "${#SERVICES[@]}" -gt 0 ]]; then IMAGE_SERVICES=("${SERVICES[@]}") fi if [[ "${RUN_MIGRATIONS}" == "true" ]]; then if [[ "${#IMAGE_SERVICES[@]}" -gt 0 ]]; then IMAGE_SERVICES=(migrate "${IMAGE_SERVICES[@]}") else IMAGE_SERVICES=(migrate) fi fi if [[ "${#IMAGE_SERVICES[@]}" -eq 0 ]]; then echo "No images selected. Pass services or set RUN_MIGRATIONS=true." >&2 exit 1 fi if ! command -v docker >/dev/null 2>&1; then echo "docker is required on the NAS/build/Gitea runner host." >&2 exit 1 fi if [[ -n "${TARGET_PASSWORD}" ]] && ! command -v sshpass >/dev/null 2>&1; then echo "sshpass is required when TARGET_PASSWORD is set." >&2 exit 1 fi registry_image() { local svc="$1" printf '%s/%s/%s/%s:%s' \ "${REGISTRY_HOST}" \ "${REGISTRY_OWNER}" \ "${REGISTRY_IMAGE_PREFIX}" \ "${svc}" \ "${IMAGE_TAG}" } local_image() { local svc="$1" printf '%s/%s:%s' "${LOCAL_PREFIX}" "${svc}" "${IMAGE_TAG}" } if [[ "${PULL_FROM_REGISTRY}" == "true" ]]; then for value_name in REGISTRY_HOST REGISTRY_OWNER REGISTRY_IMAGE_PREFIX REGISTRY_USERNAME; do value="${!value_name}" if [[ -z "${value}" ]]; then echo "${value_name} is required when PULL_FROM_REGISTRY=true." >&2 exit 1 fi done if [[ -z "${REGISTRY_PASSWORD}" ]]; then echo "REGISTRY_PASSWORD is required when PULL_FROM_REGISTRY=true." >&2 exit 1 fi case "${REGISTRY_PLAIN_HTTP}" in true|false) ;; *) echo "REGISTRY_PLAIN_HTTP must be true or false." >&2 exit 1 ;; esac if [[ "${REGISTRY_PLAIN_HTTP}" == "true" ]] \ && ! docker info 2>/dev/null \ | sed -n '/Insecure Registries:/,/Live Restore/p' \ | grep -Fq "${REGISTRY_HOST}"; then echo "${REGISTRY_HOST} is not listed in Docker insecure registries." >&2 echo "Configure the Gitea runner Docker daemon with: {\"insecure-registries\":[\"${REGISTRY_HOST}\"]}" >&2 exit 1 fi printf '%s\n' "${REGISTRY_PASSWORD}" \ | docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin for svc in "${IMAGE_SERVICES[@]}"; do src="$(registry_image "${svc}")" dst="$(local_image "${svc}")" echo "==> Pulling ${src}" if [[ -n "${TARGET_PLATFORM}" ]]; then docker pull --platform "${TARGET_PLATFORM}" "${src}" else docker pull "${src}" fi docker tag "${src}" "${dst}" done fi images=() for svc in "${IMAGE_SERVICES[@]}"; do image="$(local_image "${svc}")" if ! docker image inspect "${image}" >/dev/null 2>&1; then echo "Missing local image: ${image}" >&2 echo "Build it first, or run with PULL_FROM_REGISTRY=true and a tag that exists in Gitea." >&2 exit 1 fi images+=("${image}") done safe_tag="$(printf '%s' "${IMAGE_TAG}" | tr -c 'A-Za-z0-9_.-' '-')" run_suffix="$(printf '%s-%s-%s' "${GITHUB_RUN_ID:-manual}" "${GITHUB_RUN_ATTEMPT:-0}" "$$" | tr -c 'A-Za-z0-9_.-' '-')" archive="/tmp/geo-rankly-images-${safe_tag}-${run_suffix}.tar" remote_archive="${REMOTE_DIR}/geo-rankly-images-${safe_tag}-${run_suffix}.tar" remote_lock_dir="${REMOTE_DIR}/.k3s-image-rollout.lock" service_list="" if [[ "${#SERVICES[@]}" -gt 0 ]]; then service_list="${SERVICES[*]}" fi echo "==> Saving images:" printf ' %s\n' "${images[@]}" docker save "${images[@]}" -o "${archive}" echo " archive: ${archive} ($(du -sh "${archive}" | cut -f1))" archive_bytes="$(wc -c < "${archive}" | tr -d '[:space:]')" ssh_cmd=(ssh -p "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new) scp_cmd=(scp -P "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new) remote_ssh() { if [[ -n "${TARGET_PASSWORD}" ]]; then SSHPASS="${TARGET_PASSWORD}" sshpass -e "${ssh_cmd[@]}" "${TARGET_HOST}" "$@" else "${ssh_cmd[@]}" "${TARGET_HOST}" "$@" fi } remote_scp() { local source="$1" local target="$2" if [[ -n "${TARGET_PASSWORD}" ]]; then SSHPASS="${TARGET_PASSWORD}" sshpass -e "${scp_cmd[@]}" "${source}" "${TARGET_HOST}:${target}" else "${scp_cmd[@]}" "${source}" "${TARGET_HOST}:${target}" fi } echo "==> Uploading archive to ${TARGET_HOST}:${remote_archive}" remote_ssh "mkdir -p '${REMOTE_DIR}'" remote_scp "${archive}" "${remote_archive}" remote_ssh \ "REMOTE_ARCHIVE='${remote_archive}' EXPECTED_BYTES='${archive_bytes}' bash -se" <<'REMOTE_VERIFY' set -euo pipefail if [[ ! -f "${REMOTE_ARCHIVE}" ]]; then echo "Uploaded archive is missing on target: ${REMOTE_ARCHIVE}" >&2 ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true exit 1 fi actual_bytes="$(wc -c < "${REMOTE_ARCHIVE}" | tr -d '[:space:]')" if [[ "${actual_bytes}" != "${EXPECTED_BYTES}" ]]; then echo "Uploaded archive size mismatch: expected ${EXPECTED_BYTES}, got ${actual_bytes}" >&2 exit 1 fi echo " uploaded ${REMOTE_ARCHIVE} (${actual_bytes} bytes)" REMOTE_VERIFY echo "==> Importing images and rolling deployments on ${TARGET_HOST}" remote_ssh \ "NAMESPACE='${NAMESPACE}' LOCAL_PREFIX='${LOCAL_PREFIX}' IMAGE_TAG='${IMAGE_TAG}' SERVICES='${service_list}' REMOTE_ARCHIVE='${remote_archive}' LOCK_DIR='${remote_lock_dir}' WAIT_TIMEOUT='${WAIT_TIMEOUT}' DELETE_REMOTE_TAR='${DELETE_REMOTE_TAR}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' bash -s" <<'REMOTE' set -euo pipefail acquire_rollout_lock() { if mkdir "${LOCK_DIR}" 2>/dev/null; then printf '%s\n' "$$" > "${LOCK_DIR}/pid" printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag" date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true return 0 fi existing_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)" existing_tag="$(cat "${LOCK_DIR}/image_tag" 2>/dev/null || true)" existing_created_at="$(cat "${LOCK_DIR}/created_at" 2>/dev/null || true)" if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2 echo " pid: ${existing_pid}" >&2 echo " image tag: ${existing_tag:-unknown}" >&2 echo " created: ${existing_created_at:-unknown}" >&2 exit 1 fi echo "Removing stale k3s image rollout lock: ${LOCK_DIR}" rm -rf "${LOCK_DIR}" if ! mkdir "${LOCK_DIR}" 2>/dev/null; then echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2 exit 1 fi printf '%s\n' "$$" > "${LOCK_DIR}/pid" printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag" date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true } release_rollout_lock() { owner_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)" if [[ "${owner_pid}" == "$$" ]]; then rm -rf "${LOCK_DIR}" fi } acquire_rollout_lock trap release_rollout_lock EXIT INT TERM if ! command -v k3s >/dev/null 2>&1; then echo "k3s is required on the target host." >&2 exit 1 fi kubectl_cmd() { k3s kubectl "$@" } echo " importing ${REMOTE_ARCHIVE}" if [[ ! -s "${REMOTE_ARCHIVE}" ]]; then echo "Remote archive is missing or empty before import: ${REMOTE_ARCHIVE}" >&2 ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true exit 1 fi k3s ctr -n k8s.io images import "${REMOTE_ARCHIVE}" if [[ "${RUN_MIGRATIONS}" == "true" ]]; then migrate_image="${LOCAL_PREFIX}/migrate:${IMAGE_TAG}" echo " running migration job with ${migrate_image}" kubectl_cmd -n "${NAMESPACE}" delete job migrate --ignore-not-found cat < ${image}" kubectl_cmd -n "${NAMESPACE}" set image "deployment/${svc}" "${svc}=${image}" kubectl_cmd -n "${NAMESPACE}" rollout restart "deployment/${svc}" done for svc in ${SERVICES}; do echo " waiting deployment/${svc}" kubectl_cmd -n "${NAMESPACE}" rollout status "deployment/${svc}" --timeout="${WAIT_TIMEOUT}" done if [[ -n "${SERVICES}" ]]; then kubectl_cmd -n "${NAMESPACE}" get deploy ${SERVICES} fi if [[ "${DELETE_REMOTE_TAR}" == "true" ]]; then rm -f "${REMOTE_ARCHIVE}" fi REMOTE rm -f "${archive}" if [[ "${CLEAN_LOCAL_IMAGES}" == "true" ]]; then docker image rm "${images[@]}" >/dev/null 2>&1 || true fi echo "" echo "k3s image rollout complete: ${IMAGE_TAG}"