geo/deploy/scripts/deploy-images-to-k3s.sh

#!/usr/bin/env bash
# Deploy already-built images to a remote k3s host.
#
# The script can either use local Docker images or pull images from the Gitea
# registry, retag them as geo-rankly/<service>:<tag>, copy a Docker archive to
# the production host, import it into k3s containerd, and roll deployments.
#
# Example:
#   IMAGE_TAG=20260531-abcdef12 \
#     PULL_FROM_REGISTRY=true REGISTRY_PASSWORD=... \
#     TARGET_PASSWORD=... \
#     bash deploy/scripts/deploy-images-to-k3s.sh ops-api ops-web frontend

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TARGET_HOST="${TARGET_HOST:-root@39.105.229.239}"
TARGET_PORT="${TARGET_PORT:-22}"
TARGET_PASSWORD="${TARGET_PASSWORD:-}"
NAMESPACE="${NAMESPACE:-geo-rankly}"
LOCAL_PREFIX="${LOCAL_PREFIX:-geo-rankly}"
IMAGE_TAG="${IMAGE_TAG:-}"
REMOTE_DIR="${REMOTE_DIR:-/opt/geo-rankly/releases}"
WAIT_TIMEOUT="${WAIT_TIMEOUT:-10m}"
DELETE_REMOTE_TAR="${DELETE_REMOTE_TAR:-true}"
PULL_FROM_REGISTRY="${PULL_FROM_REGISTRY:-false}"
REGISTRY_HOST="${REGISTRY_HOST:-192.168.100.19:13000}"
REGISTRY_HOST="${REGISTRY_HOST#http://}"
REGISTRY_HOST="${REGISTRY_HOST#https://}"
REGISTRY_HOST="${REGISTRY_HOST%/}"
REGISTRY_OWNER="${REGISTRY_OWNER:-root}"
REGISTRY_IMAGE_PREFIX="${REGISTRY_IMAGE_PREFIX:-geo-rankly}"
REGISTRY_USERNAME="${REGISTRY_USERNAME:-root}"
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
REGISTRY_PLAIN_HTTP="${REGISTRY_PLAIN_HTTP:-true}"
RUN_MIGRATIONS="${RUN_MIGRATIONS:-false}"
ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT="${ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT:-false}"
SKIP_DEPLOYMENTS="${SKIP_DEPLOYMENTS:-false}"
TARGET_PLATFORM="${TARGET_PLATFORM:-}"
CLEAN_LOCAL_IMAGES="${CLEAN_LOCAL_IMAGES:-false}"

DEFAULT_SERVICES=(
  tenant-api
  browser-fetch
  ops-api
  worker-generate
  kol-assist-worker
  scheduler
  frontend
  ops-web
)

DEPLOYABLE_SERVICES=(
  tenant-api
  browser-fetch
  ops-api
  worker-generate
  kol-assist-worker
  scheduler
  frontend
  ops-web
)

MIGRATION_GATED_SERVICES=(
  tenant-api
  ops-api
  worker-generate
  kol-assist-worker
  scheduler
)

if [[ -z "${IMAGE_TAG}" ]]; then
  echo "IMAGE_TAG is required. Use a unique tag, for example: IMAGE_TAG=$(git -C "${REPO_ROOT}" rev-parse --short=8 HEAD)-$(date +%Y%m%d%H%M%S)" >&2
  exit 1
fi

if [[ "$#" -gt 0 ]]; then
  SERVICES=("$@")
elif [[ "${SKIP_DEPLOYMENTS}" == "true" ]]; then
  SERVICES=()
else
  SERVICES=("${DEFAULT_SERVICES[@]}")
fi

case "${IMAGE_TAG}" in
  *[!A-Za-z0-9_.-]*|"")
    echo "IMAGE_TAG may only contain letters, numbers, underscore, dot, and dash." >&2
    exit 1
    ;;
esac

case "${TARGET_PORT}" in
  *[!0-9]*|"")
    echo "TARGET_PORT must be numeric." >&2
    exit 1
    ;;
esac

case "${TARGET_HOST}" in
  *[!A-Za-z0-9_.@:-]*|"")
    echo "TARGET_HOST must be user@host or host without spaces/shell characters." >&2
    exit 1
    ;;
esac

case "${REMOTE_DIR}" in
  /*) ;;
  *)
    echo "REMOTE_DIR must be an absolute path." >&2
    exit 1
    ;;
esac

case "${REMOTE_DIR}" in
  *[!A-Za-z0-9_./-]*)
    echo "REMOTE_DIR contains unsupported characters." >&2
    exit 1
    ;;
esac

case "${NAMESPACE}" in
  *[!A-Za-z0-9_.-]*|"")
    echo "NAMESPACE contains unsupported characters." >&2
    exit 1
    ;;
esac

case "${LOCAL_PREFIX}" in
  *[!A-Za-z0-9_./-]*|"")
    echo "LOCAL_PREFIX contains unsupported characters." >&2
    exit 1
    ;;
esac

case "${PULL_FROM_REGISTRY}" in
  true|false) ;;
  *)
    echo "PULL_FROM_REGISTRY must be true or false." >&2
    exit 1
    ;;
esac

case "${RUN_MIGRATIONS}" in
  true|false) ;;
  *)
    echo "RUN_MIGRATIONS must be true or false." >&2
    exit 1
    ;;
esac

case "${ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT}" in
  true|false) ;;
  *)
    echo "ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT must be true or false." >&2
    exit 1
    ;;
esac

case "${SKIP_DEPLOYMENTS}" in
  true|false) ;;
  *)
    echo "SKIP_DEPLOYMENTS must be true or false." >&2
    exit 1
    ;;
esac

case "${DELETE_REMOTE_TAR}" in
  true|false) ;;
  *)
    echo "DELETE_REMOTE_TAR must be true or false." >&2
    exit 1
    ;;
esac

case "${WAIT_TIMEOUT}" in
  *[!A-Za-z0-9_.-]*|"")
    echo "WAIT_TIMEOUT contains unsupported characters." >&2
    exit 1
    ;;
esac

if [[ "${#SERVICES[@]}" -gt 0 ]]; then
  for svc in "${SERVICES[@]}"; do
    supported=false
    for known in "${DEPLOYABLE_SERVICES[@]}"; do
      if [[ "${svc}" == "${known}" ]]; then
        supported=true
        break
      fi
    done
    if [[ "${supported}" != "true" ]]; then
      echo "Unsupported service ${svc}. Pass deployment/container names only." >&2
      exit 1
    fi
  done
fi

if [[ "${RUN_MIGRATIONS}" != "true" && "${ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT}" != "true" ]]; then
  for svc in "${SERVICES[@]}"; do
    for gated in "${MIGRATION_GATED_SERVICES[@]}"; do
      if [[ "${svc}" == "${gated}" ]]; then
        echo "Refusing to roll ${svc} without RUN_MIGRATIONS=true." >&2
        echo "Use ALLOW_SKIP_MIGRATIONS_FOR_APP_ROLLOUT=true only for a reviewed no-schema-change rollout." >&2
        exit 1
      fi
    done
  done
fi

IMAGE_SERVICES=()
if [[ "${#SERVICES[@]}" -gt 0 ]]; then
  IMAGE_SERVICES=("${SERVICES[@]}")
fi
if [[ "${RUN_MIGRATIONS}" == "true" ]]; then
  if [[ "${#IMAGE_SERVICES[@]}" -gt 0 ]]; then
    IMAGE_SERVICES=(migrate "${IMAGE_SERVICES[@]}")
  else
    IMAGE_SERVICES=(migrate)
  fi
fi

if [[ "${#IMAGE_SERVICES[@]}" -eq 0 ]]; then
  echo "No images selected. Pass services or set RUN_MIGRATIONS=true." >&2
  exit 1
fi

if ! command -v docker >/dev/null 2>&1; then
  echo "docker is required on the NAS/build/Gitea runner host." >&2
  exit 1
fi

if [[ -n "${TARGET_PASSWORD}" ]] && ! command -v sshpass >/dev/null 2>&1; then
  echo "sshpass is required when TARGET_PASSWORD is set." >&2
  exit 1
fi

registry_image() {
  local svc="$1"
  printf '%s/%s/%s/%s:%s' \
    "${REGISTRY_HOST}" \
    "${REGISTRY_OWNER}" \
    "${REGISTRY_IMAGE_PREFIX}" \
    "${svc}" \
    "${IMAGE_TAG}"
}

local_image() {
  local svc="$1"
  printf '%s/%s:%s' "${LOCAL_PREFIX}" "${svc}" "${IMAGE_TAG}"
}

if [[ "${PULL_FROM_REGISTRY}" == "true" ]]; then
  for value_name in REGISTRY_HOST REGISTRY_OWNER REGISTRY_IMAGE_PREFIX REGISTRY_USERNAME; do
    value="${!value_name}"
    if [[ -z "${value}" ]]; then
      echo "${value_name} is required when PULL_FROM_REGISTRY=true." >&2
      exit 1
    fi
  done

  if [[ -z "${REGISTRY_PASSWORD}" ]]; then
    echo "REGISTRY_PASSWORD is required when PULL_FROM_REGISTRY=true." >&2
    exit 1
  fi

  case "${REGISTRY_PLAIN_HTTP}" in
    true|false) ;;
    *)
      echo "REGISTRY_PLAIN_HTTP must be true or false." >&2
      exit 1
      ;;
  esac

  if [[ "${REGISTRY_PLAIN_HTTP}" == "true" ]] \
    && ! docker info 2>/dev/null \
      | sed -n '/Insecure Registries:/,/Live Restore/p' \
      | grep -Fq "${REGISTRY_HOST}"; then
    echo "${REGISTRY_HOST} is not listed in Docker insecure registries." >&2
    echo "Configure the Gitea runner Docker daemon with: {\"insecure-registries\":[\"${REGISTRY_HOST}\"]}" >&2
    exit 1
  fi

  printf '%s\n' "${REGISTRY_PASSWORD}" \
    | docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin

  for svc in "${IMAGE_SERVICES[@]}"; do
    src="$(registry_image "${svc}")"
    dst="$(local_image "${svc}")"
    echo "==> Pulling ${src}"
    if [[ -n "${TARGET_PLATFORM}" ]]; then
      docker pull --platform "${TARGET_PLATFORM}" "${src}"
    else
      docker pull "${src}"
    fi
    docker tag "${src}" "${dst}"
  done
fi

images=()
for svc in "${IMAGE_SERVICES[@]}"; do
  image="$(local_image "${svc}")"
  if ! docker image inspect "${image}" >/dev/null 2>&1; then
    echo "Missing local image: ${image}" >&2
    echo "Build it first, or run with PULL_FROM_REGISTRY=true and a tag that exists in Gitea." >&2
    exit 1
  fi
  images+=("${image}")
done

safe_tag="$(printf '%s' "${IMAGE_TAG}" | tr -c 'A-Za-z0-9_.-' '-')"
run_suffix="$(printf '%s-%s-%s' "${GITHUB_RUN_ID:-manual}" "${GITHUB_RUN_ATTEMPT:-0}" "$$" | tr -c 'A-Za-z0-9_.-' '-')"
archive="/tmp/geo-rankly-images-${safe_tag}-${run_suffix}.tar"
remote_archive="${REMOTE_DIR}/geo-rankly-images-${safe_tag}-${run_suffix}.tar"
remote_lock_dir="${REMOTE_DIR}/.k3s-image-rollout.lock"
service_list=""
if [[ "${#SERVICES[@]}" -gt 0 ]]; then
  service_list="${SERVICES[*]}"
fi

echo "==> Saving images:"
printf '    %s\n' "${images[@]}"
docker save "${images[@]}" -o "${archive}"
echo "    archive: ${archive} ($(du -sh "${archive}" | cut -f1))"
archive_bytes="$(wc -c < "${archive}" | tr -d '[:space:]')"

ssh_cmd=(ssh -p "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new)
scp_cmd=(scp -P "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new)

remote_ssh() {
  if [[ -n "${TARGET_PASSWORD}" ]]; then
    SSHPASS="${TARGET_PASSWORD}" sshpass -e "${ssh_cmd[@]}" "${TARGET_HOST}" "$@"
  else
    "${ssh_cmd[@]}" "${TARGET_HOST}" "$@"
  fi
}

remote_scp() {
  local source="$1"
  local target="$2"
  if [[ -n "${TARGET_PASSWORD}" ]]; then
    SSHPASS="${TARGET_PASSWORD}" sshpass -e "${scp_cmd[@]}" "${source}" "${TARGET_HOST}:${target}"
  else
    "${scp_cmd[@]}" "${source}" "${TARGET_HOST}:${target}"
  fi
}

echo "==> Uploading archive to ${TARGET_HOST}:${remote_archive}"
remote_ssh "mkdir -p '${REMOTE_DIR}'"
remote_scp "${archive}" "${remote_archive}"
remote_ssh \
  "REMOTE_ARCHIVE='${remote_archive}' EXPECTED_BYTES='${archive_bytes}' bash -se" <<'REMOTE_VERIFY'
set -euo pipefail

if [[ ! -f "${REMOTE_ARCHIVE}" ]]; then
  echo "Uploaded archive is missing on target: ${REMOTE_ARCHIVE}" >&2
  ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true
  exit 1
fi

actual_bytes="$(wc -c < "${REMOTE_ARCHIVE}" | tr -d '[:space:]')"
if [[ "${actual_bytes}" != "${EXPECTED_BYTES}" ]]; then
  echo "Uploaded archive size mismatch: expected ${EXPECTED_BYTES}, got ${actual_bytes}" >&2
  exit 1
fi

echo "    uploaded ${REMOTE_ARCHIVE} (${actual_bytes} bytes)"
REMOTE_VERIFY

echo "==> Importing images and rolling deployments on ${TARGET_HOST}"
remote_ssh \
  "NAMESPACE='${NAMESPACE}' LOCAL_PREFIX='${LOCAL_PREFIX}' IMAGE_TAG='${IMAGE_TAG}' SERVICES='${service_list}' REMOTE_ARCHIVE='${remote_archive}' LOCK_DIR='${remote_lock_dir}' WAIT_TIMEOUT='${WAIT_TIMEOUT}' DELETE_REMOTE_TAR='${DELETE_REMOTE_TAR}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' bash -s" <<'REMOTE'
set -euo pipefail

acquire_rollout_lock() {
  if mkdir "${LOCK_DIR}" 2>/dev/null; then
    printf '%s\n' "$$" > "${LOCK_DIR}/pid"
    printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag"
    date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true
    return 0
  fi

  existing_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)"
  existing_tag="$(cat "${LOCK_DIR}/image_tag" 2>/dev/null || true)"
  existing_created_at="$(cat "${LOCK_DIR}/created_at" 2>/dev/null || true)"

  if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then
    echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2
    echo "  pid: ${existing_pid}" >&2
    echo "  image tag: ${existing_tag:-unknown}" >&2
    echo "  created: ${existing_created_at:-unknown}" >&2
    exit 1
  fi

  echo "Removing stale k3s image rollout lock: ${LOCK_DIR}"
  rm -rf "${LOCK_DIR}"
  if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
    echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2
    exit 1
  fi
  printf '%s\n' "$$" > "${LOCK_DIR}/pid"
  printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag"
  date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true
}

release_rollout_lock() {
  owner_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)"
  if [[ "${owner_pid}" == "$$" ]]; then
    rm -rf "${LOCK_DIR}"
  fi
}

acquire_rollout_lock
trap release_rollout_lock EXIT INT TERM

if ! command -v k3s >/dev/null 2>&1; then
  echo "k3s is required on the target host." >&2
  exit 1
fi

kubectl_cmd() {
  k3s kubectl "$@"
}

echo "    importing ${REMOTE_ARCHIVE}"
if [[ ! -s "${REMOTE_ARCHIVE}" ]]; then
  echo "Remote archive is missing or empty before import: ${REMOTE_ARCHIVE}" >&2
  ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true
  exit 1
fi
k3s ctr -n k8s.io images import "${REMOTE_ARCHIVE}"

if [[ "${RUN_MIGRATIONS}" == "true" ]]; then
  migrate_image="${LOCAL_PREFIX}/migrate:${IMAGE_TAG}"
  echo "    running migration job with ${migrate_image}"
  kubectl_cmd -n "${NAMESPACE}" delete job migrate --ignore-not-found
  cat <<EOF | kubectl_cmd -n "${NAMESPACE}" apply -f -
apiVersion: batch/v1
kind: Job
metadata:
  name: migrate
  labels:
    app.kubernetes.io/name: migrate
spec:
  backoffLimit: 6
  template:
    metadata:
      labels:
        app.kubernetes.io/name: migrate
    spec:
      restartPolicy: OnFailure
      containers:
        - name: migrate
          image: ${migrate_image}
          imagePullPolicy: IfNotPresent
          env:
            - name: CONFIG_PATH
              value: /app/configs/config.yaml
            - name: POSTGRES_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: geo-rankly-app-secret
                  key: POSTGRES_PASSWORD
          volumeMounts:
            - name: app-config
              mountPath: /app/configs
              readOnly: true
          command:
            - /bin/sh
            - -ec
            - |
              export PGPASSWORD="\${POSTGRES_PASSWORD}"
              /usr/local/bin/migrate -path /migrations -database "postgres://geo@postgres:5432/geo?sslmode=disable" up
              /usr/local/bin/migrate -path /migrations_monitoring -database "postgres://geo@monitoring-postgres:5432/geo_monitoring?sslmode=disable" up
              /usr/local/bin/migrate -path /migrations_ops -database "postgres://geo@postgres:5432/geo?sslmode=disable&x-migrations-table=schema_migrations_ops" up
              /usr/local/bin/seed-platform-templates
      volumes:
        - name: app-config
          projected:
            sources:
              - configMap:
                  name: geo-rankly-app-config
                  items:
                    - key: config.yaml
                      path: config.yaml
                    - key: prompts.yml
                      path: prompts.yml
              - secret:
                  name: geo-rankly-app-secret
                  items:
                    - key: config.local.yaml
                      path: config.local.yaml
EOF
  if ! kubectl_cmd -n "${NAMESPACE}" wait --for=condition=complete job/migrate --timeout="${WAIT_TIMEOUT}"; then
    kubectl_cmd -n "${NAMESPACE}" logs job/migrate --tail=200 || true
    kubectl_cmd -n "${NAMESPACE}" describe job migrate || true
    exit 1
  fi
fi

for svc in ${SERVICES}; do
  image="${LOCAL_PREFIX}/${svc}:${IMAGE_TAG}"
  echo "    deployment/${svc} -> ${image}"
  kubectl_cmd -n "${NAMESPACE}" set image "deployment/${svc}" "${svc}=${image}"
  kubectl_cmd -n "${NAMESPACE}" rollout restart "deployment/${svc}"
done

for svc in ${SERVICES}; do
  echo "    waiting deployment/${svc}"
  kubectl_cmd -n "${NAMESPACE}" rollout status "deployment/${svc}" --timeout="${WAIT_TIMEOUT}"
done

if [[ -n "${SERVICES}" ]]; then
  kubectl_cmd -n "${NAMESPACE}" get deploy ${SERVICES}
fi

if [[ "${DELETE_REMOTE_TAR}" == "true" ]]; then
  rm -f "${REMOTE_ARCHIVE}"
fi
REMOTE

rm -f "${archive}"

if [[ "${CLEAN_LOCAL_IMAGES}" == "true" ]]; then
  docker image rm "${images[@]}" >/dev/null 2>&1 || true
fi

echo ""
echo "k3s image rollout complete: ${IMAGE_TAG}"