diff --git a/.gitea/workflows/nas-compose-service-deploy.yml b/.gitea/workflows/nas-compose-service-deploy.yml index aa44fdd..9bb70a7 100644 --- a/.gitea/workflows/nas-compose-service-deploy.yml +++ b/.gitea/workflows/nas-compose-service-deploy.yml @@ -545,11 +545,78 @@ jobs: shared_dir="${DEPLOY_BASE}/shared" lock_dir="${DEPLOY_BASE}/.deploy.lock" - if ! mkdir "${lock_dir}" 2>/dev/null; then - echo "Another deployment is already running: ${lock_dir}" - exit 1 - fi - trap 'rmdir "${lock_dir}"' EXIT INT TERM + lock_owned=false + + write_lock_metadata() { + printf '%s\n' "$$" > "${lock_dir}/pid" + printf '%s\n' "${RELEASE_ID}" > "${lock_dir}/release_id" + printf '%s\n' "${IMAGE_TAG}" > "${lock_dir}/image_tag" + printf '%s\n' "${SERVICE_LIST}" > "${lock_dir}/services" + date -u '+%Y-%m-%dT%H:%M:%SZ' > "${lock_dir}/created_at" 2>/dev/null || true + } + + print_lock_metadata() { + echo " pid: $(cat "${lock_dir}/pid" 2>/dev/null || echo unknown)" + echo " release: $(cat "${lock_dir}/release_id" 2>/dev/null || echo unknown)" + echo " image tag: $(cat "${lock_dir}/image_tag" 2>/dev/null || echo unknown)" + echo " services: $(cat "${lock_dir}/services" 2>/dev/null || echo unknown)" + echo " created: $(cat "${lock_dir}/created_at" 2>/dev/null || echo unknown)" + } + + acquire_deploy_lock() { + if mkdir "${lock_dir}" 2>/dev/null; then + lock_owned=true + write_lock_metadata + return 0 + fi + + existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)" + if [ -z "${existing_pid}" ]; then + sleep 2 + existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)" + fi + + case "${existing_pid}" in + ""|*[!0-9]*) + echo "Another deployment may already be running: ${lock_dir}" + print_lock_metadata + echo "This lock has no PID metadata, so it may have been created by an older workflow." + echo "After confirming no NAS Compose deployment is active, remove it with:" + echo " rm -rf '${lock_dir}'" + exit 1 + ;; + *) + if kill -0 "${existing_pid}" 2>/dev/null; then + echo "Another deployment is already running: ${lock_dir}" + print_lock_metadata + exit 1 + fi + ;; + esac + + echo "Removing stale deployment lock: ${lock_dir}" + print_lock_metadata + rm -rf "${lock_dir}" + + if ! mkdir "${lock_dir}" 2>/dev/null; then + echo "Another deployment is already running: ${lock_dir}" + exit 1 + fi + lock_owned=true + write_lock_metadata + } + + release_deploy_lock() { + if [ "${lock_owned}" = "true" ]; then + owner_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)" + if [ "${owner_pid}" = "$$" ]; then + rm -rf "${lock_dir}" + fi + fi + } + + acquire_deploy_lock + trap 'release_deploy_lock' EXIT INT TERM if [ ! -f "${shared_dir}/.env" ]; then cp "${release_dir}/deploy/.env.example" "${shared_dir}/.env" diff --git a/.gitea/workflows/nas-k3s-deploy.yml b/.gitea/workflows/nas-k3s-deploy.yml index 112601a..484326f 100644 --- a/.gitea/workflows/nas-k3s-deploy.yml +++ b/.gitea/workflows/nas-k3s-deploy.yml @@ -621,11 +621,78 @@ jobs: lock_dir="${DEPLOY_BASE}/.deploy.lock" rendered_dir="${release_dir}/.k3s-rendered" - if ! mkdir "${lock_dir}" 2>/dev/null; then - echo "Another deployment is already running: ${lock_dir}" - exit 1 - fi - trap 'rmdir "${lock_dir}"' EXIT INT TERM + lock_owned=false + + write_lock_metadata() { + printf '%s\n' "$$" > "${lock_dir}/pid" + printf '%s\n' "${RELEASE_ID}" > "${lock_dir}/release_id" + printf '%s\n' "${IMAGE_TAG}" > "${lock_dir}/image_tag" + printf '%s\n' "${SELECTED_TARGET}" > "${lock_dir}/target" + date -u '+%Y-%m-%dT%H:%M:%SZ' > "${lock_dir}/created_at" 2>/dev/null || true + } + + print_lock_metadata() { + echo " pid: $(cat "${lock_dir}/pid" 2>/dev/null || echo unknown)" + echo " release: $(cat "${lock_dir}/release_id" 2>/dev/null || echo unknown)" + echo " image tag: $(cat "${lock_dir}/image_tag" 2>/dev/null || echo unknown)" + echo " target: $(cat "${lock_dir}/target" 2>/dev/null || echo unknown)" + echo " created: $(cat "${lock_dir}/created_at" 2>/dev/null || echo unknown)" + } + + acquire_deploy_lock() { + if mkdir "${lock_dir}" 2>/dev/null; then + lock_owned=true + write_lock_metadata + return 0 + fi + + existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)" + if [ -z "${existing_pid}" ]; then + sleep 2 + existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)" + fi + + case "${existing_pid}" in + ""|*[!0-9]*) + echo "Another deployment may already be running: ${lock_dir}" + print_lock_metadata + echo "This lock has no PID metadata, so it may have been created by an older workflow." + echo "After confirming no NAS k3s deployment is active, remove it with:" + echo " rm -rf '${lock_dir}'" + exit 1 + ;; + *) + if kill -0 "${existing_pid}" 2>/dev/null; then + echo "Another deployment is already running: ${lock_dir}" + print_lock_metadata + exit 1 + fi + ;; + esac + + echo "Removing stale deployment lock: ${lock_dir}" + print_lock_metadata + rm -rf "${lock_dir}" + + if ! mkdir "${lock_dir}" 2>/dev/null; then + echo "Another deployment is already running: ${lock_dir}" + exit 1 + fi + lock_owned=true + write_lock_metadata + } + + release_deploy_lock() { + if [ "${lock_owned}" = "true" ]; then + owner_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)" + if [ "${owner_pid}" = "$$" ]; then + rm -rf "${lock_dir}" + fi + fi + } + + acquire_deploy_lock + trap 'release_deploy_lock' EXIT INT TERM SUDO_PASSWORD="$(printf '%s' "${SUDO_PASSWORD_B64}" | base64 -d 2>/dev/null || true)" REGISTRY_PASSWORD="$(printf '%s' "${REGISTRY_PASSWORD_B64}" | base64 -d 2>/dev/null || true)" diff --git a/deploy/scripts/deploy-images-to-k3s.sh b/deploy/scripts/deploy-images-to-k3s.sh index 1cff998..e7b711d 100755 --- a/deploy/scripts/deploy-images-to-k3s.sh +++ b/deploy/scripts/deploy-images-to-k3s.sh @@ -270,14 +270,17 @@ for svc in "${IMAGE_SERVICES[@]}"; do done safe_tag="$(printf '%s' "${IMAGE_TAG}" | tr -c 'A-Za-z0-9_.-' '-')" -archive="/tmp/geo-rankly-images-${safe_tag}.tar" -remote_archive="${REMOTE_DIR}/geo-rankly-images-${safe_tag}.tar" +run_suffix="$(printf '%s-%s-%s' "${GITHUB_RUN_ID:-manual}" "${GITHUB_RUN_ATTEMPT:-0}" "$$" | tr -c 'A-Za-z0-9_.-' '-')" +archive="/tmp/geo-rankly-images-${safe_tag}-${run_suffix}.tar" +remote_archive="${REMOTE_DIR}/geo-rankly-images-${safe_tag}-${run_suffix}.tar" +remote_lock_dir="${REMOTE_DIR}/.k3s-image-rollout.lock" service_list="${SERVICES[*]}" echo "==> Saving images:" printf ' %s\n' "${images[@]}" docker save "${images[@]}" -o "${archive}" echo " archive: ${archive} ($(du -sh "${archive}" | cut -f1))" +archive_bytes="$(wc -c < "${archive}" | tr -d '[:space:]')" ssh_cmd=(ssh -p "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new) scp_cmd=(scp -P "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new) @@ -303,12 +306,71 @@ remote_scp() { echo "==> Uploading archive to ${TARGET_HOST}:${remote_archive}" remote_ssh "mkdir -p '${REMOTE_DIR}'" remote_scp "${archive}" "${remote_archive}" +remote_ssh \ + "REMOTE_ARCHIVE='${remote_archive}' EXPECTED_BYTES='${archive_bytes}' bash -se" <<'REMOTE_VERIFY' +set -euo pipefail + +if [[ ! -f "${REMOTE_ARCHIVE}" ]]; then + echo "Uploaded archive is missing on target: ${REMOTE_ARCHIVE}" >&2 + ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true + exit 1 +fi + +actual_bytes="$(wc -c < "${REMOTE_ARCHIVE}" | tr -d '[:space:]')" +if [[ "${actual_bytes}" != "${EXPECTED_BYTES}" ]]; then + echo "Uploaded archive size mismatch: expected ${EXPECTED_BYTES}, got ${actual_bytes}" >&2 + exit 1 +fi + +echo " uploaded ${REMOTE_ARCHIVE} (${actual_bytes} bytes)" +REMOTE_VERIFY echo "==> Importing images and rolling deployments on ${TARGET_HOST}" remote_ssh \ - "NAMESPACE='${NAMESPACE}' LOCAL_PREFIX='${LOCAL_PREFIX}' IMAGE_TAG='${IMAGE_TAG}' SERVICES='${service_list}' REMOTE_ARCHIVE='${remote_archive}' WAIT_TIMEOUT='${WAIT_TIMEOUT}' DELETE_REMOTE_TAR='${DELETE_REMOTE_TAR}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' bash -s" <<'REMOTE' + "NAMESPACE='${NAMESPACE}' LOCAL_PREFIX='${LOCAL_PREFIX}' IMAGE_TAG='${IMAGE_TAG}' SERVICES='${service_list}' REMOTE_ARCHIVE='${remote_archive}' LOCK_DIR='${remote_lock_dir}' WAIT_TIMEOUT='${WAIT_TIMEOUT}' DELETE_REMOTE_TAR='${DELETE_REMOTE_TAR}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' bash -s" <<'REMOTE' set -euo pipefail +acquire_rollout_lock() { + if mkdir "${LOCK_DIR}" 2>/dev/null; then + printf '%s\n' "$$" > "${LOCK_DIR}/pid" + printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag" + date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true + return 0 + fi + + existing_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)" + existing_tag="$(cat "${LOCK_DIR}/image_tag" 2>/dev/null || true)" + existing_created_at="$(cat "${LOCK_DIR}/created_at" 2>/dev/null || true)" + + if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then + echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2 + echo " pid: ${existing_pid}" >&2 + echo " image tag: ${existing_tag:-unknown}" >&2 + echo " created: ${existing_created_at:-unknown}" >&2 + exit 1 + fi + + echo "Removing stale k3s image rollout lock: ${LOCK_DIR}" + rm -rf "${LOCK_DIR}" + if ! mkdir "${LOCK_DIR}" 2>/dev/null; then + echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2 + exit 1 + fi + printf '%s\n' "$$" > "${LOCK_DIR}/pid" + printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag" + date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true +} + +release_rollout_lock() { + owner_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)" + if [[ "${owner_pid}" == "$$" ]]; then + rm -rf "${LOCK_DIR}" + fi +} + +acquire_rollout_lock +trap release_rollout_lock EXIT INT TERM + if ! command -v k3s >/dev/null 2>&1; then echo "k3s is required on the target host." >&2 exit 1 @@ -319,6 +381,11 @@ kubectl_cmd() { } echo " importing ${REMOTE_ARCHIVE}" +if [[ ! -s "${REMOTE_ARCHIVE}" ]]; then + echo "Remote archive is missing or empty before import: ${REMOTE_ARCHIVE}" >&2 + ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true + exit 1 +fi k3s ctr -n k8s.io images import "${REMOTE_ARCHIVE}" if [[ "${RUN_MIGRATIONS}" == "true" ]]; then