feat(deploy): enhance deployment scripts with locking mechanism and metadata tracking

.gitea/workflows/nas-compose-service-deploy.yml

@@ -545,11 +545,78 @@ jobs:
           shared_dir="${DEPLOY_BASE}/shared"
           lock_dir="${DEPLOY_BASE}/.deploy.lock"
 
-          if ! mkdir "${lock_dir}" 2>/dev/null; then
+          lock_owned=false
-            echo "Another deployment is already running: ${lock_dir}"
+
-            exit 1
+          write_lock_metadata() {
-          fi
+            printf '%s\n' "$$" > "${lock_dir}/pid"
-          trap 'rmdir "${lock_dir}"' EXIT INT TERM
+            printf '%s\n' "${RELEASE_ID}" > "${lock_dir}/release_id"
+            printf '%s\n' "${IMAGE_TAG}" > "${lock_dir}/image_tag"
+            printf '%s\n' "${SERVICE_LIST}" > "${lock_dir}/services"
+            date -u '+%Y-%m-%dT%H:%M:%SZ' > "${lock_dir}/created_at" 2>/dev/null || true
+          }
+
+          print_lock_metadata() {
+            echo "  pid: $(cat "${lock_dir}/pid" 2>/dev/null || echo unknown)"
+            echo "  release: $(cat "${lock_dir}/release_id" 2>/dev/null || echo unknown)"
+            echo "  image tag: $(cat "${lock_dir}/image_tag" 2>/dev/null || echo unknown)"
+            echo "  services: $(cat "${lock_dir}/services" 2>/dev/null || echo unknown)"
+            echo "  created: $(cat "${lock_dir}/created_at" 2>/dev/null || echo unknown)"
+          }
+
+          acquire_deploy_lock() {
+            if mkdir "${lock_dir}" 2>/dev/null; then
+              lock_owned=true
+              write_lock_metadata
+              return 0
+            fi
+
+            existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)"
+            if [ -z "${existing_pid}" ]; then
+              sleep 2
+              existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)"
+            fi
+
+            case "${existing_pid}" in
+              ""|*[!0-9]*)
+                echo "Another deployment may already be running: ${lock_dir}"
+                print_lock_metadata
+                echo "This lock has no PID metadata, so it may have been created by an older workflow."
+                echo "After confirming no NAS Compose deployment is active, remove it with:"
+                echo "  rm -rf '${lock_dir}'"
+                exit 1
+                ;;
+              *)
+                if kill -0 "${existing_pid}" 2>/dev/null; then
+                  echo "Another deployment is already running: ${lock_dir}"
+                  print_lock_metadata
+                  exit 1
+                fi
+                ;;
+            esac
+
+            echo "Removing stale deployment lock: ${lock_dir}"
+            print_lock_metadata
+            rm -rf "${lock_dir}"
+
+            if ! mkdir "${lock_dir}" 2>/dev/null; then
+              echo "Another deployment is already running: ${lock_dir}"
+              exit 1
+            fi
+            lock_owned=true
+            write_lock_metadata
+          }
+
+          release_deploy_lock() {
+            if [ "${lock_owned}" = "true" ]; then
+              owner_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)"
+              if [ "${owner_pid}" = "$$" ]; then
+                rm -rf "${lock_dir}"
+              fi
+            fi
+          }
+
+          acquire_deploy_lock
+          trap 'release_deploy_lock' EXIT INT TERM
 
           if [ ! -f "${shared_dir}/.env" ]; then
             cp "${release_dir}/deploy/.env.example" "${shared_dir}/.env"

.gitea/workflows/nas-k3s-deploy.yml

@@ -621,11 +621,78 @@ jobs:
           lock_dir="${DEPLOY_BASE}/.deploy.lock"
           rendered_dir="${release_dir}/.k3s-rendered"
 
-          if ! mkdir "${lock_dir}" 2>/dev/null; then
+          lock_owned=false
-            echo "Another deployment is already running: ${lock_dir}"
+
-            exit 1
+          write_lock_metadata() {
-          fi
+            printf '%s\n' "$$" > "${lock_dir}/pid"
-          trap 'rmdir "${lock_dir}"' EXIT INT TERM
+            printf '%s\n' "${RELEASE_ID}" > "${lock_dir}/release_id"
+            printf '%s\n' "${IMAGE_TAG}" > "${lock_dir}/image_tag"
+            printf '%s\n' "${SELECTED_TARGET}" > "${lock_dir}/target"
+            date -u '+%Y-%m-%dT%H:%M:%SZ' > "${lock_dir}/created_at" 2>/dev/null || true
+          }
+
+          print_lock_metadata() {
+            echo "  pid: $(cat "${lock_dir}/pid" 2>/dev/null || echo unknown)"
+            echo "  release: $(cat "${lock_dir}/release_id" 2>/dev/null || echo unknown)"
+            echo "  image tag: $(cat "${lock_dir}/image_tag" 2>/dev/null || echo unknown)"
+            echo "  target: $(cat "${lock_dir}/target" 2>/dev/null || echo unknown)"
+            echo "  created: $(cat "${lock_dir}/created_at" 2>/dev/null || echo unknown)"
+          }
+
+          acquire_deploy_lock() {
+            if mkdir "${lock_dir}" 2>/dev/null; then
+              lock_owned=true
+              write_lock_metadata
+              return 0
+            fi
+
+            existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)"
+            if [ -z "${existing_pid}" ]; then
+              sleep 2
+              existing_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)"
+            fi
+
+            case "${existing_pid}" in
+              ""|*[!0-9]*)
+                echo "Another deployment may already be running: ${lock_dir}"
+                print_lock_metadata
+                echo "This lock has no PID metadata, so it may have been created by an older workflow."
+                echo "After confirming no NAS k3s deployment is active, remove it with:"
+                echo "  rm -rf '${lock_dir}'"
+                exit 1
+                ;;
+              *)
+                if kill -0 "${existing_pid}" 2>/dev/null; then
+                  echo "Another deployment is already running: ${lock_dir}"
+                  print_lock_metadata
+                  exit 1
+                fi
+                ;;
+            esac
+
+            echo "Removing stale deployment lock: ${lock_dir}"
+            print_lock_metadata
+            rm -rf "${lock_dir}"
+
+            if ! mkdir "${lock_dir}" 2>/dev/null; then
+              echo "Another deployment is already running: ${lock_dir}"
+              exit 1
+            fi
+            lock_owned=true
+            write_lock_metadata
+          }
+
+          release_deploy_lock() {
+            if [ "${lock_owned}" = "true" ]; then
+              owner_pid="$(cat "${lock_dir}/pid" 2>/dev/null || true)"
+              if [ "${owner_pid}" = "$$" ]; then
+                rm -rf "${lock_dir}"
+              fi
+            fi
+          }
+
+          acquire_deploy_lock
+          trap 'release_deploy_lock' EXIT INT TERM
 
           SUDO_PASSWORD="$(printf '%s' "${SUDO_PASSWORD_B64}" | base64 -d 2>/dev/null || true)"
           REGISTRY_PASSWORD="$(printf '%s' "${REGISTRY_PASSWORD_B64}" | base64 -d 2>/dev/null || true)"

deploy/scripts/deploy-images-to-k3s.sh

@@ -270,14 +270,17 @@ for svc in "${IMAGE_SERVICES[@]}"; do
 done
 
 safe_tag="$(printf '%s' "${IMAGE_TAG}" | tr -c 'A-Za-z0-9_.-' '-')"
-archive="/tmp/geo-rankly-images-${safe_tag}.tar"
+run_suffix="$(printf '%s-%s-%s' "${GITHUB_RUN_ID:-manual}" "${GITHUB_RUN_ATTEMPT:-0}" "$$" | tr -c 'A-Za-z0-9_.-' '-')"
-remote_archive="${REMOTE_DIR}/geo-rankly-images-${safe_tag}.tar"
+archive="/tmp/geo-rankly-images-${safe_tag}-${run_suffix}.tar"
+remote_archive="${REMOTE_DIR}/geo-rankly-images-${safe_tag}-${run_suffix}.tar"
+remote_lock_dir="${REMOTE_DIR}/.k3s-image-rollout.lock"
 service_list="${SERVICES[*]}"
 
 echo "==> Saving images:"
 printf '    %s\n' "${images[@]}"
 docker save "${images[@]}" -o "${archive}"
 echo "    archive: ${archive} ($(du -sh "${archive}" | cut -f1))"
+archive_bytes="$(wc -c < "${archive}" | tr -d '[:space:]')"
 
 ssh_cmd=(ssh -p "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new)
 scp_cmd=(scp -P "${TARGET_PORT}" -o BatchMode=no -o StrictHostKeyChecking=accept-new)
@@ -303,12 +306,71 @@ remote_scp() {
 echo "==> Uploading archive to ${TARGET_HOST}:${remote_archive}"
 remote_ssh "mkdir -p '${REMOTE_DIR}'"
 remote_scp "${archive}" "${remote_archive}"
+remote_ssh \
+  "REMOTE_ARCHIVE='${remote_archive}' EXPECTED_BYTES='${archive_bytes}' bash -se" <<'REMOTE_VERIFY'
+set -euo pipefail
+
+if [[ ! -f "${REMOTE_ARCHIVE}" ]]; then
+  echo "Uploaded archive is missing on target: ${REMOTE_ARCHIVE}" >&2
+  ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true
+  exit 1
+fi
+
+actual_bytes="$(wc -c < "${REMOTE_ARCHIVE}" | tr -d '[:space:]')"
+if [[ "${actual_bytes}" != "${EXPECTED_BYTES}" ]]; then
+  echo "Uploaded archive size mismatch: expected ${EXPECTED_BYTES}, got ${actual_bytes}" >&2
+  exit 1
+fi
+
+echo "    uploaded ${REMOTE_ARCHIVE} (${actual_bytes} bytes)"
+REMOTE_VERIFY
 
 echo "==> Importing images and rolling deployments on ${TARGET_HOST}"
 remote_ssh \
-  "NAMESPACE='${NAMESPACE}' LOCAL_PREFIX='${LOCAL_PREFIX}' IMAGE_TAG='${IMAGE_TAG}' SERVICES='${service_list}' REMOTE_ARCHIVE='${remote_archive}' WAIT_TIMEOUT='${WAIT_TIMEOUT}' DELETE_REMOTE_TAR='${DELETE_REMOTE_TAR}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' bash -s" <<'REMOTE'
+  "NAMESPACE='${NAMESPACE}' LOCAL_PREFIX='${LOCAL_PREFIX}' IMAGE_TAG='${IMAGE_TAG}' SERVICES='${service_list}' REMOTE_ARCHIVE='${remote_archive}' LOCK_DIR='${remote_lock_dir}' WAIT_TIMEOUT='${WAIT_TIMEOUT}' DELETE_REMOTE_TAR='${DELETE_REMOTE_TAR}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' bash -s" <<'REMOTE'
 set -euo pipefail
 
+acquire_rollout_lock() {
+  if mkdir "${LOCK_DIR}" 2>/dev/null; then
+    printf '%s\n' "$$" > "${LOCK_DIR}/pid"
+    printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag"
+    date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true
+    return 0
+  fi
+
+  existing_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)"
+  existing_tag="$(cat "${LOCK_DIR}/image_tag" 2>/dev/null || true)"
+  existing_created_at="$(cat "${LOCK_DIR}/created_at" 2>/dev/null || true)"
+
+  if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then
+    echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2
+    echo "  pid: ${existing_pid}" >&2
+    echo "  image tag: ${existing_tag:-unknown}" >&2
+    echo "  created: ${existing_created_at:-unknown}" >&2
+    exit 1
+  fi
+
+  echo "Removing stale k3s image rollout lock: ${LOCK_DIR}"
+  rm -rf "${LOCK_DIR}"
+  if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
+    echo "Another k3s image rollout is already running: ${LOCK_DIR}" >&2
+    exit 1
+  fi
+  printf '%s\n' "$$" > "${LOCK_DIR}/pid"
+  printf '%s\n' "${IMAGE_TAG}" > "${LOCK_DIR}/image_tag"
+  date -u '+%Y-%m-%dT%H:%M:%SZ' > "${LOCK_DIR}/created_at" 2>/dev/null || true
+}
+
+release_rollout_lock() {
+  owner_pid="$(cat "${LOCK_DIR}/pid" 2>/dev/null || true)"
+  if [[ "${owner_pid}" == "$$" ]]; then
+    rm -rf "${LOCK_DIR}"
+  fi
+}
+
+acquire_rollout_lock
+trap release_rollout_lock EXIT INT TERM
+
 if ! command -v k3s >/dev/null 2>&1; then
   echo "k3s is required on the target host." >&2
   exit 1
@@ -319,6 +381,11 @@ kubectl_cmd() {
 }
 
 echo "    importing ${REMOTE_ARCHIVE}"
+if [[ ! -s "${REMOTE_ARCHIVE}" ]]; then
+  echo "Remote archive is missing or empty before import: ${REMOTE_ARCHIVE}" >&2
+  ls -la "$(dirname "${REMOTE_ARCHIVE}")" >&2 || true
+  exit 1
+fi
 k3s ctr -n k8s.io images import "${REMOTE_ARCHIVE}"
 
 if [[ "${RUN_MIGRATIONS}" == "true" ]]; then