name: Deploy NAS K3s

"on":
  workflow_dispatch:
    inputs:
      target:
        description: "Service, group, full deploy, or manifest to update"
        required: true
        default: "frontend"
        type: choice
        options:
          - frontend
          - ops-web
          - browser-fetch
          - tenant-api
          - ops-api
          - worker-generate
          - kol-assist-worker
          - scheduler
          - web
          - backend
          - all-apps
          - all
          - migrate
          - manifest
      manifest_path:
        description: "deploy/k3s yaml to apply when target=manifest, e.g. ingress.yaml or config/ops-config.yaml"
        required: false
        default: ""
      run_migrations:
        description: "Run database migrations before app rollout"
        required: true
        default: "auto"
        type: choice
        options:
          - auto
          - yes
          - no
      validate_migrations:
        description: "Validate migrations on an empty Postgres first when migrations run"
        required: true
        default: "true"
        type: choice
        options:
          - "true"
          - "false"
      host:
        description: "NAS SSH host"
        required: true
        default: "192.168.100.19"
      port:
        description: "NAS SSH port"
        required: true
        default: "22"
      user:
        description: "NAS SSH user"
        required: true
        default: "admin"
      password_source:
        description: "NAS SSH password source"
        required: true
        default: "repository_secret"
        type: choice
        options:
          - repository_secret
          - workflow_input
      ssh_password:
        description: "NAS SSH password. Used only when password_source is workflow_input."
        required: false
        default: ""
      deploy_base:
        description: "Base deployment directory on the NAS"
        required: true
        default: "/vol1/1000/application/geo-k3s"
      namespace:
        description: "Kubernetes namespace"
        required: true
        default: "geo-rankly"
      registry_host:
        description: "Gitea Registry host, without scheme"
        required: true
        default: "192.168.100.19:13000"
      registry_owner:
        description: "Gitea package owner"
        required: true
        default: "root"
      registry_image_prefix:
        description: "Image package prefix. Images become <prefix>/<service>."
        required: true
        default: "geo-rankly"
      registry_username:
        description: "Gitea Registry username"
        required: true
        default: "root"
      registry_password_source:
        description: "Gitea Registry password source"
        required: true
        default: "repository_secret"
        type: choice
        options:
          - repository_secret
          - workflow_input
      registry_password:
        description: "Gitea Registry password/token. Used only when registry_password_source is workflow_input."
        required: false
        default: ""
      registry_plain_http:
        description: "Registry uses plain HTTP and must be configured as an insecure registry."
        required: true
        default: "true"
        type: choice
        options:
          - "true"
          - "false"
      image_tag:
        description: "Docker image tag. Defaults to 'latest' (deploy whatever CI most recently published)."
        required: false
        default: ""
      keep_releases:
        description: "Number of old release directories to keep. 0 keeps all releases."
        required: false
        default: "0"

permissions:
  contents: read

jobs:
  deploy-k3s:
    name: Deploy selected k3s target to NAS
    runs-on: ubuntu-latest
    timeout-minutes: 180
    env:
      NAS_HOST: ${{ inputs.host }}
      NAS_PORT: ${{ inputs.port }}
      NAS_USER: ${{ inputs.user }}
      NAS_SSH_PASSWORD_SOURCE: ${{ inputs.password_source }}
      NAS_SSH_PASSWORD_INPUT: ${{ inputs.ssh_password }}
      NAS_SSH_PASSWORD_SECRET: ${{ secrets.NAS_SSH_PASSWORD }}
      DEPLOY_BASE: ${{ inputs.deploy_base }}
      K3S_NAMESPACE: ${{ inputs.namespace }}
      REGISTRY_HOST: ${{ inputs.registry_host }}
      REGISTRY_OWNER: ${{ inputs.registry_owner }}
      REGISTRY_IMAGE_PREFIX: ${{ inputs.registry_image_prefix }}
      REGISTRY_USERNAME: ${{ inputs.registry_username }}
      REGISTRY_PASSWORD_SOURCE: ${{ inputs.registry_password_source }}
      REGISTRY_PASSWORD_INPUT: ${{ inputs.registry_password }}
      REGISTRY_PASSWORD_SECRET: ${{ secrets.REGISTRY_PASSWORD }}
      REGISTRY_PLAIN_HTTP: ${{ inputs.registry_plain_http }}
      SELECTED_TARGET: ${{ inputs.target }}
      MANIFEST_PATH: ${{ inputs.manifest_path }}
      REQUESTED_IMAGE_TAG: ${{ inputs.image_tag }}
      REQUESTED_RUN_MIGRATIONS: ${{ inputs.run_migrations }}
      VALIDATE_MIGRATIONS: ${{ inputs.validate_migrations }}
      KEEP_RELEASES: ${{ inputs.keep_releases }}
      DOCKER_BUILDKIT: "1"

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install SSH tools
        run: |
          apt-get update
          apt-get install -y curl openssh-client sshpass tar

      - name: Resolve deployment metadata
        run: |
          set -eu

          case "${NAS_SSH_PASSWORD_SOURCE:-repository_secret}" in
            repository_secret)
              nas_ssh_password="${NAS_SSH_PASSWORD_SECRET:-}"
              ;;
            workflow_input)
              nas_ssh_password="${NAS_SSH_PASSWORD_INPUT:-}"
              ;;
            *)
              echo "::error::password_source must be repository_secret or workflow_input"
              exit 1
              ;;
          esac

          if [ -n "${nas_ssh_password}" ]; then
            echo "::add-mask::${nas_ssh_password}"
          fi

          {
            echo "NAS_SSH_PASSWORD<<__NAS_SSH_PASSWORD__"
            printf '%s\n' "${nas_ssh_password}"
            echo "__NAS_SSH_PASSWORD__"
            echo "NAS_SSH_PASSWORD_INPUT="
            echo "NAS_SSH_PASSWORD_SECRET="
          } >> "${GITHUB_ENV}"

          if [ -z "${nas_ssh_password}" ]; then
            if [ "${NAS_SSH_PASSWORD_SOURCE:-repository_secret}" = "workflow_input" ]; then
              echo "::error::Missing workflow input ssh_password"
            else
              echo "::error::Missing repository secret NAS_SSH_PASSWORD"
            fi
            exit 1
          fi

          case "${REGISTRY_PASSWORD_SOURCE:-repository_secret}" in
            repository_secret)
              registry_password="${REGISTRY_PASSWORD_SECRET:-}"
              ;;
            workflow_input)
              registry_password="${REGISTRY_PASSWORD_INPUT:-}"
              ;;
            *)
              echo "::error::registry_password_source must be repository_secret or workflow_input"
              exit 1
              ;;
          esac

          if [ -n "${registry_password}" ]; then
            echo "::add-mask::${registry_password}"
          fi

          {
            echo "REGISTRY_PASSWORD<<__REGISTRY_PASSWORD__"
            printf '%s\n' "${registry_password}"
            echo "__REGISTRY_PASSWORD__"
            echo "REGISTRY_PASSWORD_INPUT="
            echo "REGISTRY_PASSWORD_SECRET="
          } >> "${GITHUB_ENV}"

          REGISTRY_HOST="${REGISTRY_HOST#http://}"
          REGISTRY_HOST="${REGISTRY_HOST#https://}"
          REGISTRY_HOST="${REGISTRY_HOST%/}"

          case "${NAS_PORT}" in
            *[!0-9]*|"")
              echo "::error::NAS SSH port must be numeric"
              exit 1
              ;;
          esac

          case "${DEPLOY_BASE}" in
            /*) ;;
            *)
              echo "::error::deploy_base must be an absolute path"
              exit 1
              ;;
          esac

          case "${DEPLOY_BASE}" in
            *[!A-Za-z0-9_./-]*)
              echo "::error::deploy_base contains unsupported characters"
              exit 1
              ;;
          esac

          case "${K3S_NAMESPACE}" in
            *[!A-Za-z0-9_.-]*|"")
              echo "::error::namespace contains unsupported characters"
              exit 1
              ;;
          esac

          case "${REGISTRY_HOST}" in
            *[!A-Za-z0-9_.:-]*|"")
              echo "::error::registry_host must be a host[:port] without scheme"
              exit 1
              ;;
          esac

          case "${REGISTRY_OWNER}" in
            *[!A-Za-z0-9_.-]*|"")
              echo "::error::registry_owner contains unsupported characters"
              exit 1
              ;;
          esac

          case "${REGISTRY_IMAGE_PREFIX}" in
            *[!A-Za-z0-9_.-]*|"")
              echo "::error::registry_image_prefix contains unsupported characters"
              exit 1
              ;;
          esac

          case "${REGISTRY_USERNAME}" in
            *[!A-Za-z0-9_.@-]*|"")
              echo "::error::registry_username contains unsupported characters"
              exit 1
              ;;
          esac

          case "${REGISTRY_PLAIN_HTTP}" in
            true|false) ;;
            *)
              echo "::error::registry_plain_http must be true or false"
              exit 1
              ;;
          esac

          case "${KEEP_RELEASES}" in
            *[!0-9]*|"") KEEP_RELEASES=0 ;;
          esac

          all_app_images="tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web"
          backend_images="tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler"
          image_list=""
          deployment_list=""
          apply_kustomize=false
          auto_migrations=false

          case "${SELECTED_TARGET}" in
            frontend)
              image_list="frontend"
              deployment_list="frontend"
              ;;
            ops-web)
              image_list="ops-web"
              deployment_list="ops-web"
              ;;
            browser-fetch)
              image_list="browser-fetch"
              deployment_list="browser-fetch"
              ;;
            tenant-api)
              image_list="tenant-api"
              deployment_list="tenant-api"
              auto_migrations=true
              ;;
            ops-api)
              image_list="ops-api"
              deployment_list="ops-api"
              auto_migrations=true
              ;;
            worker-generate)
              image_list="worker-generate"
              deployment_list="worker-generate"
              auto_migrations=true
              ;;
            kol-assist-worker)
              image_list="kol-assist-worker"
              deployment_list="kol-assist-worker"
              auto_migrations=true
              ;;
            scheduler)
              image_list="scheduler"
              deployment_list="scheduler"
              auto_migrations=true
              ;;
            web)
              image_list="frontend ops-web"
              deployment_list="frontend ops-web"
              ;;
            backend)
              image_list="${backend_images}"
              deployment_list="${backend_images}"
              auto_migrations=true
              ;;
            all-apps)
              image_list="${all_app_images}"
              deployment_list="${all_app_images}"
              auto_migrations=true
              ;;
            all)
              image_list="${all_app_images}"
              deployment_list="${all_app_images}"
              apply_kustomize=true
              auto_migrations=true
              ;;
            migrate)
              image_list="migrate"
              auto_migrations=true
              ;;
            manifest)
              manifest_input="${MANIFEST_PATH#./}"
              manifest_input="${manifest_input#deploy/k3s/}"
              manifest_input="${manifest_input%/}"
              if [ -z "${manifest_input}" ]; then
                echo "::error::manifest_path is required when target=manifest"
                exit 1
              fi
              case "${manifest_input}" in
                /*|*..*|*" "*)
                  echo "::error::manifest_path must be a relative path under deploy/k3s without '..' or spaces"
                  exit 1
                  ;;
              esac
              case "${manifest_input}" in
                *[!A-Za-z0-9_./-]*)
                  echo "::error::manifest_path contains unsupported characters"
                  exit 1
                  ;;
              esac
              case "${manifest_input}" in
                apps.yaml|apps.yml)
                  image_list="${all_app_images}"
                  deployment_list="${all_app_images}"
                  ;;
                jobs.yaml|jobs.yml)
                  image_list="migrate"
                  ;;
                kustomization.yaml|kustomization.yml)
                  image_list="${all_app_images}"
                  deployment_list="${all_app_images}"
                  ;;
              esac
              MANIFEST_PATH="${manifest_input}"
              ;;
            *)
              echo "::error::Unsupported target: ${SELECTED_TARGET}"
              exit 1
              ;;
          esac

          case "${REQUESTED_RUN_MIGRATIONS}" in
            auto)
              run_migrations="${auto_migrations}"
              ;;
            yes)
              run_migrations=true
              ;;
            no)
              run_migrations=false
              ;;
            *)
              echo "::error::run_migrations must be auto, yes, or no"
              exit 1
              ;;
          esac

          if [ "${run_migrations}" = "true" ]; then
            case " ${image_list} " in
              *" migrate "*) ;;
              *) image_list="migrate ${image_list}" ;;
            esac
          fi

          case "${VALIDATE_MIGRATIONS}" in
            true|false) ;;
            *)
              echo "::error::validate_migrations must be true or false"
              exit 1
              ;;
          esac

          if [ -n "${image_list}" ] && [ -z "${registry_password}" ]; then
            if [ "${REGISTRY_PASSWORD_SOURCE:-repository_secret}" = "workflow_input" ]; then
              echo "::error::Missing workflow input registry_password"
            else
              echo "::error::Missing repository secret REGISTRY_PASSWORD"
            fi
            exit 1
          fi

          image_tag="${REQUESTED_IMAGE_TAG}"
          if [ -z "${image_tag}" ]; then
            image_tag="latest"
          fi

          case "${image_tag}" in
            *[!A-Za-z0-9_.-]*|"")
              echo "::error::image_tag may only contain letters, numbers, underscore, dot, and dash"
              exit 1
              ;;
          esac

          target_slug="$(printf '%s' "${SELECTED_TARGET}" | tr -c 'A-Za-z0-9_.-' '-')"
          release_id="$(date -u +%Y%m%d%H%M%S)-$(git rev-parse --short=12 HEAD)-k3s-${target_slug}"

          {
            echo "REGISTRY_HOST=${REGISTRY_HOST}"
            echo "REGISTRY_PLAIN_HTTP=${REGISTRY_PLAIN_HTTP}"
            echo "IMAGE_TAG=${image_tag}"
            echo "RELEASE_ID=${release_id}"
            echo "IMAGE_LIST=${image_list}"
            echo "DEPLOYMENT_LIST=${deployment_list}"
            echo "APPLY_KUSTOMIZE=${apply_kustomize}"
            echo "RUN_MIGRATIONS=${run_migrations}"
            echo "MANIFEST_PATH=${MANIFEST_PATH}"
            echo "KEEP_RELEASES=${KEEP_RELEASES}"
          } >> "${GITHUB_ENV}"

          echo "Selected target: ${SELECTED_TARGET}"
          echo "Manifest path: ${MANIFEST_PATH:-}"
          echo "Images to verify/deploy: ${image_list:-none}"
          echo "Deployments to roll out: ${deployment_list:-none}"
          echo "Apply full kustomize: ${apply_kustomize}"
          echo "Run migrations: ${run_migrations}"
          echo "Validate migrations: ${VALIDATE_MIGRATIONS}"
          echo "Image tag: ${image_tag}"
          echo "Release ID: ${release_id}"
          echo "Deploy base: ${DEPLOY_BASE}"
          echo "Gitea Registry: ${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/<service>:${image_tag}"

      - name: Verify current images in Gitea Registry
        run: |
          set -eu

          if [ -z "${IMAGE_LIST}" ]; then
            echo "No images selected; skipping registry verification."
            exit 0
          fi

          if ! bash deploy/scripts/gitea-registry-images.sh verify; then
            echo "::error::Missing image(s) at tag '${IMAGE_TAG}' for: ${IMAGE_LIST}."
            echo "::error::Run the matching CI workflow (Frontend CI / Backend CI) to publish images first."
            exit 1
          fi

      - name: Validate migrations on empty Postgres
        run: |
          set -eu

          if [ "${RUN_MIGRATIONS}" != "true" ] || [ "${VALIDATE_MIGRATIONS}" != "true" ]; then
            echo "Skipping migration preflight."
            exit 0
          fi

          test_name="geo-k3s-migration-preflight-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_NUMBER:-0}"
          network_name="${test_name}-net"
          postgres_name="${test_name}-postgres"
          migrate_image="${test_name}-migrate"

          cleanup() {
            docker rm -f "${postgres_name}" >/dev/null 2>&1 || true
            docker network rm "${network_name}" >/dev/null 2>&1 || true
            docker image rm "${migrate_image}" >/dev/null 2>&1 || true
          }
          trap cleanup EXIT INT TERM

          docker network create "${network_name}" >/dev/null
          docker run -d \
            --name "${postgres_name}" \
            --network "${network_name}" \
            -e POSTGRES_DB=geo \
            -e POSTGRES_USER=geo \
            -e POSTGRES_PASSWORD=geo_dev \
            postgres:16-alpine >/dev/null

          echo "Waiting for empty Postgres..."
          for i in $(seq 1 60); do
            if docker exec "${postgres_name}" pg_isready -U geo -d geo >/dev/null 2>&1; then
              break
            fi
            if [ "${i}" -eq 60 ]; then
              docker logs "${postgres_name}" || true
              echo "::error::Postgres did not become ready"
              exit 1
            fi
            sleep 1
          done

          docker exec "${postgres_name}" createdb -U geo geo_monitoring

          printf '%s\n' "${REGISTRY_PASSWORD}" \
            | docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin

          echo "Pulling migration image..."
          docker pull "${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/migrate:${IMAGE_TAG}"
          docker tag "${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/migrate:${IMAGE_TAG}" "${migrate_image}"

          database_url="postgres://geo:geo_dev@${postgres_name}:5432/geo?sslmode=disable"
          monitoring_database_url="postgres://geo:geo_dev@${postgres_name}:5432/geo_monitoring?sslmode=disable"
          ops_database_url="postgres://geo:geo_dev@${postgres_name}:5432/geo?sslmode=disable&x-migrations-table=schema_migrations_ops"

          echo "Running tenant migrations on an empty database..."
          docker run --rm --network "${network_name}" "${migrate_image}" \
            -path /migrations \
            -database "${database_url}" \
            up

          echo "Running monitoring migrations on an empty database..."
          docker run --rm --network "${network_name}" "${migrate_image}" \
            -path /migrations_monitoring \
            -database "${monitoring_database_url}" \
            up

          echo "Running ops migrations on an empty database..."
          docker run --rm --network "${network_name}" "${migrate_image}" \
            -path /migrations_ops \
            -database "${ops_database_url}" \
            up

          echo "Migration preflight passed."

      - name: Prepare SSH known hosts
        run: |
          set -eu
          mkdir -p ~/.ssh
          chmod 700 ~/.ssh
          ssh-keyscan -p "${NAS_PORT}" "${NAS_HOST}" >> ~/.ssh/known_hosts

      - name: Upload release to NAS
        run: |
          set -eu

          remote="${NAS_USER}@${NAS_HOST}"
          ssh_opts="-p ${NAS_PORT} -o BatchMode=no -o StrictHostKeyChecking=yes"
          release_dir="${DEPLOY_BASE}/releases/${RELEASE_ID}"

          SSHPASS="${NAS_SSH_PASSWORD}" sshpass -e ssh ${ssh_opts} "${remote}" \
            "mkdir -p '${release_dir}' '${DEPLOY_BASE}/shared'"

          git archive --format=tar.gz HEAD \
            | SSHPASS="${NAS_SSH_PASSWORD}" sshpass -e ssh ${ssh_opts} "${remote}" \
                "tar -xzf - -C '${release_dir}'"

      - name: Deploy selected target with k3s
        run: |
          set -eu

          remote="${NAS_USER}@${NAS_HOST}"
          ssh_opts="-p ${NAS_PORT} -o BatchMode=no -o StrictHostKeyChecking=yes"
          sudo_password_b64="$(printf '%s' "${NAS_SSH_PASSWORD}" | base64 | tr -d '\n')"
          registry_password_b64="$(printf '%s' "${REGISTRY_PASSWORD:-}" | base64 | tr -d '\n')"

          {
            printf 'SUDO_PASSWORD_B64=%s\n' "${sudo_password_b64}"
            printf 'REGISTRY_PASSWORD_B64=%s\n' "${registry_password_b64}"
            cat <<'REMOTE_SCRIPT'
          set -eu

          release_dir="${DEPLOY_BASE}/releases/${RELEASE_ID}"
          lock_dir="${DEPLOY_BASE}/.deploy.lock"
          rendered_dir="${release_dir}/.k3s-rendered"

          if ! mkdir "${lock_dir}" 2>/dev/null; then
            echo "Another deployment is already running: ${lock_dir}"
            exit 1
          fi
          trap 'rmdir "${lock_dir}"' EXIT INT TERM

          SUDO_PASSWORD="$(printf '%s' "${SUDO_PASSWORD_B64}" | base64 -d 2>/dev/null || true)"
          REGISTRY_PASSWORD="$(printf '%s' "${REGISTRY_PASSWORD_B64}" | base64 -d 2>/dev/null || true)"
          unset SUDO_PASSWORD_B64 REGISTRY_PASSWORD_B64

          if command -v sudo >/dev/null 2>&1 && [ -n "${SUDO_PASSWORD}" ] && printf '%s\n' "${SUDO_PASSWORD}" | sudo -S -p "" true >/dev/null 2>&1; then
            SUDO_MODE="sudo"
          else
            SUDO_MODE="direct"
          fi

          sudo_cmd() {
            if [ "${SUDO_MODE}" = "sudo" ]; then
              printf '%s\n' "${SUDO_PASSWORD}" | sudo -S -p "" "$@"
            else
              "$@"
            fi
          }

          # k3s writes kubeconfig at /etc/rancher/k3s/k3s.yaml mode 0600 root:root.
          # Many NAS sudo setups elevate admin to a service user instead of root,
          # so `sudo kubectl` may still hit "permission denied". Copy kubeconfig
          # once into the SSH user's home and point KUBECONFIG at the copy.
          ensure_kubeconfig() {
            kubeconfig_target="${HOME:-/tmp}/.kube/config"
            mkdir -p "$(dirname "${kubeconfig_target}")"

            if [ -r /etc/rancher/k3s/k3s.yaml ]; then
              cp /etc/rancher/k3s/k3s.yaml "${kubeconfig_target}"
            else
              kubeconfig_tmp="${kubeconfig_target}.tmp.$$"
              if sudo_cmd cat /etc/rancher/k3s/k3s.yaml > "${kubeconfig_tmp}" 2>/dev/null && [ -s "${kubeconfig_tmp}" ]; then
                mv "${kubeconfig_tmp}" "${kubeconfig_target}"
              else
                rm -f "${kubeconfig_tmp}"
                echo "Cannot read /etc/rancher/k3s/k3s.yaml as $(whoami) or via sudo."
                echo "On the NAS, fix once with one of:"
                echo "  sudo chmod 0644 /etc/rancher/k3s/k3s.yaml"
                echo "  sudo install -m 0640 -g admin /etc/rancher/k3s/k3s.yaml /etc/rancher/k3s/k3s.yaml"
                echo "or restart k3s with --write-kubeconfig-mode 0644."
                exit 1
              fi
            fi
            chmod 600 "${kubeconfig_target}"
            export KUBECONFIG="${kubeconfig_target}"
          }
          ensure_kubeconfig

          kubectl_cmd() {
            if command -v kubectl >/dev/null 2>&1; then
              kubectl "$@"
            elif command -v k3s >/dev/null 2>&1; then
              k3s kubectl "$@"
            else
              echo "Cannot find kubectl or k3s on NAS."
              exit 1
            fi
          }

          image_name() {
            printf '%s/%s/%s/%s' \
              "${REGISTRY_HOST}" \
              "${REGISTRY_OWNER}" \
              "${REGISTRY_IMAGE_PREFIX}" \
              "$1"
          }

          image_ref() {
            printf '%s:%s' "$(image_name "$1")" "${IMAGE_TAG}"
          }

          ensure_config_aliases() {
            for yaml in config.yaml config.local.yaml ops-config.yaml ops-config.local.yaml; do
              yml="${yaml%.yaml}.yml"
              if [ ! -f "${release_dir}/deploy/${yaml}" ] && [ -f "${release_dir}/deploy/${yml}" ]; then
                cp "${release_dir}/deploy/${yml}" "${release_dir}/deploy/${yaml}"
                echo "Created ${yaml} from ${yml} for deployment mounts."
              fi
            done
            if [ ! -f "${release_dir}/deploy/prompts.yml" ] && [ -f "${release_dir}/deploy/prompts.yaml" ]; then
              cp "${release_dir}/deploy/prompts.yaml" "${release_dir}/deploy/prompts.yml"
              echo "Created prompts.yml from prompts.yaml for deployment mounts."
            fi
          }

          config_file_path() {
            base="$1"
            for file in "${release_dir}/deploy/${base}.yaml" "${release_dir}/deploy/${base}.yml"; do
              if [ -f "${file}" ]; then
                printf '%s' "${file}"
                return 0
              fi
            done
          }

          read_object_storage_provider() {
            provider="minio"
            for file in "$(config_file_path config)" "$(config_file_path config.local)"; do
              if [ -z "${file}" ] || [ ! -f "${file}" ]; then
                continue
              fi
              next="$(
                awk '
                  /^[[:space:]]*object_storage:[[:space:]]*$/ {
                    in_section=1
                    base=match($0, /[^[:space:]]/) - 1
                    next
                  }
                  in_section {
                    indent=match($0, /[^[:space:]]/) - 1
                    if ($0 ~ /^[[:space:]]*$/ || $0 ~ /^[[:space:]]*#/) {
                      next
                    }
                    if (indent <= base) {
                      in_section=0
                      next
                    }
                    if ($0 ~ /^[[:space:]]*provider[[:space:]]*:/) {
                      value=$0
                      sub(/^[[:space:]]*provider[[:space:]]*:[[:space:]]*/, "", value)
                      sub(/[[:space:]]+#.*$/, "", value)
                      gsub(/^["'\'']|["'\'']$/, "", value)
                      print value
                    }
                  }
                ' "${file}" | tail -1
              )"
              if [ -n "${next}" ]; then
                provider="${next}"
              fi
            done
            printf '%s' "$(printf '%s' "${provider}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
          }

          use_minio_for_object_storage() {
            case "$(read_object_storage_provider)" in
              ""|minio|mino)
                return 0
                ;;
              aliyun|aliyun_oss|aliyun-oss|oss)
                return 1
                ;;
              *)
                return 0
                ;;
            esac
          }

          sync_k3s_config() {
            mkdir -p "${release_dir}/deploy/k3s/config"
            for name in config ops-config prompts; do
              output="${name}.yaml"
              if [ "${name}" = "prompts" ]; then
                output="prompts.yml"
              fi
              source=""
              for candidate in "${release_dir}/deploy/${name}.yaml" "${release_dir}/deploy/${name}.yml"; do
                if [ -f "${candidate}" ]; then
                  source="${candidate}"
                  break
                fi
              done
              if [ -n "${source}" ]; then
                cp "${source}" "${release_dir}/deploy/k3s/config/${output}"
              fi
            done
          }

          echo "Sudo mode: ${SUDO_MODE}"
          kubectl_cmd version --client

          restart_k3s_service() {
            for unit in k3s k3s-server k3s-agent; do
              if sudo_cmd systemctl is-active "${unit}" >/dev/null 2>&1; then
                echo "Restarting systemd unit ${unit}..."
                sudo_cmd systemctl restart "${unit}"
                return 0
              fi
            done
            for svc in k3s k3s-server k3s-agent; do
              if [ -x "/etc/init.d/${svc}" ]; then
                echo "Restarting init.d service ${svc}..."
                sudo_cmd "/etc/init.d/${svc}" restart
                return 0
              fi
            done
            echo "Cannot find a k3s service unit to restart."
            echo "Manually restart k3s on the NAS, then re-run this workflow."
            return 1
          }

          wait_for_k3s_ready() {
            i=0
            while [ "${i}" -lt 60 ]; do
              if kubectl_cmd get --raw=/readyz >/dev/null 2>&1; then
                return 0
              fi
              i=$((i + 1))
              sleep 2
            done
            echo "k3s API did not return to ready within 120s."
            return 1
          }

          ensure_plain_http_registry() {
            if [ "${REGISTRY_PLAIN_HTTP}" != "true" ]; then
              return 0
            fi

            registry_file="/etc/rancher/k3s/registries.yaml"
            desired_endpoint="http://${REGISTRY_HOST}"

            if sudo_cmd test -f "${registry_file}" \
              && sudo_cmd grep -F "${desired_endpoint}" "${registry_file}" >/dev/null 2>&1; then
              return 0
            fi

            echo "Writing k3s plain-HTTP mirror config to ${registry_file}..."
            registry_yaml="$(printf 'mirrors:\n  "%s":\n    endpoint:\n      - "%s"\n' \
              "${REGISTRY_HOST}" "${desired_endpoint}")"

            if ! printf '%s' "${registry_yaml}" | sudo_cmd tee "${registry_file}" >/dev/null; then
              echo "Failed to write ${registry_file} via sudo."
              echo "Create it manually on the NAS with:"
              echo "${registry_yaml}"
              echo "and restart k3s, then re-run this workflow."
              exit 1
            fi

            if ! restart_k3s_service; then
              exit 1
            fi
            if ! wait_for_k3s_ready; then
              exit 1
            fi
          }

          ensure_namespace() {
            kubectl_cmd create namespace "${K3S_NAMESPACE}" --dry-run=client -o yaml \
              | kubectl_cmd apply -f -
          }

          ensure_registry_secret() {
            if [ -z "${REGISTRY_PASSWORD}" ]; then
              echo "Registry password was not provided; keeping existing k3s imagePullSecrets."
              return 0
            fi

            kubectl_cmd -n "${K3S_NAMESPACE}" create secret docker-registry gitea-registry \
              --docker-server="${REGISTRY_HOST}" \
              --docker-username="${REGISTRY_USERNAME}" \
              --docker-password="${REGISTRY_PASSWORD}" \
              --dry-run=client -o yaml \
              | kubectl_cmd -n "${K3S_NAMESPACE}" apply -f -

            kubectl_cmd -n "${K3S_NAMESPACE}" patch serviceaccount default \
              --type=merge \
              -p '{"imagePullSecrets":[{"name":"gitea-registry"}]}' >/dev/null
          }

          patch_kustomize_image() {
            old_name="$1"
            svc="$2"
            new_name="$(image_name "${svc}")"
            sed -i "\#name: ${old_name}#,\#newTag:# s#newName: .*#newName: ${new_name}#" "${rendered_dir}/kustomization.yaml"
            sed -i "\#name: ${old_name}#,\#newTag:# s#newTag: .*#newTag: ${IMAGE_TAG}#" "${rendered_dir}/kustomization.yaml"
          }

          apply_object_storage_kustomize_mode() {
            sync_k3s_config
            provider="$(read_object_storage_provider)"
            if use_minio_for_object_storage; then
              echo "object_storage.provider=${provider}; deploying MinIO."
              return 0
            fi

            echo "object_storage.provider=${provider}; skipping MinIO."
            sed -i '/^patches:[[:space:]]*$/,/^[^[:space:]-]/{/^patches:[[:space:]]*$/d;/^[^[:space:]-]/!d;}' "${rendered_dir}/kustomization.yaml"
            mkdir -p "${rendered_dir}/patches"
            cat > "${rendered_dir}/patches/object-storage-env.yaml" <<'EOF'
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: tenant-api
          spec:
            template:
              spec:
                containers:
                  - name: tenant-api
                    env:
                      - name: OBJECT_STORAGE_ACCESS_KEY
                        $patch: delete
                      - name: OBJECT_STORAGE_SECRET_KEY
                        $patch: delete
          ---
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: worker-generate
          spec:
            template:
              spec:
                containers:
                  - name: worker-generate
                    env:
                      - name: OBJECT_STORAGE_ACCESS_KEY
                        $patch: delete
                      - name: OBJECT_STORAGE_SECRET_KEY
                        $patch: delete
          ---
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: kol-assist-worker
          spec:
            template:
              spec:
                containers:
                  - name: kol-assist-worker
                    env:
                      - name: OBJECT_STORAGE_ACCESS_KEY
                        $patch: delete
                      - name: OBJECT_STORAGE_SECRET_KEY
                        $patch: delete
          ---
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: scheduler
          spec:
            template:
              spec:
                containers:
                  - name: scheduler
                    env:
                      - name: OBJECT_STORAGE_ACCESS_KEY
                        $patch: delete
                      - name: OBJECT_STORAGE_SECRET_KEY
                        $patch: delete
          EOF
            cat >> "${rendered_dir}/kustomization.yaml" <<'EOF'
          patches:
            - target:
                version: v1
                kind: Service
                name: minio
              patch: |-
                $patch: delete
                apiVersion: v1
                kind: Service
                metadata:
                  name: minio
            - target:
                group: apps
                version: v1
                kind: StatefulSet
                name: minio
              patch: |-
                $patch: delete
                apiVersion: apps/v1
                kind: StatefulSet
                metadata:
                  name: minio
            - target:
                group: batch
                version: v1
                kind: Job
                name: minio-init
              patch: |-
                $patch: delete
                apiVersion: batch/v1
                kind: Job
                metadata:
                  name: minio-init
            - path: patches/object-storage-env.yaml
          EOF
          }

          cleanup_minio_if_external_object_storage() {
            if use_minio_for_object_storage; then
              return 0
            fi
            kubectl_cmd -n "${K3S_NAMESPACE}" delete statefulset minio --ignore-not-found
            kubectl_cmd -n "${K3S_NAMESPACE}" delete service minio --ignore-not-found
            kubectl_cmd -n "${K3S_NAMESPACE}" delete job minio-init --ignore-not-found
            for svc in tenant-api worker-generate kol-assist-worker scheduler; do
              kubectl_cmd -n "${K3S_NAMESPACE}" set env "deployment/${svc}" OBJECT_STORAGE_ACCESS_KEY- OBJECT_STORAGE_SECRET_KEY- >/dev/null 2>&1 || true
            done
          }

          render_kustomize() {
            rm -rf "${rendered_dir}"
            mkdir -p "${rendered_dir}"
            cp -R "${release_dir}/deploy/k3s/." "${rendered_dir}/"

            if grep -q '^namespace:' "${rendered_dir}/kustomization.yaml"; then
              sed -i "s#^namespace:.*#namespace: ${K3S_NAMESPACE}#" "${rendered_dir}/kustomization.yaml"
            fi

            # Migrations are applied explicitly by this workflow after preflight.
            sed -i '/^[[:space:]]*- jobs.yaml[[:space:]]*$/d' "${rendered_dir}/kustomization.yaml"
            apply_object_storage_kustomize_mode

            patch_kustomize_image "geo-rankly/migrate" "migrate"
            patch_kustomize_image "geo-rankly/tenant-api" "tenant-api"
            patch_kustomize_image "geo-rankly/browser-fetch" "browser-fetch"
            patch_kustomize_image "geo-rankly/worker-generate" "worker-generate"
            patch_kustomize_image "geo-rankly/kol-assist-worker" "kol-assist-worker"
            patch_kustomize_image "geo-rankly/scheduler" "scheduler"
            patch_kustomize_image "geo-rankly/ops-api" "ops-api"
            patch_kustomize_image "geo-rankly/frontend" "frontend"
            patch_kustomize_image "geo-rankly/ops-web" "ops-web"
          }

          apply_rendered_kustomize() {
            render_kustomize
            kubectl_cmd apply -k "${rendered_dir}"
          }

          apply_rendered_infra_kustomize() {
            render_kustomize
            sed -i '/^[[:space:]]*- apps.yaml[[:space:]]*$/d' "${rendered_dir}/kustomization.yaml"
            kubectl_cmd apply -k "${rendered_dir}"
          }

          apply_config_map() {
            sync_k3s_config
            kubectl_cmd -n "${K3S_NAMESPACE}" create configmap geo-rankly-app-config \
              --from-file=config.yaml="${release_dir}/deploy/k3s/config/config.yaml" \
              --from-file=prompts.yml="${release_dir}/deploy/k3s/config/prompts.yml" \
              --from-file=ops-config.yaml="${release_dir}/deploy/k3s/config/ops-config.yaml" \
              --dry-run=client -o yaml \
              | kubectl_cmd -n "${K3S_NAMESPACE}" apply -f -
          }

          apply_jobs() {
            rendered_jobs="${rendered_dir}/jobs.yaml"
            mkdir -p "${rendered_dir}"
            cp "${release_dir}/deploy/k3s/jobs.yaml" "${rendered_jobs}"
            sed -i "s#image: geo-rankly/migrate:latest#image: $(image_ref migrate)#" "${rendered_jobs}"

            kubectl_cmd -n "${K3S_NAMESPACE}" delete job migrate minio-init --ignore-not-found
            if ! use_minio_for_object_storage; then
              awk '
                /^---[[:space:]]*$/ { doc=doc "\n" $0 "\n"; next }
                { doc=doc $0 "\n" }
                END {
                  n=split(doc, docs, /\n---\n/)
                  for (i=1; i<=n; i++) {
                    if (docs[i] ~ /kind:[[:space:]]*Job/ && docs[i] ~ /name:[[:space:]]*minio-init/) {
                      continue
                    }
                    if (docs[i] ~ /[^[:space:]]/) {
                      if (printed) print "---"
                      printf "%s", docs[i]
                      if (substr(docs[i], length(docs[i]), 1) != "\n") print ""
                      printed=1
                    }
                  }
                }
              ' "${rendered_jobs}" > "${rendered_jobs}.tmp"
              mv "${rendered_jobs}.tmp" "${rendered_jobs}"
            fi
            kubectl_cmd -n "${K3S_NAMESPACE}" apply -f "${rendered_jobs}"
          }

          run_migrations() {
            echo "Running k3s migration job..."
            apply_jobs
            kubectl_cmd -n "${K3S_NAMESPACE}" wait --for=condition=complete --timeout=900s job/migrate
            kubectl_cmd -n "${K3S_NAMESPACE}" logs job/migrate --tail=120
          }

          rollout_deployments() {
            if [ -z "${DEPLOYMENT_LIST}" ]; then
              return 0
            fi

            # Bootstrap missing deployments from kustomize so single-target
            # deploys work on a fresh cluster instead of failing with
            # "deployments.apps \"frontend\" not found".
            missing=""
            for svc in ${DEPLOYMENT_LIST}; do
              if ! kubectl_cmd -n "${K3S_NAMESPACE}" get "deployment/${svc}" >/dev/null 2>&1; then
                missing="${missing} ${svc}"
              fi
            done
            if [ -n "${missing}" ]; then
              echo "Bootstrapping missing deployment(s):${missing}"
              apply_rendered_kustomize
            fi

            for svc in ${DEPLOYMENT_LIST}; do
              kubectl_cmd -n "${K3S_NAMESPACE}" set image "deployment/${svc}" "${svc}=$(image_ref "${svc}")"
            done
            for svc in ${DEPLOYMENT_LIST}; do
              kubectl_cmd -n "${K3S_NAMESPACE}" rollout status "deployment/${svc}" --timeout=600s
            done
          }

          rollout_all_app_deployments() {
            for svc in tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web; do
              if kubectl_cmd -n "${K3S_NAMESPACE}" get "deployment/${svc}" >/dev/null 2>&1; then
                kubectl_cmd -n "${K3S_NAMESPACE}" rollout status "deployment/${svc}" --timeout=600s
              fi
            done
          }

          restart_config_consumers() {
            for svc in tenant-api worker-generate kol-assist-worker scheduler ops-api; do
              if kubectl_cmd -n "${K3S_NAMESPACE}" get "deployment/${svc}" >/dev/null 2>&1; then
                kubectl_cmd -n "${K3S_NAMESPACE}" rollout restart "deployment/${svc}"
                kubectl_cmd -n "${K3S_NAMESPACE}" rollout status "deployment/${svc}" --timeout=600s
              fi
            done
          }

          apply_manifest() {
            input="${MANIFEST_PATH}"
            input="${input#./}"
            input="${input#deploy/k3s/}"
            input="${input%/}"

            case "${input}" in
              ""|"."|"kustomization.yaml"|"kustomization.yml")
                apply_rendered_kustomize
                rollout_all_app_deployments
                return 0
                ;;
              /*|*..*|*" "*)
                echo "Invalid manifest path: ${MANIFEST_PATH}"
                exit 1
                ;;
            esac

            full_path="${release_dir}/deploy/k3s/${input}"
            if [ ! -f "${full_path}" ]; then
              echo "Manifest file not found: ${full_path}"
              exit 1
            fi

            case "${input}" in
              config/*)
                echo "Config source changed; updating generated ConfigMap and restarting config consumers."
                apply_config_map
                restart_config_consumers
                ;;
              apps.yaml|apps.yml)
                echo "Apps manifest changed; applying rendered kustomize with Gitea Registry images."
                apply_rendered_kustomize
                rollout_all_app_deployments
                ;;
              secrets.yaml|secrets.yml)
                kubectl_cmd -n "${K3S_NAMESPACE}" apply -f "${full_path}"
                restart_config_consumers
                ;;
              jobs.yaml|jobs.yml)
                apply_jobs
                ;;
              namespace.yaml|namespace.yml)
                ensure_namespace
                ;;
              *.yaml|*.yml)
                kubectl_cmd -n "${K3S_NAMESPACE}" apply -f "${full_path}"
                ;;
              *)
                echo "manifest_path must point to a .yaml/.yml file under deploy/k3s"
                exit 1
                ;;
            esac
          }

          if [ -n "${IMAGE_LIST}" ]; then
            ensure_plain_http_registry
          fi
          ensure_config_aliases
          ensure_namespace
          cleanup_minio_if_external_object_storage
          if [ -n "${IMAGE_LIST}" ]; then
            ensure_registry_secret
          else
            echo "No application images selected; skipping registry pull-secret setup."
          fi

          if [ "${APPLY_KUSTOMIZE}" = "true" ]; then
            if [ "${RUN_MIGRATIONS}" = "true" ]; then
              echo "Applying k3s infrastructure before migrations..."
              apply_rendered_infra_kustomize
            else
              echo "Applying full kustomize deployment with Gitea Registry images..."
              apply_rendered_kustomize
            fi
          fi

          if [ "${SELECTED_TARGET}" = "manifest" ]; then
            echo "Applying selected manifest: ${MANIFEST_PATH}"
            apply_manifest
          fi

          if [ "${RUN_MIGRATIONS}" = "true" ]; then
            run_migrations
          fi

          if [ "${APPLY_KUSTOMIZE}" = "true" ] && [ "${RUN_MIGRATIONS}" = "true" ]; then
            echo "Applying app deployments after migrations..."
            apply_rendered_kustomize
          fi

          rollout_deployments

          ln -sfn "${release_dir}" "${DEPLOY_BASE}/current"

          kubectl_cmd -n "${K3S_NAMESPACE}" get pods -o wide

          if [ "${KEEP_RELEASES}" -gt 0 ] 2>/dev/null; then
            find "${DEPLOY_BASE}/releases" -mindepth 1 -maxdepth 1 -type d \
              | sort -r \
              | awk "NR>${KEEP_RELEASES}" \
              | xargs -r rm -rf
          else
            echo "Release cleanup disabled for k3s deployment."
          fi
          REMOTE_SCRIPT
          } | SSHPASS="${NAS_SSH_PASSWORD}" sshpass -e ssh ${ssh_opts} "${remote}" \
            "DEPLOY_BASE='${DEPLOY_BASE}' RELEASE_ID='${RELEASE_ID}' IMAGE_TAG='${IMAGE_TAG}' IMAGE_LIST='${IMAGE_LIST}' DEPLOYMENT_LIST='${DEPLOYMENT_LIST}' APPLY_KUSTOMIZE='${APPLY_KUSTOMIZE}' RUN_MIGRATIONS='${RUN_MIGRATIONS}' SELECTED_TARGET='${SELECTED_TARGET}' MANIFEST_PATH='${MANIFEST_PATH}' K3S_NAMESPACE='${K3S_NAMESPACE}' KEEP_RELEASES='${KEEP_RELEASES}' REGISTRY_HOST='${REGISTRY_HOST}' REGISTRY_OWNER='${REGISTRY_OWNER}' REGISTRY_IMAGE_PREFIX='${REGISTRY_IMAGE_PREFIX}' REGISTRY_USERNAME='${REGISTRY_USERNAME}' REGISTRY_PLAIN_HTTP='${REGISTRY_PLAIN_HTTP}' sh -se"

      - name: Deployment summary
        run: |
          summary_file="${GITHUB_STEP_SUMMARY:-/tmp/nas-k3s-deploy-summary.md}"
          {
            echo "### NAS k3s deployment"
            echo ""
            echo "- Host: ${NAS_HOST}:${NAS_PORT}"
            echo "- Deploy base: ${DEPLOY_BASE}"
            echo "- Namespace: ${K3S_NAMESPACE}"
            echo "- Release: ${RELEASE_ID}"
            echo "- Image tag: ${IMAGE_TAG}"
            echo "- Selected target: ${SELECTED_TARGET}"
            echo "- Manifest path: ${MANIFEST_PATH:-}"
            echo "- Gitea Registry: ${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/<service>:${IMAGE_TAG}"
            echo "- Plain HTTP registry: ${REGISTRY_PLAIN_HTTP}"
            echo "- Images: ${IMAGE_LIST:-none}"
            echo "- Deployments: ${DEPLOYMENT_LIST:-none}"
            echo "- Run migrations: ${RUN_MIGRATIONS}"
            echo ""
            echo "Useful NAS commands:"
            echo ""
            echo '```sh'
            echo "kubectl -n ${K3S_NAMESPACE} get pods"
            echo "kubectl -n ${K3S_NAMESPACE} rollout status deploy/frontend"
            echo "kubectl -n ${K3S_NAMESPACE} logs job/migrate"
            echo '```'
          } >> "${summary_file}"