fix(ci): prune old registry package versions
This commit is contained in:
@@ -18,14 +18,16 @@ REGISTRY_SCHEME="${REGISTRY_SCHEME:-}"
|
||||
IMAGE_TAG="${IMAGE_TAG:-}"
|
||||
IMAGE_LIST="${IMAGE_LIST:-tenant-api browser-fetch ops-api worker-generate kol-assist-worker scheduler frontend ops-web migrate}"
|
||||
DELETE_OLD_IMAGE_TAGS="${DELETE_OLD_IMAGE_TAGS:-true}"
|
||||
KEEP_IMAGE_TAGS="${KEEP_IMAGE_TAGS:-2}"
|
||||
GITEA_PACKAGE_PAGE_LIMIT="${GITEA_PACKAGE_PAGE_LIMIT:-100}"
|
||||
PUSH_LATEST="${PUSH_LATEST:-true}"
|
||||
FORCE_BUILD="${FORCE_BUILD:-false}"
|
||||
TARGET_PLATFORM="${TARGET_PLATFORM:-}"
|
||||
|
||||
case "${MODE}" in
|
||||
ensure|verify) ;;
|
||||
ensure|verify|cleanup) ;;
|
||||
*)
|
||||
echo "Usage: $0 [ensure|verify]" >&2
|
||||
echo "Usage: $0 [ensure|verify|cleanup]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -70,11 +72,34 @@ case "${REGISTRY_PLAIN_HTTP}" in
|
||||
;;
|
||||
esac
|
||||
|
||||
case "${KEEP_IMAGE_TAGS}" in
|
||||
''|*[!0-9]*)
|
||||
echo "KEEP_IMAGE_TAGS must be a non-negative integer." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "${GITEA_PACKAGE_PAGE_LIMIT}" in
|
||||
''|*[!0-9]*)
|
||||
echo "GITEA_PACKAGE_PAGE_LIMIT must be a positive integer." >&2
|
||||
exit 1
|
||||
;;
|
||||
0)
|
||||
echo "GITEA_PACKAGE_PAGE_LIMIT must be greater than 0." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
echo "curl is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "python3 is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${MODE}" == "ensure" ]]; then
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "docker is required." >&2
|
||||
@@ -116,10 +141,78 @@ registry_api_base() {
|
||||
"$(registry_repo_path "${svc}")"
|
||||
}
|
||||
|
||||
gitea_api_base() {
|
||||
printf '%s://%s/api/v1' "${REGISTRY_SCHEME}" "${REGISTRY_HOST}"
|
||||
}
|
||||
|
||||
gitea_package_base_url() {
|
||||
local svc="$1" encoded_owner encoded_name
|
||||
encoded_owner="$(url_encode_segment "${REGISTRY_OWNER}")"
|
||||
encoded_name="$(url_encode_segment "$(package_name "${svc}")")"
|
||||
printf '%s/packages/%s/container/%s' \
|
||||
"$(gitea_api_base)" \
|
||||
"${encoded_owner}" \
|
||||
"${encoded_name}"
|
||||
}
|
||||
|
||||
gitea_package_list_url() {
|
||||
local svc="$1" page="$2" encoded_owner encoded_query
|
||||
encoded_owner="$(url_encode_segment "${REGISTRY_OWNER}")"
|
||||
encoded_query="$(url_encode_segment "$(package_name "${svc}")")"
|
||||
printf '%s/packages/%s/?type=container&q=%s&page=%s&limit=%s' \
|
||||
"$(gitea_api_base)" \
|
||||
"${encoded_owner}" \
|
||||
"${encoded_query}" \
|
||||
"${page}" \
|
||||
"${GITEA_PACKAGE_PAGE_LIMIT}"
|
||||
}
|
||||
|
||||
package_name() {
|
||||
local svc="$1"
|
||||
printf '%s/%s' "${REGISTRY_IMAGE_PREFIX}" "${svc}"
|
||||
}
|
||||
|
||||
url_encode_segment() {
|
||||
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"
|
||||
}
|
||||
|
||||
curl_registry() {
|
||||
curl -fsS -u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" "$@"
|
||||
}
|
||||
|
||||
curl_gitea_api_json() {
|
||||
local url="$1" body_file status body
|
||||
body_file="$(mktemp)"
|
||||
status="$(
|
||||
curl -sS \
|
||||
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
||||
-o "${body_file}" \
|
||||
-w '%{http_code}' \
|
||||
--path-as-is \
|
||||
"${url}" || true
|
||||
)"
|
||||
body="$(<"${body_file}")"
|
||||
rm -f "${body_file}"
|
||||
|
||||
case "${status}" in
|
||||
200)
|
||||
printf '%s' "${body}"
|
||||
;;
|
||||
401|403)
|
||||
echo "Gitea package API authentication failed." >&2
|
||||
exit 1
|
||||
;;
|
||||
404)
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
echo "Gitea package API request failed; HTTP ${status}. URL: ${url}" >&2
|
||||
[[ -z "${body}" ]] || echo "${body}" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
manifest_digest() {
|
||||
local svc="$1" tag="$2" base headers status digest
|
||||
base="$(registry_api_base "${svc}")"
|
||||
@@ -163,23 +256,8 @@ tag_exists() {
|
||||
manifest_digest "${svc}" "${tag}" >/dev/null
|
||||
}
|
||||
|
||||
list_tags() {
|
||||
local svc="$1" base json
|
||||
base="$(registry_api_base "${svc}")"
|
||||
json="$(curl_registry "${base}/tags/list" 2>/dev/null || true)"
|
||||
if [[ -z "${json}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "${json}" \
|
||||
| tr -d '\n' \
|
||||
| sed -n 's/.*"tags":[[]\([^]]*\)[]].*/\1/p' \
|
||||
| tr ',' '\n' \
|
||||
| tr -d '" ' \
|
||||
| sed '/^$/d'
|
||||
}
|
||||
|
||||
delete_manifest_digest() {
|
||||
local svc="$1" digest="$2" base status
|
||||
delete_manifest_reference() {
|
||||
local svc="$1" reference="$2" base status
|
||||
base="$(registry_api_base "${svc}")"
|
||||
status="$(
|
||||
curl -sS \
|
||||
@@ -187,7 +265,7 @@ delete_manifest_digest() {
|
||||
-X DELETE \
|
||||
-o /dev/null \
|
||||
-w '%{http_code}' \
|
||||
"${base}/manifests/${digest}" || true
|
||||
"${base}/manifests/${reference}" || true
|
||||
)"
|
||||
|
||||
case "${status}" in
|
||||
@@ -195,48 +273,167 @@ delete_manifest_digest() {
|
||||
return 0
|
||||
;;
|
||||
405)
|
||||
echo "Registry does not allow deleting manifests for ${svc}; skipping cleanup." >&2
|
||||
return 0
|
||||
echo "Registry does not allow deleting ${svc}:${reference}; cleanup failed." >&2
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
echo "Failed to delete old manifest ${svc}@${digest}; HTTP ${status}. Continuing." >&2
|
||||
return 0
|
||||
echo "Failed to delete old image tag ${svc}:${reference}; HTTP ${status}." >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
delete_gitea_package_version() {
|
||||
local svc="$1" tag="$2" encoded_tag url status
|
||||
encoded_tag="$(url_encode_segment "${tag}")"
|
||||
url="$(gitea_package_base_url "${svc}")/${encoded_tag}"
|
||||
|
||||
status="$(
|
||||
curl -sS \
|
||||
-u "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" \
|
||||
-X DELETE \
|
||||
-o /dev/null \
|
||||
-w '%{http_code}' \
|
||||
--path-as-is \
|
||||
"${url}" || true
|
||||
)"
|
||||
|
||||
case "${status}" in
|
||||
200|202|204)
|
||||
return 0
|
||||
;;
|
||||
404)
|
||||
return 1
|
||||
;;
|
||||
401|403)
|
||||
echo "Gitea package API authentication failed while deleting ${svc}:${tag}." >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "Gitea package API could not delete ${svc}:${tag}; HTTP ${status}." >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
delete_old_hash_tag() {
|
||||
local svc="$1" tag="$2"
|
||||
|
||||
if delete_gitea_package_version "${svc}" "${tag}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Falling back to registry tag delete for ${svc}:${tag}"
|
||||
delete_manifest_reference "${svc}" "${tag}"
|
||||
}
|
||||
|
||||
list_gitea_hash_versions() {
|
||||
local svc="$1" page=1 json count expected_name
|
||||
expected_name="$(package_name "${svc}")"
|
||||
|
||||
while true; do
|
||||
json="$(
|
||||
curl_gitea_api_json "$(gitea_package_list_url "${svc}" "${page}")"
|
||||
)" || return 1
|
||||
|
||||
count="$(
|
||||
printf '%s' "${json}" \
|
||||
| python3 -c 'import json, sys; data = json.load(sys.stdin); print(len(data) if isinstance(data, list) else 0)'
|
||||
)"
|
||||
|
||||
printf '%s' "${json}" \
|
||||
| python3 -c '
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
expected_name = sys.argv[1].lower()
|
||||
data = json.load(sys.stdin)
|
||||
if not isinstance(data, list):
|
||||
raise SystemExit("Gitea package API response was not a list")
|
||||
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get("type") not in ("", "container"):
|
||||
continue
|
||||
if (item.get("name") or "").lower() != expected_name:
|
||||
continue
|
||||
version = item.get("version") or ""
|
||||
if not re.fullmatch(r"[0-9a-f]{8}", version):
|
||||
continue
|
||||
created_at = item.get("created_at") or ""
|
||||
print(f"{created_at}\t{version}")
|
||||
' "${expected_name}"
|
||||
|
||||
if (( count < GITEA_PACKAGE_PAGE_LIMIT )); then
|
||||
break
|
||||
fi
|
||||
page=$((page + 1))
|
||||
done | sort -r -k1,1 -k2,2
|
||||
}
|
||||
|
||||
cleanup_old_hash_tags() {
|
||||
local svc="$1" tag digest current_digest latest_digest
|
||||
local svc="$1" kept=0 hash_tags_to_keep line tag versions_file keep_file
|
||||
|
||||
if [[ "${DELETE_OLD_IMAGE_TAGS}" != "true" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
current_digest="$(manifest_digest "${svc}" "${IMAGE_TAG}" 2>/dev/null || true)"
|
||||
latest_digest="$(manifest_digest "${svc}" "latest" 2>/dev/null || true)"
|
||||
hash_tags_to_keep="${KEEP_IMAGE_TAGS}"
|
||||
if [[ "${PUSH_LATEST}" == "true" && "${hash_tags_to_keep}" -gt 0 ]]; then
|
||||
hash_tags_to_keep=$((hash_tags_to_keep - 1))
|
||||
fi
|
||||
|
||||
while IFS= read -r tag; do
|
||||
versions_file="$(mktemp)"
|
||||
keep_file="$(mktemp)"
|
||||
if ! list_gitea_hash_versions "${svc}" >"${versions_file}"; then
|
||||
rm -f "${versions_file}" "${keep_file}"
|
||||
echo "Failed to list Gitea package versions for ${svc}; cleanup aborted." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if (( hash_tags_to_keep > 0 )); then
|
||||
while IFS= read -r line; do
|
||||
tag="${line#*$'\t'}"
|
||||
if [[ "${tag}" == "${IMAGE_TAG}" ]]; then
|
||||
printf '%s\n' "${tag}" >"${keep_file}"
|
||||
break
|
||||
fi
|
||||
done <"${versions_file}"
|
||||
|
||||
while IFS= read -r line; do
|
||||
tag="${line#*$'\t'}"
|
||||
[[ -n "${tag}" ]] || continue
|
||||
if grep -Fxq "${tag}" "${keep_file}"; then
|
||||
continue
|
||||
fi
|
||||
kept="$(wc -l <"${keep_file}" | tr -d ' ')"
|
||||
if (( kept >= hash_tags_to_keep )); then
|
||||
break
|
||||
fi
|
||||
printf '%s\n' "${tag}" >>"${keep_file}"
|
||||
done <"${versions_file}"
|
||||
fi
|
||||
|
||||
kept=0
|
||||
while IFS= read -r line; do
|
||||
tag="${line#*$'\t'}"
|
||||
[[ -n "${tag}" ]] || continue
|
||||
[[ "${tag}" != "${IMAGE_TAG}" ]] || continue
|
||||
[[ "${tag}" != "latest" ]] || continue
|
||||
if [[ ! "${tag}" =~ ^[0-9a-f]{8}$ ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
digest="$(manifest_digest "${svc}" "${tag}" 2>/dev/null || true)"
|
||||
if [[ -z "${digest}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ -n "${current_digest}" && "${digest}" == "${current_digest}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ -n "${latest_digest}" && "${digest}" == "${latest_digest}" ]]; then
|
||||
if grep -Fxq "${tag}" "${keep_file}"; then
|
||||
kept=$((kept + 1))
|
||||
echo "Keeping image tag ${svc}:${tag} (${kept}/${hash_tags_to_keep} hash tags, total target ${KEEP_IMAGE_TAGS} including latest)"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Deleting old image tag ${svc}:${tag}"
|
||||
delete_manifest_digest "${svc}" "${digest}"
|
||||
done < <(list_tags "${svc}")
|
||||
if ! delete_old_hash_tag "${svc}" "${tag}"; then
|
||||
rm -f "${versions_file}" "${keep_file}"
|
||||
return 1
|
||||
fi
|
||||
done <"${versions_file}"
|
||||
rm -f "${versions_file}" "${keep_file}"
|
||||
}
|
||||
|
||||
platform_args=()
|
||||
@@ -351,11 +548,14 @@ verify_image() {
|
||||
echo "Gitea Registry: ${REGISTRY_SCHEME}://${REGISTRY_HOST}/${REGISTRY_OWNER}/${REGISTRY_IMAGE_PREFIX}/<service>"
|
||||
echo "Image tag: ${IMAGE_TAG}"
|
||||
echo "Image list: ${IMAGE_LIST}"
|
||||
echo "Keeping image versions per service: ${KEEP_IMAGE_TAGS} (latest counts as one when PUSH_LATEST=true)"
|
||||
|
||||
missing=0
|
||||
for svc in ${IMAGE_LIST}; do
|
||||
if [[ "${MODE}" == "verify" ]]; then
|
||||
verify_image "${svc}" || missing=1
|
||||
elif [[ "${MODE}" == "cleanup" ]]; then
|
||||
cleanup_old_hash_tags "${svc}"
|
||||
else
|
||||
ensure_image "${svc}"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user