#!/usr/bin/env bash # Registry retention for this repo's OWN images (toolbox, agent-runner, conformance-runner): # keep what is deployed and what is recent, delete the rest. DRY RUN unless --apply. # # ./infra/gitea/registry-retention.sh # show what would be deleted # ./infra/gitea/registry-retention.sh --apply # delete it # KEEP=15 ./infra/gitea/registry-retention.sh # keep the 15 newest sha- tags (default 10) # # RUNNING IT IS AN OPERATOR ACTION. It deletes registry content, so it is not wired into CI and # no agent runs it with --apply. # # WHY A SCRIPT, AND NOT GITEA'S BUILT-IN "CLEANUP RULE" — measured against Gitea 1.24.3's source: # # 1. There is NO API for cleanup rules. routers/api/v1/api.go has no such route; the rules live # only behind the web UI's session+CSRF forms (routers/web/web.go, org settings # `/packages/rules/add`). A rule therefore cannot be created, read or drift-checked as code — # which is also why ci/gitea-settings.sh cannot assert one exists. # 2. A built-in rule would DELETE WHAT IS DEPLOYED. Every reference to these images is # digest-only (`@sha256:...`, required by admission). Gitea stores a manifest as a # file OF THE TAG VERSION it was pushed under (routers/api/packages/container/manifest.go # createPackageAndVersion), and resolves a by-digest pull by searching those files # (models/packages/container/search.go). So deleting the tag a digest was pushed under makes # that digest unpullable — MANIFEST_UNKNOWN, i.e. ImagePullBackOff on the next reschedule. # The built-in rule keeps "the newest N" and a name pattern, and knows nothing about which # digest git currently pins. Renovate's bump PRs can lag, so "newest N" is not a guarantee. # # WHAT IS KEPT, per image: # * the `main` and `latest` tags; # * the KEEP newest `sha-` tags (by creation time); # * EVERY tag whose manifest digest is referenced anywhere in the tracked tree at REF # (default origin/main) — i.e. whatever Flux is deploying, and the toolbox digest # agent-runner builds FROM; # * every version whose NAME is a digest (`sha256:...`): those are index children, and Gitea's # own cleanup handles them (services/packages/container/cleanup.go ShouldBeSkipped). # Everything else is deleted. Gitea's `cleanup_packages` cron then reclaims the unreferenced # blobs (default: daily, blobs older than 24h) — space comes back the day after, not instantly. # # THE STORAGE IT PROTECTS. Packages live on the SAME 10Gi Longhorn PVC as every repository and # Gitea's database (catalog/gitea.yaml). The toolbox is ~1.4 GB uncompressed and every main push # that touches its inputs adds a complete, non-reproducible copy. Unpruned, the registry is what # fills the volume the git host lives on. # # CREDENTIAL: the one `git` has stored for the host (as the other scripts here). It needs # `read:package` to list and `write:package` to --apply, and the account needs package write on # the org. The secret is never printed. set -euo pipefail HOST=gitea.home.keithjackson.dev; OWNER=Home-Network; NS=home-network IMAGES=(toolbox agent-runner conformance-runner) KEEP="${KEEP:-10}" REF="${REF:-origin/main}" APPLY=0 for a in "$@"; do case "$a" in --apply) APPLY=1 ;; -h|--help) sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "unknown argument: $a" >&2; exit 2 ;; esac done [[ "$KEEP" =~ ^[0-9]+$ ]] && [ "$KEEP" -ge 1 ] || { echo "KEEP must be a positive integer" >&2; exit 2; } command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" git -C "$REPO_ROOT" fetch -q origin main 2>/dev/null || echo " (could not fetch origin/main; using the local ${REF})" >&2 git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" >/dev/null || { echo "no such ref: ${REF}" >&2; exit 1; } creds="$(printf 'protocol=https\nhost=%s\n\n' "$HOST" | git credential fill)" U="$(sed -n 's/^username=//p' <<<"$creds")"; P="$(sed -n 's/^password=//p' <<<"$creds")" [ -n "$U" ] && [ -n "$P" ] || { echo "no stored credential for $HOST" >&2; exit 1; } echo "authenticating as '${U}' (secret not shown)" api() { curl -sS --config <(printf 'user = "%s:%s"\n' "$U" "$P") "$@"; } REG_TOKEN="$(api "https://${HOST}/v2/token?service=container_registry" | jq -r '.token // empty')" [ -n "$REG_TOKEN" ] || { echo "registry token exchange failed (does the credential have read:package?)" >&2; exit 1; } manifest_digest() { # -> digest, or nothing curl -sS -I --config <(printf 'header = "Authorization: Bearer %s"\n' "$REG_TOKEN") \ -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ -H 'Accept: application/vnd.oci.image.index.v1+json' \ -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ "https://${HOST}/v2/${NS}/$1/manifests/$2" \ | tr -d '\r' | awk -F': ' 'tolower($1)=="docker-content-digest"{print $2}' | tail -1 } TOTAL_DEL=0 for img in "${IMAGES[@]}"; do echo echo "==> ${NS}/${img}" # Digests git pins at REF. The marker convention puts the digest on the line after # `# renovate-digest: image=`; the plain `@sha256:` grep catches any other use. repo="${HOST}/${NS}/${img}" pinned="$( { git -C "$REPO_ROOT" grep -h -A1 -F "renovate-digest: image=${repo} " "$REF" -- . 2>/dev/null || true git -C "$REPO_ROOT" grep -h -F "${repo}@sha256:" "$REF" -- . 2>/dev/null || true; } \ | grep -oE 'sha256:[0-9a-f]{64}' | grep -v 'sha256:0\{64\}' | sort -u || true)" echo " pinned at ${REF}: $(wc -w <<<"$pinned") digest(s)" versions="[]"; page=1 while :; do chunk="$(api "https://${HOST}/api/v1/packages/${OWNER}?type=container&q=${img}&limit=50&page=${page}")" jq -e 'type=="array"' >/dev/null <<<"$chunk" || { echo " package list failed: $(head -c 200 <<<"$chunk")" >&2; exit 1; } [ "$(jq length <<<"$chunk")" -gt 0 ] || break versions="$(jq -c --argjson a "$versions" --arg n "$img" '$a + [ .[] | select(.name==$n) | {version, created_at} ]' <<<"$chunk")" page=$((page + 1)) done echo " versions: $(jq length <<<"$versions")" keep_sha="$(jq -r '[ .[] | select(.version|startswith("sha-")) ] | sort_by(.created_at) | reverse | .[:'"$KEEP"'][] .version' <<<"$versions")" while IFS= read -r v; do [ -n "$v" ] || continue case "$v" in main|latest) echo " keep ${v} (moving tag)"; continue ;; sha256:*) echo " keep ${v} (digest-named version; Gitea's cleanup owns those)"; continue ;; esac if grep -qxF "$v" <<<"$keep_sha"; then echo " keep ${v} (among the ${KEEP} newest)"; continue; fi d="$(manifest_digest "$img" "$v")" if [ -z "$d" ]; then echo " keep ${v} (could not read its digest — never delete what cannot be checked)"; continue; fi if grep -qxF "$d" <<<"$pinned"; then echo " keep ${v} (${d} is pinned at ${REF})"; continue; fi if [ "$APPLY" -eq 1 ]; then code="$(api -o /dev/null -w '%{http_code}' -X DELETE \ "https://${HOST}/api/v1/packages/${OWNER}/container/${img}/$(jq -rn --arg v "$v" '$v|@uri')")" if [ "$code" = 204 ]; then echo " DELETED ${v} (${d})"; TOTAL_DEL=$((TOTAL_DEL + 1)); else echo " FAILED ${v} HTTP ${code}" >&2; fi else echo " would delete ${v} (${d})"; TOTAL_DEL=$((TOTAL_DEL + 1)) fi done < <(jq -r '.[].version' <<<"$versions") done echo if [ "$APPLY" -eq 1 ]; then echo "deleted ${TOTAL_DEL} version(s). Blobs are reclaimed by Gitea's cleanup_packages cron." else echo "DRY RUN: ${TOTAL_DEL} version(s) would be deleted. Re-run with --apply to delete them."; fi