feat(images): build, verify and publish the three images in CI; Renovate carries the digests

Operator rule 2026-10-29: image creation, hosting and lifecycle belong to CI/CD, never to a workstation build/push. Nothing had ever been pushed to the Gitea registry. - .gitea/workflows/images.yaml + scripts/images.sh: PRs build+verify (no credential); main builds, verifies and pushes toolbox, agent-runner (FROM toolbox) and conformance-runner as sha-<commit> + main, reading the digest back from the registry. Only changed images build. - Each image's own build-image.sh does the build and in-image verification, now engine-neutral (DinD: no podman-only calls, probe over stdin not a bind mount). New toolbox build-image.sh. - Digest references carry `# renovate-digest:` markers; renovate.json5 gains a custom.regex manager for them. The worker Job template is deliberately untracked (it would loop). - renovate.yaml pins renovate-version 44.167.0: the action's default (39.264.0) rejects the current config's managerFilePatterns, so every run would have died at CONFIG_VALIDATION. - Push credential: REGISTRY_PUSH_TOKEN secret; the Actions token has no package scope in Gitea 1.24.3 (container.go Authenticate). Retention: infra/gitea/registry-retention.sh. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-29 21:34:06 -05:00
1 parent 1301af9a4c
commit 0da34ddbc6
@@ -1,26 +1,36 @@
 #!/usr/bin/env bash
 ###############################################################################
-# infra/agent/build-image.sh — build the pinned `agent-runner` image and print its digest.
+# infra/agent/build-image.sh — build the pinned `agent-runner` image and prove it works.
 #
-#   ./build-image.sh                 # build + verify + print the digest and push command
+#   ./build-image.sh                 # build + verify + print the image id
 #   ./build-image.sh --no-verify     # build only (skip the in-image proof)
 #   ENGINE=docker ./build-image.sh   # build with docker instead of podman
+#   AGENT_BASE_IMAGE=home-k3s-dev:local ./build-image.sh
+#                                    # build on a LOCAL toolbox instead of the pinned one
+#                                    #   (`infra/devcontainer/dev.sh build` makes that tag)
+#   IMAGE_TAG=foo ./build-image.sh   # override the local tag (CI uses this)
 #
-# THIS SCRIPT NEVER PUSHES. Publishing is an operator action: the registry credential is not
-# an agent's to hold, and the pushed digest is what a fail-closed admission policy then
-# trusts. So this builds locally, proves the toolchain and both entrypoints work INSIDE the
-# image as the non-root uid with the read-only root filesystem the policy forces, and prints
-# the exact push line for a human.
+# THIS SCRIPT NEVER PUSHES — and nobody pushes this image from a workstation either.
+# Publishing is done by CI: .gitea/workflows/images.yaml calls this script, and on a push to
+# `main` it pushes what this script built AND verified to the Gitea registry, records the
+# registry-returned digest in the job summary, and Renovate then opens the pull request that
+# moves platform/pm-agent/ onto that digest. So this script's job is to build, and to prove
+# the toolchain and both entrypoints work INSIDE the image as the non-root uid with the
+# read-only root filesystem the admission policy forces — the same proof locally and in CI.
 #
-# VERSIONS ARE NOT DEFINED HERE. infra/agent/versions.env holds the four pins this image
-# adds (base image, node, Claude Code CLI, model) and infra/devcontainer/versions.env holds
-# the platform toolchain the base already carries. Nothing in this file invents a version.
+# VERSIONS ARE NOT DEFINED HERE. infra/agent/versions.env holds the pins this image adds
+# (base image, node, Claude Code CLI, model) and infra/devcontainer/versions.env holds the
+# platform toolchain the base already carries. Nothing in this file invents a version.
 #
-# THE DIGEST CAVEAT — read it before pasting a digest into anything. The digest printed here
-# is the LOCAL manifest digest. It is normally identical to the registry digest, but that is
-# a property of the push rather than a guarantee, so the push command below uses
-# `--digestfile`: the digest the REGISTRY returns is the authoritative one, and that is the
-# one that goes into platform/pm-agent/configmap.yaml.
+# THE BASE. AGENT_BASE_IMAGE in versions.env is the PUBLISHED toolbox, pinned by digest and
+# kept current by Renovate. An AGENT_BASE_IMAGE in the environment overrides it — CI passes
+# the toolbox it built in the same run, and a developer can pass `home-k3s-dev:local`.
+#
+# ENGINE PORTABILITY. CI runs this with ENGINE=docker against a Docker-in-Docker daemon, so
+# nothing below may be podman-only (`image exists`, `--digestfile`, `{{.Digest}}`), and
+# nothing may BIND-MOUNT a file from this checkout into a container: under DinD the daemon
+# resolves bind sources on ITS filesystem, where this checkout does not exist, and silently
+# mounts an empty directory instead. Files go in over stdin.
 ###############################################################################
 set -euo pipefail

@@ -46,8 +56,11 @@ command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGI
 # --- pins: sourced, never invented -------------------------------------------
 AGENT_ENV="${HERE}/versions.env"
 [ -f "${AGENT_ENV}" ] || die "missing ${AGENT_ENV} — the pins live there, not here"
+BASE_OVERRIDE="${AGENT_BASE_IMAGE:-}"
+TAG_OVERRIDE="${IMAGE_TAG:-}"
 # shellcheck disable=SC1090
 source "${AGENT_ENV}"
+[ -z "${BASE_OVERRIDE}" ] || AGENT_BASE_IMAGE="${BASE_OVERRIDE}"
 : "${AGENT_BASE_IMAGE:?AGENT_BASE_IMAGE not set in ${AGENT_ENV}}"
 : "${NODEJS_MAJOR:?NODEJS_MAJOR not set in ${AGENT_ENV}}"
 : "${CLAUDE_CODE_VERSION:?CLAUDE_CODE_VERSION not set in ${AGENT_ENV}}"
@@ -55,16 +68,28 @@ source "${AGENT_ENV}"
 : "${AGENT_MODEL:?AGENT_MODEL not set in ${AGENT_ENV}}"
 : "${AGENT_IMAGE_REPO:?AGENT_IMAGE_REPO not set in ${AGENT_ENV}}"

-# --- the base must already exist ---------------------------------------------
-# Deliberately NOT built on demand. The toolbox build is its own reviewed artefact with its
-# own pins, and silently building it here would mean this script could change the toolchain
-# a worker gates with as a side effect of rebuilding the agent image.
-if ! "${ENGINE}" image exists "${AGENT_BASE_IMAGE}" 2>/dev/null; then
-  die "base image '${AGENT_BASE_IMAGE}' is not present. Build the pinned toolbox first:
-       ${REPO_ROOT}/infra/devcontainer/dev.sh build"
+# --- the base must exist, locally or in the registry ---------------------------
+# Deliberately NOT built on demand. The toolbox is its own reviewed artefact with its own
+# pins, and silently building it here would mean this script could change the toolchain a
+# worker gates with as a side effect of rebuilding the agent image.
+case "${AGENT_BASE_IMAGE}" in
+  *@sha256:0000000000000000000000000000000000000000000000000000000000000000)
+    die "AGENT_BASE_IMAGE is still the all-zero placeholder in ${AGENT_ENV}: the toolbox has
+       not been published yet. Either let CI publish it (.gitea/workflows/images.yaml) and
+       merge Renovate's digest bump, or build on a local toolbox:
+         ${REPO_ROOT}/infra/devcontainer/dev.sh build
+         AGENT_BASE_IMAGE=home-k3s-dev:local $0" ;;
+esac
+if ! "${ENGINE}" image inspect "${AGENT_BASE_IMAGE}" >/dev/null 2>&1; then
+  info "base ${AGENT_BASE_IMAGE} is not local; pulling it"
+  "${ENGINE}" pull "${AGENT_BASE_IMAGE}" >/dev/null \
+    || die "base image '${AGENT_BASE_IMAGE}' is neither local nor pullable (registry login?).
+       To build on a local toolbox instead:
+         ${REPO_ROOT}/infra/devcontainer/dev.sh build
+         AGENT_BASE_IMAGE=home-k3s-dev:local $0"
 fi

-IMAGE_TAG="claude${CLAUDE_CODE_VERSION}-node${NODEJS_MAJOR}"
+IMAGE_TAG="${TAG_OVERRIDE:-claude${CLAUDE_CODE_VERSION}-node${NODEJS_MAJOR}}"
 IMAGE_REF="${AGENT_IMAGE_REPO}:${IMAGE_TAG}"

 log "Building ${IMAGE_REF}"
@@ -75,7 +100,11 @@ info "  node        ${NODEJS_MAJOR}"
 info "  claude      ${CLAUDE_CODE_VERSION} (floor ${CLAUDE_CODE_MIN_VERSION})"
 info "  model       ${AGENT_MODEL}"

-"${ENGINE}" build \
+# Provenance: CI passes the commit it is building, so a pulled image names its source tree.
+LABEL_ARGS=()
+[ -z "${BUILD_REVISION:-}" ] || LABEL_ARGS+=(--label "org.opencontainers.image.revision=${BUILD_REVISION}")
+
+"${ENGINE}" build "${LABEL_ARGS[@]}" \
  --file "${HERE}/Containerfile" \
  --tag "${IMAGE_REF}" \
  --build-arg "AGENT_BASE_IMAGE=${AGENT_BASE_IMAGE}" \
@@ -138,63 +167,32 @@ if [ "${VERIFY}" -eq 1 ]; then
  # that will actually run in the cluster. The probe is mounted read-only rather than baked
  # in: it is a build-time check, not something a worker Job has any business carrying.
  log "Probing pm-worker's input validation inside the image"
-  "${ENGINE}" run --rm \
+  #
+  # The probe goes in over STDIN, not as a bind mount: under Docker-in-Docker (CI) a bind
+  # source is resolved on the daemon's filesystem, where this file does not exist, and the
+  # container would get an empty directory. PM_WORKER_PATH points it at the baked worker,
+  # since the probe's own default (a sibling of the probe file) has no meaning on stdin.
+  "${ENGINE}" run --rm -i \
    --user 65532:65532 \
    --read-only \
    --tmpfs /workspace:rw,mode=1777 \
    --tmpfs /tmp:rw,mode=1777 \
    --network none \
-    --volume "${HERE}/probe-validation.sh:/probe-validation.sh:ro,Z" \
+    -e PM_WORKER_PATH=/usr/local/bin/pm-worker \
    --entrypoint "" \
    "${IMAGE_REF}" \
-    /bin/bash /probe-validation.sh
+    /bin/bash -s < "${HERE}/probe-validation.sh"

  log "Image verified"
 fi

-# --- digest ------------------------------------------------------------------
-DIGEST="$("${ENGINE}" image inspect --format '{{.Digest}}' "${IMAGE_REF}")"
+# --- result ------------------------------------------------------------------
+# No digest is printed: a manifest digest does not exist until a registry has the manifest,
+# and the only digest anything may pin is the one the REGISTRY returns. CI captures that at
+# push time and prints it in the job summary (.gitea/workflows/scripts/images.sh).
 SIZE="$("${ENGINE}" image inspect --format '{{.Size}}' "${IMAGE_REF}")"
-
 log "Built ${IMAGE_REF}"
-info "local manifest digest  ${DIGEST}"
-info "size                   $(( SIZE / 1024 / 1024 )) MiB"
-
-cat <<EOF
-
-================================================================================
-NOT PUSHED. Publishing is an operator action — this script holds no registry
-credential and will never acquire one.
-
-OPERATOR: run these, then put the digest from the digestfile into
-platform/pm-agent/configmap.yaml (PM_WORKER_IMAGE) via a pull request:
-
-    ${ENGINE} login gitea.home.keithjackson.dev
-
-    ${ENGINE} push --digestfile /tmp/agent-runner.digest ${IMAGE_REF}
-
-    cat /tmp/agent-runner.digest
-    # -> sha256:....   THIS is the authoritative digest, not the local one above.
-
-    # platform/pm-agent/configmap.yaml
-    PM_WORKER_IMAGE: "${AGENT_IMAGE_REPO}@\$(cat /tmp/agent-runner.digest)"
-
-WHY THE DIGEST AND NOT THE TAG: platform/kyverno/admission/pm-agent-job-shape.yaml is a
-fail-closed Enforce policy. It DENIES any worker Job whose image is not
-${AGENT_IMAGE_REPO}@sha256:<digest> — a tag reference is denied,
-and so is any other repository path. The dispatcher also refuses to start if
-PM_WORKER_IMAGE carries no digest, so the failure names its own cause.
-
-THE SAME IMAGE RUNS THE DISPATCHER. platform/pm-agent/deployment.yaml pins it too, so a
-new digest is a two-line change in one pull request: the Deployment's image and
-PM_WORKER_IMAGE. They must be the same digest — the dispatcher and its workers share the
-committed Job template and the env-name contract, and a mismatched pair would disagree
-about it.
-
-TIGHTEN THE BOUNDARY AFTERWARDS: the policy admits the wildcard
-\`${AGENT_IMAGE_REPO}@sha256:*\`, so anyone who can push to that registry path can publish
-a different digest. Replacing the wildcard with the LITERAL digest closes that gap; it is a
-one-line change, reviewed in a pull request, which is exactly where a change to the
-security boundary belongs.
-================================================================================
-EOF
+info "image id  $("${ENGINE}" image inspect --format '{{.Id}}' "${IMAGE_REF}")"
+info "size      $(( SIZE / 1024 / 1024 )) MiB"
+info "Not pushed. CI publishes this image from main (.gitea/workflows/images.yaml); see"
+info "platform/pm-agent/README.md for how a new digest reaches the cluster."
@@ -22,7 +22,9 @@
 ###############################################################################
 set -uo pipefail

-HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# `:-$0` because build-image.sh feeds this script over STDIN (`bash -s`), where BASH_SOURCE is
+# empty and `set -u` would abort on it; PM_WORKER_PATH is always set in that case.
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
 WORKER="${PM_WORKER_PATH:-${HERE}/pm-worker}"
 [ -r "$WORKER" ] || { echo "cannot read ${WORKER}" >&2; exit 2; }

@@ -9,10 +9,16 @@
 # truth and, the first time the two drifted, the worker's "green" would stop meaning the
 # gate's green. So this file pins ONLY what the toolbox does not carry.

-# The toolbox image the runner is layered on. Built by `infra/devcontainer/dev.sh build`
-# (tag `home-k3s-dev:local`); infra/agent/build-image.sh asserts it exists rather than
-# silently building a different base.
-AGENT_BASE_IMAGE=home-k3s-dev:local
+# The toolbox image the runner is layered on: the PUBLISHED toolbox, by digest. CI builds and
+# publishes it (.gitea/workflows/images.yaml) and Renovate moves this line onto each new digest
+# of its `main` tag via the marker below, in a pull request — nobody edits it by hand.
+#
+# ALL-ZERO UNTIL THE FIRST PUBLISH. The first run of the image pipeline on main sees this
+# placeholder, builds and publishes the toolbox first, and builds agent-runner FROM that fresh
+# digest. For a LOCAL build on a local toolbox, override it:
+#   infra/devcontainer/dev.sh build && AGENT_BASE_IMAGE=home-k3s-dev:local infra/agent/build-image.sh
+# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
+AGENT_BASE_IMAGE=gitea.home.keithjackson.dev/home-network/toolbox@sha256:0000000000000000000000000000000000000000000000000000000000000000

 # Node.js — the runtime for the Claude Code CLI, installed from the Fedora base's module
 # stream. A major version, not a patch: the CLI supports the active LTS line and the
@@ -80,6 +80,12 @@ spec:
          # Replaced by the dispatcher with the digest-pinned image. A tag here would be
          # DENIED by admission; the dispatcher refuses a non-digest PM_WORKER_IMAGE at
          # startup so the error names the real cause.
+          #
+          # DELIBERATELY ALL-ZERO, AND DELIBERATELY NOT TRACKED BY RENOVATE. This file is
+          # baked INTO agent-runner, so it cannot carry agent-runner's own digest: bumping it
+          # would change the image's inputs, CI would publish a new digest, Renovate would bump
+          # it again — forever. pm-dispatch overwrites this field from PM_WORKER_IMAGE
+          # (platform/pm-agent/configmap.yaml, which IS tracked) on every Job it creates.
          image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:0000000000000000000000000000000000000000000000000000000000000000
          imagePullPolicy: IfNotPresent
          # The pinned entrypoint, with NO args. Args on a pinned command are
@@ -34,6 +34,22 @@ cd infra/devcontainer
 ```
 `ENGINE=docker ./dev.sh` to use docker instead of rootless podman.

+### The published toolbox
+
+`./dev.sh build` delegates to [`build-image.sh`](build-image.sh), which is the **one** build
+recipe: CI runs the same script (with its in-image verification — every tool executes, and the
+version-sensitive ones report their `versions.env` pins) and, on a push to `main` that changes
+`Containerfile`, `versions.env` or `build-image.sh`, **publishes** the result as
+`gitea.home.keithjackson.dev/home-network/toolbox` (tags `sha-<commit>` and `main`) —
+see [`.gitea/workflows/images.yaml`](../../.gitea/workflows/images.yaml). `agent-runner` is
+built `FROM` that published digest (`infra/agent/versions.env`, kept current by Renovate).
+Nobody pushes this image from a workstation. To use the published one locally instead of
+building, after `podman login gitea.home.keithjackson.dev` (a token with `read:package`):
+
+```bash
+IMAGE=gitea.home.keithjackson.dev/home-network/toolbox:main ./dev.sh run make -C ci gate
+```
+
 ## Choosing a cluster — `CLUSTER=dev|prod`

 ```bash
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+###############################################################################
+# infra/devcontainer/build-image.sh — build the pinned dev `toolbox` image and prove it works.
+#
+#   ./build-image.sh                 # build + verify (tag: home-k3s-dev:local)
+#   ./build-image.sh --no-verify     # build only
+#   IMAGE=foo:bar ./build-image.sh   # build under another local name
+#   ENGINE=docker ./build-image.sh   # build with docker instead of podman
+#
+# ONE BUILD DEFINITION, TWO CALLERS. `dev.sh build` calls this for local development, and the
+# CI image pipeline (.gitea/workflows/images.yaml -> .gitea/workflows/scripts/images.sh) calls
+# it to build the image it then PUBLISHES as gitea.home.keithjackson.dev/home-network/toolbox.
+# Keeping the build args in exactly one place is what makes "the toolbox CI publishes" and "the
+# toolbox you gate with locally" the same recipe.
+#
+# THIS SCRIPT NEVER PUSHES. Publishing is CI's job (images.yaml, on push to main), not a
+# workstation's: an image that exists only because someone ran `podman push` from a laptop has
+# no build log, no recorded inputs and no verification anyone else can read.
+#
+# VERSIONS ARE NOT DEFINED HERE. infra/devcontainer/versions.env is the single source of truth.
+###############################################################################
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENGINE="${ENGINE:-podman}"
+IMAGE="${IMAGE:-home-k3s-dev:local}"
+
+VERIFY=1
+for arg in "$@"; do
+  case "$arg" in
+    --no-verify) VERIFY=0 ;;
+    -h|--help) sed -n '2,/^####/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown argument: $arg" >&2; exit 2 ;;
+  esac
+done
+
+log()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
+die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
+
+command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
+
+# shellcheck disable=SC1091
+source "${HERE}/versions.env"
+
+# Provenance: CI passes the commit it is building, so a pulled image names its source tree.
+LABEL_ARGS=()
+[ -z "${BUILD_REVISION:-}" ] || LABEL_ARGS+=(--label "org.opencontainers.image.revision=${BUILD_REVISION}")
+
+log "Building ${IMAGE}"
+"$ENGINE" build "${LABEL_ARGS[@]}" \
+  --build-arg FEDORA_VERSION="$FEDORA_VERSION" \
+  --build-arg OPENTOFU_VERSION="$OPENTOFU_VERSION" \
+  --build-arg AGE_VERSION="$AGE_VERSION" \
+  --build-arg SOPS_VERSION="$SOPS_VERSION" \
+  --build-arg FLUX_VERSION="$FLUX_VERSION" \
+  --build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
+  --build-arg HELM_VERSION="$HELM_VERSION" \
+  --build-arg KYVERNO_VERSION="$KYVERNO_VERSION" \
+  --build-arg KYVERNO_JSON_VERSION="$KYVERNO_JSON_VERSION" \
+  --build-arg CHAINSAW_VERSION="$CHAINSAW_VERSION" \
+  --build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \
+  --build-arg YQ_VERSION="$YQ_VERSION" \
+  --build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \
+  --build-arg PROMTOOL_VERSION="$PROMTOOL_VERSION" \
+  -t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
+
+# --- verify ------------------------------------------------------------------
+# A build that succeeded is not evidence the toolbox can run the gate. Every tool the gate and
+# the runbooks call must EXECUTE, and the version-sensitive ones must report the version
+# versions.env pins — promtool above all, because ci/versions.env asserts a promtool/Prometheus
+# lockstep and a drifted engine evaluates alert-rule tests differently (see versions.env).
+if [ "${VERIFY}" -eq 1 ]; then
+  log "Verifying the toolchain inside ${IMAGE}"
+  "$ENGINE" run --rm --network none --entrypoint "" \
+    -e PROMTOOL_VERSION="${PROMTOOL_VERSION}" \
+    -e KUBECTL_VERSION="${KUBECTL_VERSION}" \
+    -e CHAINSAW_VERSION="${CHAINSAW_VERSION}" \
+    -e KYVERNO_VERSION="${KYVERNO_VERSION}" \
+    -e SOPS_VERSION="${SOPS_VERSION}" \
+    "$IMAGE" /bin/bash -euo pipefail -c '
+      echo "--- baked versions ---"; cat /etc/dev-toolchain-versions
+      echo "--- every tool is on PATH ---"
+      for t in tofu age age-keygen sops flux kubectl helm kyverno kyverno-json chainsaw yq \
+               yamllint check-jsonschema promtool ansible-playbook git make jq diff python3 \
+               wg qrencode; do
+        if command -v "$t" >/dev/null 2>&1; then printf "    ok      %s\n" "$t";
+        else printf "    MISSING %s\n" "$t"; missing=1; fi
+      done
+      [ -z "${missing:-}" ] || { echo "toolbox is missing tools"; exit 1; }
+      echo "--- version-sensitive tools report their pins ---"
+      check() { # name pin actual-output
+        if printf "%s" "$3" | grep -qF "$2"; then printf "    ok      %-10s %s\n" "$1" "$2";
+        else printf "    DRIFT   %-10s want %s, got: %s\n" "$1" "$2" "$(printf "%s" "$3" | head -1)"; drift=1; fi
+      }
+      check promtool "${PROMTOOL_VERSION}" "$(promtool --version 2>&1)"
+      check kubectl  "${KUBECTL_VERSION}"  "$(kubectl version --client 2>&1)"
+      check chainsaw "${CHAINSAW_VERSION}" "$(chainsaw version 2>&1)"
+      check kyverno  "${KYVERNO_VERSION}"  "$(kyverno version 2>&1)"
+      check sops     "${SOPS_VERSION}"     "$(sops --version --disable-version-check 2>&1 || sops --version 2>&1)"
+      [ -z "${drift:-}" ] || { echo "a tool does not report its pinned version"; exit 1; }
+      echo "--- python can import what the renderer and the gate need ---"
+      python3 -c "import yaml; print(\"    PyYAML\", yaml.__version__)"
+    '
+  log "Toolbox verified"
+fi
+
+log "Built ${IMAGE}"
+printf '    image id  %s\n' "$("$ENGINE" image inspect --format '{{.Id}}' "$IMAGE")"
@@ -72,22 +72,10 @@ esac
 CLUSTER_KUBECONFIG="${REPO_ROOT}/infra/ansible/.cache/${CLUSTER}.kubeconfig"

 build() {
-  "$ENGINE" build \
-    --build-arg FEDORA_VERSION="$FEDORA_VERSION" \
-    --build-arg OPENTOFU_VERSION="$OPENTOFU_VERSION" \
-    --build-arg AGE_VERSION="$AGE_VERSION" \
-    --build-arg SOPS_VERSION="$SOPS_VERSION" \
-    --build-arg FLUX_VERSION="$FLUX_VERSION" \
-    --build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
-    --build-arg HELM_VERSION="$HELM_VERSION" \
-    --build-arg KYVERNO_VERSION="$KYVERNO_VERSION" \
-    --build-arg KYVERNO_JSON_VERSION="$KYVERNO_JSON_VERSION" \
-    --build-arg CHAINSAW_VERSION="$CHAINSAW_VERSION" \
-    --build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \
-    --build-arg YQ_VERSION="$YQ_VERSION" \
-    --build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \
-    --build-arg PROMTOOL_VERSION="$PROMTOOL_VERSION" \
-    -t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
+  # The build recipe lives in build-image.sh so that CI (.gitea/workflows/images.yaml), which
+  # builds and PUBLISHES this image, and a local `dev.sh build` cannot drift apart. --no-verify
+  # keeps the local loop as fast as it always was; CI runs the verification.
+  IMAGE="$IMAGE" ENGINE="$ENGINE" "${HERE}/build-image.sh" --no-verify
 }

 run() {
@@ -252,11 +252,16 @@ step "preconditions — repo side (these block ONLY arming ${VARIABLE}; the secr

 # R1: the runner image is built and digest-pinned. Without it the Job cannot run run.sh at
 # all (the stock chainsaw image has no bash/python3/kubectl/git) — plan §3 item 1.
+# EVALUATED, not grepped: the line is `CONFORMANCE_RUNNER_IMAGE="${..._REPO}@${..._DIGEST}"`, so
+# its literal text never contains 64 hex digits and the grep this replaced could never pass. The
+# all-zero digest is the "not published yet" placeholder (Renovate replaces it after CI's first
+# publish, .gitea/workflows/images.yaml) and must not count as pinned.
 VERSIONS="${REPO_ROOT}/conformance/versions.env"
-if grep -Eq '^[[:space:]]*CONFORMANCE_RUNNER_IMAGE=.*@sha256:[0-9a-f]{64}' "$VERSIONS" 2>/dev/null; then
-  pass "CONFORMANCE_RUNNER_IMAGE is digest-pinned in conformance/versions.env"
+runner_img="$( (set +u; . "$VERSIONS" >/dev/null 2>&1; printf '%s' "${CONFORMANCE_RUNNER_IMAGE:-}") )"
+if [[ "$runner_img" =~ @sha256:[0-9a-f]{64}$ ]] && [[ ! "$runner_img" =~ @sha256:0{64}$ ]]; then
+  pass "CONFORMANCE_RUNNER_IMAGE is digest-pinned in conformance/versions.env (${runner_img##*@})"
 else
-  fail_arm "conformance/versions.env has no digest-pinned CONFORMANCE_RUNNER_IMAGE (plan §3 item 1) — the Job cannot run ./run.sh"
+  fail_arm "conformance/versions.env has no PUBLISHED digest for CONFORMANCE_RUNNER_IMAGE (got '${runner_img}') — the image pipeline has not published it yet, or Renovate's bump is unmerged (plan §3 item 1)"
 fi

 # R2: the Job template is the real thing, not the skeleton: the admitted image AND an
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# Registry retention for this repo's OWN images (toolbox, agent-runner, conformance-runner):
+# keep what is deployed and what is recent, delete the rest. DRY RUN unless --apply.
+#
+#   ./infra/gitea/registry-retention.sh              # show what would be deleted
+#   ./infra/gitea/registry-retention.sh --apply      # delete it
+#   KEEP=15 ./infra/gitea/registry-retention.sh      # keep the 15 newest sha- tags (default 10)
+#
+# RUNNING IT IS AN OPERATOR ACTION. It deletes registry content, so it is not wired into CI and
+# no agent runs it with --apply.
+#
+# WHY A SCRIPT, AND NOT GITEA'S BUILT-IN "CLEANUP RULE" — measured against Gitea 1.24.3's source:
+#
+#   1. There is NO API for cleanup rules. routers/api/v1/api.go has no such route; the rules live
+#      only behind the web UI's session+CSRF forms (routers/web/web.go, org settings
+#      `/packages/rules/add`). A rule therefore cannot be created, read or drift-checked as code —
+#      which is also why ci/gitea-settings.sh cannot assert one exists.
+#   2. A built-in rule would DELETE WHAT IS DEPLOYED. Every reference to these images is
+#      digest-only (`<repo>@sha256:...`, required by admission). Gitea stores a manifest as a
+#      file OF THE TAG VERSION it was pushed under (routers/api/packages/container/manifest.go
+#      createPackageAndVersion), and resolves a by-digest pull by searching those files
+#      (models/packages/container/search.go). So deleting the tag a digest was pushed under makes
+#      that digest unpullable — MANIFEST_UNKNOWN, i.e. ImagePullBackOff on the next reschedule.
+#      The built-in rule keeps "the newest N" and a name pattern, and knows nothing about which
+#      digest git currently pins. Renovate's bump PRs can lag, so "newest N" is not a guarantee.
+#
+# WHAT IS KEPT, per image:
+#   * the `main` and `latest` tags;
+#   * the KEEP newest `sha-<commit>` tags (by creation time);
+#   * EVERY tag whose manifest digest is referenced anywhere in the tracked tree at REF
+#     (default origin/main) — i.e. whatever Flux is deploying, and the toolbox digest
+#     agent-runner builds FROM;
+#   * every version whose NAME is a digest (`sha256:...`): those are index children, and Gitea's
+#     own cleanup handles them (services/packages/container/cleanup.go ShouldBeSkipped).
+# Everything else is deleted. Gitea's `cleanup_packages` cron then reclaims the unreferenced
+# blobs (default: daily, blobs older than 24h) — space comes back the day after, not instantly.
+#
+# THE STORAGE IT PROTECTS. Packages live on the SAME 10Gi Longhorn PVC as every repository and
+# Gitea's database (catalog/gitea.yaml). The toolbox is ~1.4 GB uncompressed and every main push
+# that touches its inputs adds a complete, non-reproducible copy. Unpruned, the registry is what
+# fills the volume the git host lives on.
+#
+# CREDENTIAL: the one `git` has stored for the host (as the other scripts here). It needs
+# `read:package` to list and `write:package` to --apply, and the account needs package write on
+# the org. The secret is never printed.
+set -euo pipefail
+HOST=gitea.home.keithjackson.dev; OWNER=Home-Network; NS=home-network
+IMAGES=(toolbox agent-runner conformance-runner)
+KEEP="${KEEP:-10}"
+REF="${REF:-origin/main}"
+APPLY=0
+for a in "$@"; do
+  case "$a" in
+    --apply) APPLY=1 ;;
+    -h|--help) sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "unknown argument: $a" >&2; exit 2 ;;
+  esac
+done
+[[ "$KEEP" =~ ^[0-9]+$ ]] && [ "$KEEP" -ge 1 ] || { echo "KEEP must be a positive integer" >&2; exit 2; }
+command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; }
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+git -C "$REPO_ROOT" fetch -q origin main 2>/dev/null || echo "   (could not fetch origin/main; using the local ${REF})" >&2
+git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" >/dev/null || { echo "no such ref: ${REF}" >&2; exit 1; }
+
+creds="$(printf 'protocol=https\nhost=%s\n\n' "$HOST" | git credential fill)"
+U="$(sed -n 's/^username=//p' <<<"$creds")"; P="$(sed -n 's/^password=//p' <<<"$creds")"
+[ -n "$U" ] && [ -n "$P" ] || { echo "no stored credential for $HOST" >&2; exit 1; }
+echo "authenticating as '${U}' (secret not shown)"
+api() { curl -sS --config <(printf 'user = "%s:%s"\n' "$U" "$P") "$@"; }
+
+REG_TOKEN="$(api "https://${HOST}/v2/token?service=container_registry" | jq -r '.token // empty')"
+[ -n "$REG_TOKEN" ] || { echo "registry token exchange failed (does the credential have read:package?)" >&2; exit 1; }
+manifest_digest() { # <image> <tag> -> digest, or nothing
+  curl -sS -I --config <(printf 'header = "Authorization: Bearer %s"\n' "$REG_TOKEN") \
+    -H 'Accept: application/vnd.oci.image.manifest.v1+json' \
+    -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
+    -H 'Accept: application/vnd.oci.image.index.v1+json' \
+    -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
+    "https://${HOST}/v2/${NS}/$1/manifests/$2" \
+    | tr -d '\r' | awk -F': ' 'tolower($1)=="docker-content-digest"{print $2}' | tail -1
+}
+
+TOTAL_DEL=0
+for img in "${IMAGES[@]}"; do
+  echo
+  echo "==> ${NS}/${img}"
+  # Digests git pins at REF. The marker convention puts the digest on the line after
+  # `# renovate-digest: image=<repo>`; the plain `<repo>@sha256:` grep catches any other use.
+  repo="${HOST}/${NS}/${img}"
+  pinned="$( { git -C "$REPO_ROOT" grep -h -A1 -F "renovate-digest: image=${repo} " "$REF" -- . 2>/dev/null || true
+               git -C "$REPO_ROOT" grep -h -F "${repo}@sha256:" "$REF" -- . 2>/dev/null || true; } \
+             | grep -oE 'sha256:[0-9a-f]{64}' | grep -v 'sha256:0\{64\}' | sort -u || true)"
+  echo "    pinned at ${REF}: $(wc -w <<<"$pinned") digest(s)"
+
+  versions="[]"; page=1
+  while :; do
+    chunk="$(api "https://${HOST}/api/v1/packages/${OWNER}?type=container&q=${img}&limit=50&page=${page}")"
+    jq -e 'type=="array"' >/dev/null <<<"$chunk" || { echo "    package list failed: $(head -c 200 <<<"$chunk")" >&2; exit 1; }
+    [ "$(jq length <<<"$chunk")" -gt 0 ] || break
+    versions="$(jq -c --argjson a "$versions" --arg n "$img" '$a + [ .[] | select(.name==$n) | {version, created_at} ]' <<<"$chunk")"
+    page=$((page + 1))
+  done
+  echo "    versions: $(jq length <<<"$versions")"
+
+  keep_sha="$(jq -r '[ .[] | select(.version|startswith("sha-")) ] | sort_by(.created_at) | reverse | .[:'"$KEEP"'][] .version' <<<"$versions")"
+  while IFS= read -r v; do
+    [ -n "$v" ] || continue
+    case "$v" in
+      main|latest)  echo "    keep    ${v}  (moving tag)"; continue ;;
+      sha256:*)     echo "    keep    ${v}  (digest-named version; Gitea's cleanup owns those)"; continue ;;
+    esac
+    if grep -qxF "$v" <<<"$keep_sha"; then echo "    keep    ${v}  (among the ${KEEP} newest)"; continue; fi
+    d="$(manifest_digest "$img" "$v")"
+    if [ -z "$d" ]; then echo "    keep    ${v}  (could not read its digest — never delete what cannot be checked)"; continue; fi
+    if grep -qxF "$d" <<<"$pinned"; then echo "    keep    ${v}  (${d} is pinned at ${REF})"; continue; fi
+    if [ "$APPLY" -eq 1 ]; then
+      code="$(api -o /dev/null -w '%{http_code}' -X DELETE \
+        "https://${HOST}/api/v1/packages/${OWNER}/container/${img}/$(jq -rn --arg v "$v" '$v|@uri')")"
+      if [ "$code" = 204 ]; then echo "    DELETED ${v}  (${d})"; TOTAL_DEL=$((TOTAL_DEL + 1));
+      else echo "    FAILED  ${v}  HTTP ${code}" >&2; fi
+    else
+      echo "    would delete ${v}  (${d})"; TOTAL_DEL=$((TOTAL_DEL + 1))
+    fi
+  done < <(jq -r '.[].version' <<<"$versions")
+done
+
+echo
+if [ "$APPLY" -eq 1 ]; then echo "deleted ${TOTAL_DEL} version(s). Blobs are reclaimed by Gitea's cleanup_packages cron."
+else echo "DRY RUN: ${TOTAL_DEL} version(s) would be deleted. Re-run with --apply to delete them."; fi