home-network-k3s-impl/.gitea/workflows/scripts/images.sh

#!/usr/bin/env bash
###############################################################################
# .gitea/workflows/scripts/images.sh — the image pipeline's driver (images.yaml calls this).
#
# Builds, verifies and (on main) PUBLISHES the three images this repo ships:
#
#   toolbox             infra/devcontainer/    the pinned dev/gate toolchain
#   agent-runner        infra/agent/           FROM toolbox; the PM dispatcher + worker
#   conformance-runner  conformance/incluster/ the in-cluster L2/L3 Chainsaw runner
#
# THE BUILDER (operator decision on PR #81): a ROOTLESS BuildKit daemon, platform/buildkit/,
# reached over mTLS. No CI job may reach a privileged daemon, so there is no `docker build` and no
# `docker run` here: this script drives buildkitd with a pinned `buildx` (remote driver) and
# nothing else. The client certificate arrives as Actions secrets (BUILDKIT_CA,
# BUILDKIT_CLIENT_CERT, BUILDKIT_CLIENT_KEY; infra/gitea/buildkit-client-secret.sh puts them
# there). If the daemon is unreachable this FAILS LOUDLY — it never skips.
#
# VERIFICATION IS A BUILD TARGET. Each Containerfile ends in a `verify` stage that runs every
# in-image assertion at build time (as the non-root uid, with RUN --network=none); this script
# builds `<image>-verify` and only then the image. Build args come from each image's own
# `build-image.sh --print-build-args`, the same list the local podman path uses.
#
# MODES (env MODE):
#   verify   build the verify targets, never push, hold no registry credential. Pull requests.
#   publish  verify, then build + push to ${REGISTRY}/${NAMESPACE}/<image> with tags
#            `sha-<12 hex>` (immutable: never re-pushed) and `main` (moving; Renovate follows
#            it). The digest recorded is the one the REGISTRY returns, read back after the push.
#
# WHAT GETS BUILT (env IMAGES): `auto` (default) diffs BASE_SHA..HEAD_SHA against each image's
# inputs; otherwise a space-separated list. A change to the pipeline itself (images.yaml, this
# file, platform/buildkit/versions.env) builds everything in verify mode, and nothing extra in
# publish mode: re-publishing unchanged inputs would only mint new digests and churn Renovate PRs.
#
# THE agent-runner BASE:
#   * verify mode -> FROM the toolbox built in this run, linked with a bake `target:` context —
#     a pull request holds no registry credential and that toolbox exists in no registry. If the
#     toolbox's inputs did not change it is built anyway, as the base.
#   * publish mode, toolbox published in this run -> FROM the digest the registry just returned.
#   * publish mode otherwise -> AGENT_BASE_IMAGE from infra/agent/versions.env (the published
#     toolbox digest Renovate keeps current). If that is still the all-zero placeholder the
#     toolbox has never been published, so it is built and published first (the bootstrap).
#
# LOOP-FREEDOM. Renovate turns each publish into a digest-bump PR. Those bumps land in
# platform/pm-agent/ (no image input), conformance/versions.env (no image input) and
# infra/agent/versions.env (agent-runner input -> ONE more agent-runner build, whose own bump
# lands in platform/pm-agent/ and stops). The worker Job template is deliberately NOT tracked:
# it is baked INTO agent-runner, so bumping it would rebuild agent-runner, mint a new digest,
# bump it again, forever. pm-dispatch overwrites that field at runtime anyway.
###############################################################################
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
cd "${REPO_ROOT}"

# shellcheck disable=SC1091
BK_HOST_OVERRIDE="${BUILDKIT_HOST:-}"; BK_NAME_OVERRIDE="${BUILDKIT_SERVERNAME:-}"
. ./platform/buildkit/versions.env
BUILDKIT_HOST="${BK_HOST_OVERRIDE:-${BUILDKIT_HOST}}"
BUILDKIT_SERVERNAME="${BK_NAME_OVERRIDE:-${BUILDKIT_SERVERNAME}}"

MODE="${MODE:-verify}"
IMAGES="${IMAGES:-auto}"
REGISTRY="${REGISTRY:-gitea.home.keithjackson.dev}"
NAMESPACE="${NAMESPACE:-home-network}"
HEAD_SHA="${HEAD_SHA:-$(git rev-parse HEAD)}"
BASE_SHA="${BASE_SHA:-}"
SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
ZERO_DIGEST="sha256:0000000000000000000000000000000000000000000000000000000000000000"

ALL_IMAGES=(toolbox agent-runner conformance-runner)
SHA_TAG="sha-${HEAD_SHA:0:12}"
MOVING_TAG="main"

log()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
info() { printf '    %s\n' "$*"; }
die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; printf -- '- **ERROR**: %s\n' "$*" >>"${SUMMARY}"; exit 1; }
yn()   { if [ -n "$1" ]; then echo "$2"; else echo "$3"; fi; }

case "${MODE}" in verify|publish) ;; *) die "MODE must be verify or publish, got '${MODE}'" ;; esac
for t in git jq curl sha256sum; do command -v "$t" >/dev/null || die "the job image lacks '$t'"; done

# --- per-image facts ------------------------------------------------------------------------
# Inputs: a change under any of these rebuilds that image. conformance-runner takes its pins from
# infra/devcontainer/versions.env. conformance/versions.env is NOT an input (it only RECORDS the
# published digest), and must not become one: Renovate writes it after every publish -> a loop.
inputs() {
  case "$1" in
    toolbox)            echo "infra/devcontainer/Containerfile infra/devcontainer/versions.env infra/devcontainer/build-image.sh" ;;
    agent-runner)       echo "infra/agent/" ;;
    conformance-runner) echo "conformance/incluster/Containerfile conformance/incluster/conformance-clone conformance/incluster/build-image.sh infra/devcontainer/versions.env" ;;
  esac
}
context_of() { case "$1" in toolbox) echo infra/devcontainer ;; agent-runner) echo infra/agent ;; conformance-runner) echo conformance/incluster ;; esac; }
PIPELINE_FILES=".gitea/workflows/images.yaml .gitea/workflows/scripts/images.sh platform/buildkit/versions.env"

declare -A WANT=() BUILT=() DIGEST=() SECS=() NOTE=()

changed_any() { # <paths...> -> 0 if BASE_SHA..HEAD_SHA touches any
  [ -n "$(git diff --name-only "${BASE_SHA}" "${HEAD_SHA}" -- "$@")" ]
}

select_images() {
  local i
  if [ "${IMAGES}" != "auto" ]; then
    for i in ${IMAGES}; do
      case " ${ALL_IMAGES[*]} " in *" ${i} "*) WANT[$i]=1; NOTE[$i]="requested" ;; *) die "unknown image '${i}'" ;; esac
    done
    return
  fi
  if [ -z "${BASE_SHA}" ] || [[ "${BASE_SHA}" =~ ^0+$ ]] || ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
    info "no usable diff base ('${BASE_SHA}'): building everything"
    for i in "${ALL_IMAGES[@]}"; do WANT[$i]=1; NOTE[$i]="no diff base"; done
    return
  fi
  info "diff base ${BASE_SHA} .. ${HEAD_SHA}"
  for i in "${ALL_IMAGES[@]}"; do
    # shellcheck disable=SC2046
    if changed_any $(inputs "$i"); then WANT[$i]=1; NOTE[$i]="inputs changed"; fi
  done
  # shellcheck disable=SC2086
  if [ "${MODE}" = verify ] && changed_any ${PIPELINE_FILES}; then
    for i in "${ALL_IMAGES[@]}"; do [ -n "${WANT[$i]:-}" ] || { WANT[$i]=1; NOTE[$i]="pipeline changed"; }; done
  fi
}

# --- the builder: pinned buildx + mTLS to rootless buildkitd ---------------------------------
WORK="$(mktemp -d)"
trap 'rm -rf "${WORK}"' EXIT          # client key + registry auth live only here, only this long
export BUILDX_CONFIG="${WORK}/buildx" DOCKER_CONFIG="${WORK}/docker"
# Single-manifest images: the admission policies, Renovate and registry-retention.sh all reason
# about one manifest per tag. (Provenance attestations would turn every push into an index.)
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
mkdir -p "${BUILDX_CONFIG}" "${DOCKER_CONFIG}" "${WORK}/certs"
echo '{}' > "${DOCKER_CONFIG}/config.json"

setup_builder() {
  log "Builder: rootless buildkitd at ${BUILDKIT_HOST} (mTLS)"
  BUILDX="${BUILDX_BIN:-}"
  if [ -z "${BUILDX}" ]; then
    BUILDX="${WORK}/buildx-bin"
    curl -fsSL --retry 3 -o "${BUILDX}" \
      "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64" \
      || die "could not download buildx v${BUILDX_VERSION}"
    echo "${BUILDX_SHA256}  ${BUILDX}" | sha256sum -c --quiet - \
      || die "buildx v${BUILDX_VERSION} does not match the sha256 pinned in platform/buildkit/versions.env — refusing to run it"
    chmod +x "${BUILDX}"
  fi
  info "client: $("${BUILDX}" version)"

  local missing=()
  [ -n "${BUILDKIT_CA:-}" ]          || missing+=(BUILDKIT_CA)
  [ -n "${BUILDKIT_CLIENT_CERT:-}" ] || missing+=(BUILDKIT_CLIENT_CERT)
  [ -n "${BUILDKIT_CLIENT_KEY:-}" ]  || missing+=(BUILDKIT_CLIENT_KEY)
  [ "${#missing[@]}" -eq 0 ] || die "the builder's client certificate is missing: ${missing[*]} not set.
       These Actions secrets are copied from the cert-manager Secret buildkit/buildkit-client-ci by
       infra/gitea/buildkit-client-secret.sh (an operator action, after platform/buildkit is
       deployed). Without them nothing can be built — this fails rather than skipping ON PURPOSE."
  umask 077
  printf '%s\n' "${BUILDKIT_CA}"          > "${WORK}/certs/ca.pem"
  printf '%s\n' "${BUILDKIT_CLIENT_CERT}" > "${WORK}/certs/cert.pem"
  printf '%s\n' "${BUILDKIT_CLIENT_KEY}"  > "${WORK}/certs/key.pem"
  umask 022

  "${BUILDX}" create --name ci-remote --driver remote \
    --driver-opt "cacert=${WORK}/certs/ca.pem,cert=${WORK}/certs/cert.pem,key=${WORK}/certs/key.pem,servername=${BUILDKIT_SERVERNAME}" \
    "${BUILDKIT_HOST}" >/dev/null
  if ! out="$(timeout 90 "${BUILDX}" inspect --bootstrap --builder ci-remote 2>&1)"; then
    printf '%s\n' "${out}" >&2
    die "buildkitd is UNREACHABLE at ${BUILDKIT_HOST} — nothing can be built, and this job fails
       rather than skipping. In order of likelihood:
         1. platform/buildkit is not deployed yet. Flux applies it only after it merges to main
            (Kustomization 'buildkit'); a pull request that ADDS it cannot reach it.
         2. the BUILDKIT_* secrets are stale (the client cert was renewed) or from another CA:
            re-run infra/gitea/buildkit-client-secret.sh.
         3. the NetworkPolicy (platform/buildkit/networkpolicy.yaml) does not admit this runner."
  fi
  printf '%s\n' "${out}" | grep -E '^(Name|Driver|Status|BuildKit version|Platforms):' | sed 's/^/    /' || true
}

# KEY=VALUE lines from an image's build-image.sh --print-build-args -> a JSON object.
build_args_json() {
  local ctx; ctx="$(context_of "$1")"
  "./${ctx}/build-image.sh" --print-build-args | jq -Rn '[inputs | select(length>0) | capture("^(?<k>[^=]+)=(?<v>.*)$")] | map({(.k): .v}) | add'
}

# bake_run <bakefile> <targets...>
bake_run() {
  local f="$1"; shift
  "${BUILDX}" bake --builder ci-remote --progress plain -f "$f" \
    --metadata-file "${WORK}/meta.json" "$@"
}

# Write a bake file with a verify target and a ship target for one image.
#   $1 image, $2 agent base mode: "" | "link" | <image ref>
write_bake() {
  local img="$1" base="${2:-}" ctx args f
  ctx="$(context_of "$img")"
  args="$(build_args_json "$img")"
  f="${WORK}/bake-${img}.json"
  local contexts='{}'
  if [ "${img}" = agent-runner ]; then
    if [ "${base}" = link ]; then
      # FROM ${AGENT_BASE_IMAGE} resolves to the named context below, i.e. the toolbox target
      # of this same bake file — built here, never pushed anywhere.
      args="$(jq -c '. + {AGENT_BASE_IMAGE: "toolbox-in-this-run"}' <<<"${args}")"
      contexts='{"toolbox-in-this-run": "target:toolbox"}'
    else
      args="$(jq -c --arg b "${base}" '. + {AGENT_BASE_IMAGE: $b}' <<<"${args}")"
    fi
  fi
  jq -n --arg img "$img" --arg ctx "$ctx" --argjson args "$args" --argjson contexts "$contexts" \
        --arg tbargs "$( [ "${base}" = link ] && build_args_json toolbox || echo '{}')" \
        --arg rev "${HEAD_SHA}" --arg repo "${REGISTRY}/${NAMESPACE}/${img}" \
        --arg sha "${SHA_TAG}" --arg moving "${MOVING_TAG}" --arg mode "${MODE}" '
    {target: (
      {
        ($img + "-verify"): {context: $ctx, dockerfile: "Containerfile", target: "verify",
                              args: $args, contexts: $contexts, output: ["type=cacheonly"]},
        ($img): ({context: $ctx, dockerfile: "Containerfile", target: $img, args: $args,
                  contexts: $contexts,
                  labels: {"org.opencontainers.image.revision": $rev}}
                 + (if $mode == "publish"
                    then {tags: [($repo + ":" + $sha), ($repo + ":" + $moving)], output: ["type=image,push=true"]}
                    else {output: ["type=cacheonly"]} end))
      }
      + (if ($contexts | has("toolbox-in-this-run"))
         then {toolbox: {context: "infra/devcontainer", dockerfile: "Containerfile", target: "toolbox",
                         args: ($tbargs | fromjson), output: ["type=cacheonly"]}}
         else {} end)
    )}' > "$f"
  echo "$f"
}

# --- registry helpers (publish mode only) ----------------------------------------------------
REG_TOKEN=""
registry_token() {
  [ -n "${REG_TOKEN}" ] && return 0
  REG_TOKEN="$(curl -fsS --config <(printf 'user = "%s:%s"\n' "${REGISTRY_USER}" "${REGISTRY_TOKEN}") \
    "https://${REGISTRY}/v2/token?service=container_registry&scope=*" | jq -r '.token // empty')" \
    || die "could not obtain a registry bearer token from https://${REGISTRY}/v2/token"
  [ -n "${REG_TOKEN}" ] || die "registry returned no bearer token"
}
# Prints the manifest digest the REGISTRY serves for <image>:<tag>; nothing if it is absent (404).
# Any other answer is fatal: "cannot tell" must never be read as "not published", or the
# immutable-tag guard would be bypassed by an auth hiccup.
registry_digest() {
  registry_token
  local hdrs code digest
  hdrs="$(curl -sS -I --config <(printf 'header = "Authorization: Bearer %s"\n' "${REG_TOKEN}") \
    -H 'Accept: application/vnd.oci.image.manifest.v1+json' \
    -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
    -H 'Accept: application/vnd.oci.image.index.v1+json' \
    -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
    "https://${REGISTRY}/v2/${NAMESPACE}/$1/manifests/$2" | tr -d '\r')" \
    || die "registry unreachable while reading $1:$2"
  code="$(awk 'NR==1{print $2}' <<<"${hdrs}")"
  digest="$(awk -F': ' 'tolower($1)=="docker-content-digest"{print $2}' <<<"${hdrs}" | tail -1)"
  case "${code}" in
    200) [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || die "registry answered 200 for $1:$2 with no usable digest"
         echo "${digest}" ;;
    404) ;;
    *)   die "registry answered HTTP ${code} for $1:$2 — cannot tell whether it exists" ;;
  esac
}

# build_one <image> [agent base]  — verify, then (publish) build + push + read back.
build_one() {
  local img="$1" base="${2:-}" t0 f pushed got existing
  t0=$(date +%s)
  f="$(write_bake "$img" "$base")"
  log "VERIFY ${img} (Containerfile target: verify, on rootless BuildKit)"
  bake_run "$f" "${img}-verify"
  BUILT[$img]=1
  if [ "${MODE}" = publish ]; then
    existing="$(registry_digest "${img}" "${SHA_TAG}")"
    # Immutable tag: never overwrite. A rebuild is not byte-reproducible, so re-pushing would
    # silently change what `sha-<commit>` means after something may already have pinned it.
    [ -z "${existing}" ] || die "${REGISTRY}/${NAMESPACE}/${img}:${SHA_TAG} already exists (${existing}); tags are immutable"
    log "BUILD + PUSH ${img} -> ${REGISTRY}/${NAMESPACE}/${img}:{${SHA_TAG},${MOVING_TAG}}"
    bake_run "$f" "${img}"
    pushed="$(jq -r --arg t "$img" '.[$t]["containerimage.digest"] // empty' "${WORK}/meta.json")"
    # The AUTHORITATIVE digest is what the registry serves, read back — not what the client
    # reported. They should agree; if they do not, nothing downstream may trust either.
    got="$(registry_digest "${img}" "${SHA_TAG}")"
    [ -n "${got}" ] || die "pushed ${img}:${SHA_TAG} but the registry does not serve it"
    [ -z "${pushed}" ] || [ "${pushed}" = "${got}" ] || die "${img}: client reported ${pushed} but the registry serves ${got}"
    [ "$(registry_digest "${img}" "${MOVING_TAG}")" = "${got}" ] \
      || die "${img}: ${MOVING_TAG} does not point at the digest just pushed (${got})"
    DIGEST[$img]="${got}"
    info "${img}: registry digest ${got}"
  fi
  SECS[$img]=$(( $(date +%s) - t0 ))
  info "${img}: done in ${SECS[$img]}s"
}

# --- main ---------------------------------------------------------------------------------
log "image pipeline: MODE=${MODE} IMAGES=${IMAGES} HEAD=${HEAD_SHA}"
select_images

if [ "${MODE}" = publish ]; then
  : "${REGISTRY_TOKEN:?REGISTRY_TOKEN is required in publish mode}"
  REGISTRY_USER="${REGISTRY_USER:-ci-image-publisher}"
  # buildx (and through it buildkitd, which does the push) reads registry auth from this file.
  jq -n --arg r "${REGISTRY}" --arg a "$(printf '%s:%s' "${REGISTRY_USER}" "${REGISTRY_TOKEN}" | base64 -w0)" \
     '{auths: {($r): {auth: $a}}}' > "${DOCKER_CONFIG}/config.json"
  chmod 600 "${DOCKER_CONFIG}/config.json"
  registry_token
  # Publish idempotence: an image already published at THIS commit is not rebuilt. That is what
  # makes a re-run after a half-finished publish safe — the immutable tag is never pushed twice.
  for i in "${ALL_IMAGES[@]}"; do
    [ -n "${WANT[$i]:-}" ] || continue
    d="$(registry_digest "${i}" "${SHA_TAG}")"
    if [ -n "${d}" ]; then
      info "${i}: already published at ${SHA_TAG} (${d}); not rebuilding"
      unset "WANT[$i]"; DIGEST[$i]="${d}"; NOTE[$i]="already published at this commit"
    fi
  done
fi

# Decide the agent-runner base (see header).
AGENT_BASE=""
if [ -n "${WANT[agent-runner]:-}" ]; then
  PINNED_BASE="$(. ./infra/agent/versions.env; echo "${AGENT_BASE_IMAGE}")"
  if [ "${MODE}" = verify ]; then
    AGENT_BASE=link
    [ -n "${WANT[toolbox]:-}" ] || { WANT[toolbox]=1; NOTE[toolbox]="built as agent-runner's base (verify only)"; }
  elif [ -z "${WANT[toolbox]:-}" ] && [ -z "${DIGEST[toolbox]:-}" ]; then
    case "${PINNED_BASE}" in
      *@"${ZERO_DIGEST}") WANT[toolbox]=1; NOTE[toolbox]="never published; required as agent-runner's base" ;;
      *@sha256:*)         AGENT_BASE="${PINNED_BASE}" ;;
      *) die "infra/agent/versions.env AGENT_BASE_IMAGE='${PINNED_BASE}' is not digest-pinned; CI will not build on a floating base" ;;
    esac
  fi
fi

for i in "${ALL_IMAGES[@]}"; do info "plan: ${i}: $(yn "${WANT[$i]:-}" BUILD skip)  (${NOTE[$i]:-inputs unchanged})"; done
if [ -z "${WANT[toolbox]:-}${WANT[agent-runner]:-}${WANT[conformance-runner]:-}" ]; then
  info "nothing to build"
else
  setup_builder
fi

if [ -n "${WANT[toolbox]:-}" ]; then build_one toolbox; fi
if [ -n "${WANT[agent-runner]:-}" ]; then
  if [ -z "${AGENT_BASE}" ]; then
    AGENT_BASE="${REGISTRY}/${NAMESPACE}/toolbox@${DIGEST[toolbox]:?no toolbox digest to build agent-runner from}"
  fi
  info "agent-runner base: $( [ "${AGENT_BASE}" = link ] && echo "the toolbox built in this run (bake target: context)" || echo "${AGENT_BASE}")"
  build_one agent-runner "${AGENT_BASE}"
fi
if [ -n "${WANT[conformance-runner]:-}" ]; then build_one conformance-runner; fi

# --- summary ------------------------------------------------------------------------------
{
  echo "## Images (${MODE})"
  echo
  echo "commit \`${HEAD_SHA}\` — builder: rootless BuildKit \`${BUILDKIT_HOST}\` (mTLS)"
  echo
  echo "| image | verify target built | seconds | published digest (registry-returned) | note |"
  echo "|---|---|---|---|---|"
  for i in "${ALL_IMAGES[@]}"; do
    d="${DIGEST[$i]:-}"
    echo "| \`${i}\` | $(yn "${BUILT[$i]:-}" yes no) | ${SECS[$i]:--} | $(yn "${d}" "\`${REGISTRY}/${NAMESPACE}/${i}@${d}\`" -) | ${NOTE[$i]:-inputs unchanged} |"
  done
  echo
  if [ "${MODE}" = publish ]; then
    echo "Tags pushed: \`${SHA_TAG}\` (immutable) and \`${MOVING_TAG}\` (moving)."
    echo "Nothing here writes to git. Renovate opens the digest-bump PR on its next run."
  else
    echo "Verify mode: nothing was pushed and no registry credential was present."
  fi
} >>"${SUMMARY}"
log "done"