feat(images): build on rootless BuildKit over mTLS; verification moves into Containerfile verify stages
Operator decision on PR #81: no CI job may reach a privileged daemon. The act runner is untouched. - platform/buildkit (+ clusters/base/buildkit.yaml): moby/buildkit v0.34.1-rootless pinned by digest, uid 1000, not privileged; caps drop ALL + SETUID/SETGID, seccomp/AppArmor Unconfined, read-only rootfs (each measured under podman). cert-manager CA + server/client certs, ClusterIP :1234, default-deny with ingress only from the act-runner pod (DinD masquerades jobs to it). No Kyverno exception: every applied policy matches app-namespace=true only. - Each Containerfile names its shipped stage and ends in a `verify` stage holding every former `docker run` assertion (uid 65532, RUN --network=none, writable-path inventory standing in for the read-only rootfs). The old `&& echo` checks that could never fail are real assertions now. - images.sh drives buildkitd with a pinned buildx (sha256-checked) via bake; PRs build verify targets with agent-runner FROM the in-run toolbox (target: context); main verifies, then pushes. Fails loudly when the builder secrets are absent or buildkitd is unreachable. - infra/gitea/buildkit-client-secret.sh copies the client cert into Actions secrets. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed
1 parent
65a7769f55
commit
310deb0f7d
28 files changed
+1490
-608
No files matched your search
@@ -50,7 +50,10 @@
|
||||
# default is deliberately EMPTY so a build that forgets it fails loudly instead of
|
||||
# resolving some other image named `agent-base`.
|
||||
ARG AGENT_BASE_IMAGE=
|
||||
FROM ${AGENT_BASE_IMAGE}
|
||||
# Stage `agent-runner` is the image that ships; stage `verify` (end of file) is a TEST stage built
|
||||
# FROM it that only runs assertions and is never pushed. build-image.sh and the CI pipeline
|
||||
# build `--target verify` to prove the image and `--target agent-runner` for the image itself.
|
||||
FROM ${AGENT_BASE_IMAGE} AS agent-runner
|
||||
|
||||
# Re-declared AFTER the FROM: an ARG from before the first FROM is not in scope in the
|
||||
# build stage, so the provenance layer below would otherwise record an empty base.
|
||||
@@ -208,3 +211,80 @@ WORKDIR /workspace
|
||||
# paths above; an ENTRYPOINT here would either be ignored or prepend something the policy
|
||||
# cannot see. `bash` is the interactive fallback for a human debugging the image locally.
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
# =============================================================================
|
||||
# verify — a TEST stage, never shipped. Every assertion infra/agent/build-image.sh used to make
|
||||
# with `run --user 65532 --read-only --tmpfs /workspace --tmpfs /tmp --network none` now runs
|
||||
# here, at build time, because CI builds on rootless BuildKit where no container runtime exists.
|
||||
#
|
||||
# WHAT A BUILD STEP CAN AND CANNOT REPRODUCE, stated so nobody over-reads a green verify:
|
||||
# * uid 65532 ............ exact: USER is inherited from the agent-runner stage.
|
||||
# * --network none ....... exact: RUN --network=none, and the first check proves it.
|
||||
# * --read-only rootfs ... NOT reproducible: a RUN step always has a writable layer. It is
|
||||
# APPROXIMATED by permission: as uid 65532, every path the process can
|
||||
# write is inventoried and must lie under /workspace, /tmp or /var/tmp
|
||||
# (the pod mounts emptyDirs on the first two; /var/tmp is read-only
|
||||
# there, and TMPDIR points into /workspace so nothing should use it).
|
||||
# The pod's readOnlyRootFilesystem is enforced by admission
|
||||
# (platform/kyverno/admission/pm-agent-job-shape.yaml), not by this.
|
||||
# * --tmpfs /workspace ... approximated by the image's own /workspace (owned 65532), which is
|
||||
# what the pod's emptyDir replaces.
|
||||
# The probe script is COPYed into THIS stage only, so it never ships in the image.
|
||||
# =============================================================================
|
||||
FROM agent-runner AS verify
|
||||
COPY probe-validation.sh /opt/verify/probe-validation.sh
|
||||
RUN --network=none /bin/bash -euo pipefail <<'VERIFY'
|
||||
echo "--- no network (the build-time stand-in for run --network none) ---"
|
||||
if getent hosts registry.npmjs.org >/dev/null 2>&1 || curl -sS --max-time 3 -o /dev/null https://1.1.1.1 2>/dev/null; then
|
||||
echo "UNEXPECTED: network reachable in the verify stage"; exit 1
|
||||
fi
|
||||
echo " confirmed: no network"
|
||||
echo "--- id ---"; id
|
||||
echo "--- HOME ---"; echo "HOME=${HOME} CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR}"
|
||||
echo "--- baked versions ---"; cat /etc/agent-runner-versions
|
||||
echo "--- the gate toolchain the worker needs (CLAUDE.md §3) ---"
|
||||
for t in git python3 bash yamllint check-jsonschema kyverno promtool chainsaw flux sops age kubectl jq diff; do
|
||||
if command -v "$t" >/dev/null 2>&1; then printf " ok %s\n" "$t";
|
||||
else printf " MISSING %s\n" "$t"; missing=1; fi
|
||||
done
|
||||
[ -z "${missing:-}" ] || { echo "the gate cannot run in this image"; exit 1; }
|
||||
echo "--- python can import the renderer dependency ---"
|
||||
python3 -c "import yaml; print('PyYAML', yaml.__version__)"
|
||||
echo "--- claude ---"; claude --version
|
||||
echo "--- the two entrypoints refuse arguments ---"
|
||||
# Each must REFUSE. (The old docker-run version wrote `cmd | grep -q ... && echo`, which under
|
||||
# `set -e` + pipefail could never fail the script; these are real assertions.)
|
||||
for e in pm-worker pm-dispatch; do
|
||||
out="$(/usr/local/bin/$e --anything 2>&1 || true)"
|
||||
grep -q "takes no arguments" <<<"$out" || { echo "UNEXPECTED: $e accepted an argument: $out"; exit 1; }
|
||||
echo " $e: rejected"
|
||||
done
|
||||
echo "--- the dispatcher selftest (branch vocabulary, result envelopes, Job template, persona) ---"
|
||||
PM_SELFTEST=1 /usr/local/bin/pm-dispatch
|
||||
echo "--- the entrypoints are root-owned and NOT writable by the runtime uid ---"
|
||||
for f in /usr/local/bin/pm-worker /usr/local/bin/pm-dispatch /etc/pm-agent/worker-job-template.yaml; do
|
||||
stat -c " %n %U:%G %a" "$f"
|
||||
if [ -w "$f" ]; then echo "UNEXPECTED: uid $(id -u) can write $f"; exit 1; fi
|
||||
done
|
||||
echo " confirmed: not writable by uid $(id -u)"
|
||||
echo "--- the root filesystem is not writable by the runtime uid (expected: a failure) ---"
|
||||
if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: / is writable by uid $(id -u)"; exit 1; fi
|
||||
echo " confirmed: / is not writable"
|
||||
echo "--- everything uid $(id -u) CAN write lies under /workspace, /tmp or /var/tmp ---"
|
||||
stray="$(find / -xdev \( -path /proc -o -path /sys -o -path /dev \) -prune -o -writable ! -type l -print 2>/dev/null \
|
||||
| grep -vE '^/(workspace|tmp|var/tmp)(/|$)' || true)"
|
||||
if [ -n "${stray}" ]; then echo "UNEXPECTED: writable outside the allowed paths:"; printf ' %s\n' ${stray} | head -20; exit 1; fi
|
||||
echo " confirmed: no writable path outside /workspace, /tmp, /var/tmp"
|
||||
echo "--- the two writable paths really are writable ---"
|
||||
touch "${HOME}/.probe" /tmp/.probe || { echo "UNEXPECTED: ${HOME} or /tmp not writable"; exit 1; }
|
||||
echo " confirmed: ${HOME} and /tmp are writable"
|
||||
echo "--- uid is non-zero ---"
|
||||
[ "$(id -u)" -ne 0 ] || { echo "UNEXPECTED: running as root"; exit 1; }
|
||||
echo " confirmed: uid $(id -u), not root"
|
||||
echo "--- no Anthropic API key is baked into the image (it would outrank the subscription token) ---"
|
||||
[ -z "${ANTHROPIC_API_KEY:-}${ANTHROPIC_AUTH_TOKEN:-}" ] || { echo "UNEXPECTED: an Anthropic key is baked in"; exit 1; }
|
||||
echo " confirmed: neither is set"
|
||||
echo "--- pm-worker's input validation, with the interpreter and script that run in the cluster ---"
|
||||
PM_WORKER_PATH=/usr/local/bin/pm-worker /bin/bash /opt/verify/probe-validation.sh
|
||||
echo "AGENT-RUNNER VERIFIED"
|
||||
VERIFY
|
||||
+39
-91
@@ -2,7 +2,8 @@
|
||||
###############################################################################
|
||||
# infra/agent/build-image.sh — build the pinned `agent-runner` image and prove it works.
|
||||
#
|
||||
# ./build-image.sh # build + verify + print the image id
|
||||
# ./build-image.sh # build the verify target, then the image
|
||||
# ./build-image.sh --print-build-args # print KEY=VALUE build args (what CI consumes)
|
||||
# ./build-image.sh --no-verify # build only (skip the in-image proof)
|
||||
# ENGINE=docker ./build-image.sh # build with docker instead of podman
|
||||
# AGENT_BASE_IMAGE=home-k3s-dev:local ./build-image.sh
|
||||
@@ -11,12 +12,12 @@
|
||||
# IMAGE_TAG=foo ./build-image.sh # override the local tag (CI uses this)
|
||||
#
|
||||
# THIS SCRIPT NEVER PUSHES — and nobody pushes this image from a workstation either.
|
||||
# Publishing is done by CI: .gitea/workflows/images.yaml calls this script, and on a push to
|
||||
# `main` it pushes what this script built AND verified to the Gitea registry, records the
|
||||
# registry-returned digest in the job summary, and Renovate then opens the pull request that
|
||||
# moves platform/pm-agent/ onto that digest. So this script's job is to build, and to prove
|
||||
# the toolchain and both entrypoints work INSIDE the image as the non-root uid with the
|
||||
# read-only root filesystem the admission policy forces — the same proof locally and in CI.
|
||||
# Publishing is done by CI: .gitea/workflows/images.yaml builds the SAME Containerfile on
|
||||
# rootless BuildKit (platform/buildkit/), taking its build args from `--print-build-args`, and
|
||||
# on a push to `main` pushes it to the Gitea registry and records the registry-returned digest
|
||||
# in the job summary. Renovate then opens the pull request that moves platform/pm-agent/ onto
|
||||
# that digest. This script is the LOCAL path (podman by default); its proof — the
|
||||
# Containerfile's `verify` stage — is the same one CI runs.
|
||||
#
|
||||
# VERSIONS ARE NOT DEFINED HERE. infra/agent/versions.env holds the pins this image adds
|
||||
# (base image, node, Claude Code CLI, model) and infra/devcontainer/versions.env holds the
|
||||
@@ -26,11 +27,10 @@
|
||||
# kept current by Renovate. An AGENT_BASE_IMAGE in the environment overrides it — CI passes
|
||||
# the toolbox it built in the same run, and a developer can pass `home-k3s-dev:local`.
|
||||
#
|
||||
# ENGINE PORTABILITY. CI runs this with ENGINE=docker against a Docker-in-Docker daemon, so
|
||||
# nothing below may be podman-only (`image exists`, `--digestfile`, `{{.Digest}}`), and
|
||||
# nothing may BIND-MOUNT a file from this checkout into a container: under DinD the daemon
|
||||
# resolves bind sources on ITS filesystem, where this checkout does not exist, and silently
|
||||
# mounts an empty directory instead. Files go in over stdin.
|
||||
# NO `run` STEP. Verification used to `run` the image as uid 65532 with a read-only rootfs.
|
||||
# CI has no container runtime (rootless BuildKit only), so the assertions moved into the
|
||||
# Containerfile's `verify` stage and this script builds that target. `--print-build-args`
|
||||
# prints the pins as KEY=VALUE for the CI pipeline; AGENT_BASE_IMAGE is the caller's choice.
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
@@ -39,9 +39,11 @@ REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
|
||||
ENGINE="${ENGINE:-podman}"
|
||||
|
||||
VERIFY=1
|
||||
PRINT_ARGS=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-verify) VERIFY=0 ;;
|
||||
--print-build-args) PRINT_ARGS=1 ;;
|
||||
-h|--help) sed -n '2,/^####/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -51,8 +53,6 @@ log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# --- pins: sourced, never invented -------------------------------------------
|
||||
AGENT_ENV="${HERE}/versions.env"
|
||||
[ -f "${AGENT_ENV}" ] || die "missing ${AGENT_ENV} — the pins live there, not here"
|
||||
@@ -68,6 +68,17 @@ source "${AGENT_ENV}"
|
||||
: "${AGENT_MODEL:?AGENT_MODEL not set in ${AGENT_ENV}}"
|
||||
: "${AGENT_IMAGE_REPO:?AGENT_IMAGE_REPO not set in ${AGENT_ENV}}"
|
||||
|
||||
# THE build args, in ONE place; `--print-build-args` hands them to CI (which builds on rootless
|
||||
# BuildKit and supplies its own base reference). AGENT_BASE_IMAGE is deliberately NOT in this
|
||||
# list: the caller decides the base.
|
||||
BUILD_ARGS=(
|
||||
"NODEJS_MAJOR=${NODEJS_MAJOR}" "CLAUDE_CODE_VERSION=${CLAUDE_CODE_VERSION}"
|
||||
"CLAUDE_CODE_MIN_VERSION=${CLAUDE_CODE_MIN_VERSION}" "AGENT_MODEL=${AGENT_MODEL}"
|
||||
)
|
||||
if [ "${PRINT_ARGS}" -eq 1 ]; then printf '%s\n' "${BUILD_ARGS[@]}"; exit 0; fi
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# --- the base must exist, locally or in the registry ---------------------------
|
||||
# Deliberately NOT built on demand. The toolbox is its own reviewed artefact with its own
|
||||
# pins, and silently building it here would mean this script could change the toolchain a
|
||||
@@ -103,89 +114,26 @@ info " model ${AGENT_MODEL}"
|
||||
# Provenance: CI passes the commit it is building, so a pulled image names its source tree.
|
||||
LABEL_ARGS=()
|
||||
[ -z "${BUILD_REVISION:-}" ] || LABEL_ARGS+=(--label "org.opencontainers.image.revision=${BUILD_REVISION}")
|
||||
|
||||
"${ENGINE}" build "${LABEL_ARGS[@]}" \
|
||||
--file "${HERE}/Containerfile" \
|
||||
--tag "${IMAGE_REF}" \
|
||||
--build-arg "AGENT_BASE_IMAGE=${AGENT_BASE_IMAGE}" \
|
||||
--build-arg "NODEJS_MAJOR=${NODEJS_MAJOR}" \
|
||||
--build-arg "CLAUDE_CODE_VERSION=${CLAUDE_CODE_VERSION}" \
|
||||
--build-arg "CLAUDE_CODE_MIN_VERSION=${CLAUDE_CODE_MIN_VERSION}" \
|
||||
--build-arg "AGENT_MODEL=${AGENT_MODEL}" \
|
||||
"${HERE}"
|
||||
ARG_FLAGS=(--build-arg "AGENT_BASE_IMAGE=${AGENT_BASE_IMAGE}")
|
||||
for a in "${BUILD_ARGS[@]}"; do ARG_FLAGS+=(--build-arg "$a"); done
|
||||
|
||||
# --- verify: the point of the whole exercise ---------------------------------
|
||||
# A build that merely succeeded is not evidence this image is usable. Prove the tools
|
||||
# EXECUTE, as the NON-ROOT uid, with a READ-ONLY root filesystem and only the two volumes
|
||||
# the admission policy admits — the exact shape a worker Job runs in. Anything that works
|
||||
# here but not there is a 3am discovery.
|
||||
# A build that merely succeeded is not evidence this image is usable. The proof is the
|
||||
# Containerfile's `verify` stage: as the NON-ROOT uid, with no network, the toolchain executes,
|
||||
# both entrypoints refuse arguments, the dispatcher selftest passes, the entrypoints are not
|
||||
# writable, nothing outside /workspace and /tmp is writable, and pm-worker's input guards
|
||||
# (probe-validation.sh) refuse every hostile input. It is a build target rather than a
|
||||
# `run` so the SAME proof runs here and in CI, where no container runtime exists — see the
|
||||
# stage's header for what a build step can and cannot reproduce of the pod's shape.
|
||||
if [ "${VERIFY}" -eq 1 ]; then
|
||||
log "Verifying inside the image (uid 65532, read-only rootfs, workspace+tmp only)"
|
||||
"${ENGINE}" run --rm \
|
||||
--user 65532:65532 \
|
||||
--read-only \
|
||||
--tmpfs /workspace:rw,mode=1777 \
|
||||
--tmpfs /tmp:rw,mode=1777 \
|
||||
--network none \
|
||||
--entrypoint "" \
|
||||
"${IMAGE_REF}" \
|
||||
/bin/bash -euo pipefail -c '
|
||||
echo "--- id ---"; id
|
||||
echo "--- HOME ---"; echo "HOME=${HOME} CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR}"
|
||||
echo "--- baked versions ---"; cat /etc/agent-runner-versions
|
||||
echo "--- the gate toolchain the worker needs (CLAUDE.md §3) ---"
|
||||
for t in git python3 bash yamllint check-jsonschema kyverno promtool chainsaw flux sops age kubectl jq diff; do
|
||||
if command -v "$t" >/dev/null 2>&1; then printf " ok %s\n" "$t";
|
||||
else printf " MISSING %s\n" "$t"; missing=1; fi
|
||||
done
|
||||
[ -z "${missing:-}" ] || { echo "the gate cannot run in this image"; exit 1; }
|
||||
echo "--- python can import the renderer dependency ---"
|
||||
python3 -c "import yaml; print(\"PyYAML\", yaml.__version__)"
|
||||
echo "--- claude ---"; claude --version
|
||||
echo "--- the two entrypoints refuse arguments ---"
|
||||
/usr/local/bin/pm-worker --anything 2>&1 | grep -q "takes no arguments" && echo " pm-worker: rejected"
|
||||
/usr/local/bin/pm-dispatch --anything 2>&1 | grep -q "takes no arguments" && echo " pm-dispatch: rejected"
|
||||
echo "--- the dispatcher selftest (branch vocabulary, result envelopes, Job template, persona) ---"
|
||||
PM_SELFTEST=1 /usr/local/bin/pm-dispatch
|
||||
echo "--- the entrypoints are root-owned and NOT writable by the runtime uid ---"
|
||||
for f in /usr/local/bin/pm-worker /usr/local/bin/pm-dispatch /etc/pm-agent/worker-job-template.yaml; do
|
||||
stat -c " %n %U:%G %a" "$f"
|
||||
if [ -w "$f" ]; then echo "UNEXPECTED: uid $(id -u) can write $f"; exit 1; fi
|
||||
done
|
||||
echo " confirmed: not writable by uid $(id -u)"
|
||||
echo "--- rootfs is read-only (expected: a failure) ---"
|
||||
if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: rootfs is writable"; exit 1; fi
|
||||
echo " confirmed: / is read-only"
|
||||
echo "--- the two writable paths really are writable ---"
|
||||
touch "${HOME}/.probe" /tmp/.probe && echo " confirmed: ${HOME} and /tmp are writable"
|
||||
echo "--- uid is non-zero ---"
|
||||
[ "$(id -u)" -ne 0 ] && echo " confirmed: uid $(id -u), not root"
|
||||
echo "--- no Anthropic API key is baked into the image (it would outrank the subscription token) ---"
|
||||
[ -z "${ANTHROPIC_API_KEY:-}${ANTHROPIC_AUTH_TOKEN:-}" ] && echo " confirmed: neither is set"
|
||||
'
|
||||
# The worker's input guards, exercised in the image, with the interpreter and the script
|
||||
# that will actually run in the cluster. The probe is mounted read-only rather than baked
|
||||
# in: it is a build-time check, not something a worker Job has any business carrying.
|
||||
log "Probing pm-worker's input validation inside the image"
|
||||
#
|
||||
# The probe goes in over STDIN, not as a bind mount: under Docker-in-Docker (CI) a bind
|
||||
# source is resolved on the daemon's filesystem, where this file does not exist, and the
|
||||
# container would get an empty directory. PM_WORKER_PATH points it at the baked worker,
|
||||
# since the probe's own default (a sibling of the probe file) has no meaning on stdin.
|
||||
"${ENGINE}" run --rm -i \
|
||||
--user 65532:65532 \
|
||||
--read-only \
|
||||
--tmpfs /workspace:rw,mode=1777 \
|
||||
--tmpfs /tmp:rw,mode=1777 \
|
||||
--network none \
|
||||
-e PM_WORKER_PATH=/usr/local/bin/pm-worker \
|
||||
--entrypoint "" \
|
||||
"${IMAGE_REF}" \
|
||||
/bin/bash -s < "${HERE}/probe-validation.sh"
|
||||
|
||||
log "Verifying (Containerfile target: verify)"
|
||||
"${ENGINE}" build "${ARG_FLAGS[@]}" --target verify --file "${HERE}/Containerfile" "${HERE}"
|
||||
log "Image verified"
|
||||
fi
|
||||
|
||||
"${ENGINE}" build "${LABEL_ARGS[@]}" "${ARG_FLAGS[@]}" --target agent-runner \
|
||||
--file "${HERE}/Containerfile" --tag "${IMAGE_REF}" "${HERE}"
|
||||
|
||||
# --- result ------------------------------------------------------------------
|
||||
# No digest is printed: a manifest digest does not exist until a registry has the manifest,
|
||||
# and the only digest anything may pin is the one the REGISTRY returns. CI captures that at
|
||||
|
||||
@@ -16,14 +16,14 @@
|
||||
# scrubbed environment and fake credentials, and if a guard ever stopped firing, the case
|
||||
# would fail here rather than by really cloning something.
|
||||
#
|
||||
# Run offline, no cluster, no Gitea, no model. infra/agent/build-image.sh runs it inside the
|
||||
# built image as well, where the interpreter and the script are the ones that will actually
|
||||
# run in the cluster.
|
||||
# Run offline, no cluster, no Gitea, no model. The Containerfile's `verify` stage runs it inside
|
||||
# the built image as well (as uid 65532, no network), where the interpreter and the script are
|
||||
# the ones that will actually run in the cluster — locally via build-image.sh and in CI.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
# `:-$0` because build-image.sh feeds this script over STDIN (`bash -s`), where BASH_SOURCE is
|
||||
# empty and `set -u` would abort on it; PM_WORKER_PATH is always set in that case.
|
||||
# `:-$0` keeps this safe if it is ever fed over stdin (`bash -s`), where BASH_SOURCE is empty
|
||||
# and `set -u` would abort on it.
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
|
||||
WORKER="${PM_WORKER_PATH:-${HERE}/pm-worker}"
|
||||
[ -r "$WORKER" ] || { echo "cannot read ${WORKER}" >&2; exit 2; }
|
||||
|
||||
@@ -8,7 +8,10 @@
|
||||
# packages are retained as harmless disk-image utilities. Fedora base is pinned for
|
||||
# reproducibility.
|
||||
ARG FEDORA_VERSION=44
|
||||
FROM registry.fedoraproject.org/fedora:${FEDORA_VERSION}
|
||||
# Stage `toolbox` is the image that ships. Stage `verify` (end of file) is a TEST stage: it is
|
||||
# built FROM the finished image and only runs assertions, and it is never pushed. Build with
|
||||
# `--target toolbox` for the image and `--target verify` to prove it (build-image.sh does both).
|
||||
FROM registry.fedoraproject.org/fedora:${FEDORA_VERSION} AS toolbox
|
||||
|
||||
ARG OPENTOFU_VERSION
|
||||
ARG AGE_VERSION
|
||||
@@ -95,3 +98,48 @@ RUN { echo "opentofu=${OPENTOFU_VERSION}"; echo "age=${AGE_VERSION}"; echo "sops
|
||||
|
||||
WORKDIR /work
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
# =============================================================================
|
||||
# verify — a TEST stage, never shipped. It runs every assertion against the finished image AT
|
||||
# BUILD TIME, so CI needs no container runtime to prove the image: the image pipeline builds
|
||||
# this target on rootless BuildKit (platform/buildkit/), where `docker run` does not exist.
|
||||
# `RUN --network=none` reproduces the old `run --network none` exactly, and the first check
|
||||
# below proves it (it fails if any network is reachable).
|
||||
# =============================================================================
|
||||
FROM toolbox AS verify
|
||||
ARG PROMTOOL_VERSION
|
||||
ARG KUBECTL_VERSION
|
||||
ARG CHAINSAW_VERSION
|
||||
ARG KYVERNO_VERSION
|
||||
ARG SOPS_VERSION
|
||||
RUN --network=none /bin/bash -euo pipefail <<'VERIFY'
|
||||
echo "--- no network (the build-time stand-in for run --network none) ---"
|
||||
if getent hosts registry.fedoraproject.org >/dev/null 2>&1 || curl -sS --max-time 3 -o /dev/null https://1.1.1.1 2>/dev/null; then
|
||||
echo "UNEXPECTED: network reachable in the verify stage"; exit 1
|
||||
fi
|
||||
echo " confirmed: no network"
|
||||
echo "--- baked versions ---"; cat /etc/dev-toolchain-versions
|
||||
echo "--- every tool is on PATH ---"
|
||||
for t in tofu age age-keygen sops flux kubectl helm kyverno kyverno-json chainsaw yq \
|
||||
yamllint check-jsonschema promtool ansible-playbook git make jq diff python3 \
|
||||
wg qrencode; do
|
||||
if command -v "$t" >/dev/null 2>&1; then printf " ok %s\n" "$t";
|
||||
else printf " MISSING %s\n" "$t"; missing=1; fi
|
||||
done
|
||||
[ -z "${missing:-}" ] || { echo "toolbox is missing tools"; exit 1; }
|
||||
echo "--- version-sensitive tools report their pins ---"
|
||||
: "${PROMTOOL_VERSION:?}" "${KUBECTL_VERSION:?}" "${CHAINSAW_VERSION:?}" "${KYVERNO_VERSION:?}" "${SOPS_VERSION:?}"
|
||||
check() { # name pin actual-output
|
||||
if printf "%s" "$3" | grep -qF "$2"; then printf " ok %-10s %s\n" "$1" "$2";
|
||||
else printf " DRIFT %-10s want %s, got: %s\n" "$1" "$2" "$(printf "%s" "$3" | head -1)"; drift=1; fi
|
||||
}
|
||||
check promtool "${PROMTOOL_VERSION}" "$(promtool --version 2>&1)"
|
||||
check kubectl "${KUBECTL_VERSION}" "$(kubectl version --client 2>&1)"
|
||||
check chainsaw "${CHAINSAW_VERSION}" "$(chainsaw version 2>&1)"
|
||||
check kyverno "${KYVERNO_VERSION}" "$(kyverno version 2>&1)"
|
||||
check sops "${SOPS_VERSION}" "$(sops --version --disable-version-check 2>&1 || sops --version 2>&1)"
|
||||
[ -z "${drift:-}" ] || { echo "a tool does not report its pinned version"; exit 1; }
|
||||
echo "--- python can import what the renderer and the gate need ---"
|
||||
python3 -c "import yaml; print(' PyYAML', yaml.__version__)"
|
||||
echo "TOOLBOX VERIFIED"
|
||||
VERIFY
|
||||
@@ -36,9 +36,11 @@ cd infra/devcontainer
|
||||
|
||||
### The published toolbox
|
||||
|
||||
`./dev.sh build` delegates to [`build-image.sh`](build-image.sh), which is the **one** build
|
||||
recipe: CI runs the same script (with its in-image verification — every tool executes, and the
|
||||
version-sensitive ones report their `versions.env` pins) and, on a push to `main` that changes
|
||||
`./dev.sh build` delegates to [`build-image.sh`](build-image.sh). The recipe is the
|
||||
`Containerfile` plus that script's build-arg list, and the proof is the Containerfile's `verify`
|
||||
stage (every tool on `PATH`, the version-sensitive ones report their `versions.env` pins, no
|
||||
network). CI builds the same Containerfile on rootless BuildKit with the args from
|
||||
`build-image.sh --print-build-args`, and, on a push to `main` that changes
|
||||
`Containerfile`, `versions.env` or `build-image.sh`, **publishes** the result as
|
||||
`gitea.home.keithjackson.dev/home-network/toolbox` (tags `sha-<commit>` and `main`) —
|
||||
see [`.gitea/workflows/images.yaml`](../../.gitea/workflows/images.yaml). `agent-runner` is
|
||||
|
||||
@@ -6,12 +6,14 @@
|
||||
# ./build-image.sh --no-verify # build only
|
||||
# IMAGE=foo:bar ./build-image.sh # build under another local name
|
||||
# ENGINE=docker ./build-image.sh # build with docker instead of podman
|
||||
# ./build-image.sh --print-build-args # print KEY=VALUE build args (what CI consumes)
|
||||
#
|
||||
# ONE BUILD DEFINITION, TWO CALLERS. `dev.sh build` calls this for local development, and the
|
||||
# CI image pipeline (.gitea/workflows/images.yaml -> .gitea/workflows/scripts/images.sh) calls
|
||||
# it to build the image it then PUBLISHES as gitea.home.keithjackson.dev/home-network/toolbox.
|
||||
# Keeping the build args in exactly one place is what makes "the toolbox CI publishes" and "the
|
||||
# toolbox you gate with locally" the same recipe.
|
||||
# ONE BUILD DEFINITION, TWO CALLERS. `dev.sh build` calls this for local development. The CI
|
||||
# image pipeline (.gitea/workflows/images.yaml -> .gitea/workflows/scripts/images.sh) builds the
|
||||
# SAME Containerfile on rootless BuildKit (platform/buildkit/) — no container engine exists in a
|
||||
# CI job — taking its build args from `--print-build-args`, and PUBLISHES the result as
|
||||
# gitea.home.keithjackson.dev/home-network/toolbox. The Containerfile (including its `verify`
|
||||
# test stage) and the arg list here are the one recipe both use.
|
||||
#
|
||||
# THIS SCRIPT NEVER PUSHES. Publishing is CI's job (images.yaml, on push to main), not a
|
||||
# workstation's: an image that exists only because someone ran `podman push` from a laptop has
|
||||
@@ -26,9 +28,11 @@ ENGINE="${ENGINE:-podman}"
|
||||
IMAGE="${IMAGE:-home-k3s-dev:local}"
|
||||
|
||||
VERIFY=1
|
||||
PRINT_ARGS=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-verify) VERIFY=0 ;;
|
||||
--print-build-args) PRINT_ARGS=1 ;;
|
||||
-h|--help) sed -n '2,/^####/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -37,72 +41,44 @@ done
|
||||
log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "${HERE}/versions.env"
|
||||
|
||||
# THE build args, in ONE place. `--print-build-args` hands exactly this list to the CI pipeline
|
||||
# (.gitea/workflows/scripts/images.sh), which builds on rootless BuildKit rather than calling
|
||||
# "$ENGINE build" — so CI and a local build cannot drift apart on a pin.
|
||||
BUILD_ARGS=(
|
||||
"FEDORA_VERSION=$FEDORA_VERSION" "OPENTOFU_VERSION=$OPENTOFU_VERSION" "AGE_VERSION=$AGE_VERSION"
|
||||
"SOPS_VERSION=$SOPS_VERSION" "FLUX_VERSION=$FLUX_VERSION" "KUBECTL_VERSION=$KUBECTL_VERSION"
|
||||
"HELM_VERSION=$HELM_VERSION" "KYVERNO_VERSION=$KYVERNO_VERSION"
|
||||
"KYVERNO_JSON_VERSION=$KYVERNO_JSON_VERSION" "CHAINSAW_VERSION=$CHAINSAW_VERSION"
|
||||
"YAMLLINT_VERSION=$YAMLLINT_VERSION" "YQ_VERSION=$YQ_VERSION"
|
||||
"CHECK_JSONSCHEMA_VERSION=$CHECK_JSONSCHEMA_VERSION" "PROMTOOL_VERSION=$PROMTOOL_VERSION"
|
||||
)
|
||||
if [ "${PRINT_ARGS}" -eq 1 ]; then printf '%s\n' "${BUILD_ARGS[@]}"; exit 0; fi
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
ARG_FLAGS=()
|
||||
for a in "${BUILD_ARGS[@]}"; do ARG_FLAGS+=(--build-arg "$a"); done
|
||||
# Provenance: CI passes the commit it is building, so a pulled image names its source tree.
|
||||
LABEL_ARGS=()
|
||||
[ -z "${BUILD_REVISION:-}" ] || LABEL_ARGS+=(--label "org.opencontainers.image.revision=${BUILD_REVISION}")
|
||||
|
||||
log "Building ${IMAGE}"
|
||||
"$ENGINE" build "${LABEL_ARGS[@]}" \
|
||||
--build-arg FEDORA_VERSION="$FEDORA_VERSION" \
|
||||
--build-arg OPENTOFU_VERSION="$OPENTOFU_VERSION" \
|
||||
--build-arg AGE_VERSION="$AGE_VERSION" \
|
||||
--build-arg SOPS_VERSION="$SOPS_VERSION" \
|
||||
--build-arg FLUX_VERSION="$FLUX_VERSION" \
|
||||
--build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
|
||||
--build-arg HELM_VERSION="$HELM_VERSION" \
|
||||
--build-arg KYVERNO_VERSION="$KYVERNO_VERSION" \
|
||||
--build-arg KYVERNO_JSON_VERSION="$KYVERNO_JSON_VERSION" \
|
||||
--build-arg CHAINSAW_VERSION="$CHAINSAW_VERSION" \
|
||||
--build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \
|
||||
--build-arg YQ_VERSION="$YQ_VERSION" \
|
||||
--build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \
|
||||
--build-arg PROMTOOL_VERSION="$PROMTOOL_VERSION" \
|
||||
-t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
|
||||
|
||||
# --- verify ------------------------------------------------------------------
|
||||
# A build that succeeded is not evidence the toolbox can run the gate. Every tool the gate and
|
||||
# the runbooks call must EXECUTE, and the version-sensitive ones must report the version
|
||||
# versions.env pins — promtool above all, because ci/versions.env asserts a promtool/Prometheus
|
||||
# lockstep and a drifted engine evaluates alert-rule tests differently (see versions.env).
|
||||
# --- verify --------------------------------------------------------------------
|
||||
# The assertions live in the Containerfile's `verify` stage (every tool on PATH, the
|
||||
# version-sensitive ones report their versions.env pins — promtool above all, because
|
||||
# ci/versions.env asserts a promtool/Prometheus lockstep — and no network), so the SAME proof
|
||||
# runs here under podman and in CI under rootless BuildKit. A failing assertion fails the build.
|
||||
if [ "${VERIFY}" -eq 1 ]; then
|
||||
log "Verifying the toolchain inside ${IMAGE}"
|
||||
"$ENGINE" run --rm --network none --entrypoint "" \
|
||||
-e PROMTOOL_VERSION="${PROMTOOL_VERSION}" \
|
||||
-e KUBECTL_VERSION="${KUBECTL_VERSION}" \
|
||||
-e CHAINSAW_VERSION="${CHAINSAW_VERSION}" \
|
||||
-e KYVERNO_VERSION="${KYVERNO_VERSION}" \
|
||||
-e SOPS_VERSION="${SOPS_VERSION}" \
|
||||
"$IMAGE" /bin/bash -euo pipefail -c '
|
||||
echo "--- baked versions ---"; cat /etc/dev-toolchain-versions
|
||||
echo "--- every tool is on PATH ---"
|
||||
for t in tofu age age-keygen sops flux kubectl helm kyverno kyverno-json chainsaw yq \
|
||||
yamllint check-jsonschema promtool ansible-playbook git make jq diff python3 \
|
||||
wg qrencode; do
|
||||
if command -v "$t" >/dev/null 2>&1; then printf " ok %s\n" "$t";
|
||||
else printf " MISSING %s\n" "$t"; missing=1; fi
|
||||
done
|
||||
[ -z "${missing:-}" ] || { echo "toolbox is missing tools"; exit 1; }
|
||||
echo "--- version-sensitive tools report their pins ---"
|
||||
check() { # name pin actual-output
|
||||
if printf "%s" "$3" | grep -qF "$2"; then printf " ok %-10s %s\n" "$1" "$2";
|
||||
else printf " DRIFT %-10s want %s, got: %s\n" "$1" "$2" "$(printf "%s" "$3" | head -1)"; drift=1; fi
|
||||
}
|
||||
check promtool "${PROMTOOL_VERSION}" "$(promtool --version 2>&1)"
|
||||
check kubectl "${KUBECTL_VERSION}" "$(kubectl version --client 2>&1)"
|
||||
check chainsaw "${CHAINSAW_VERSION}" "$(chainsaw version 2>&1)"
|
||||
check kyverno "${KYVERNO_VERSION}" "$(kyverno version 2>&1)"
|
||||
check sops "${SOPS_VERSION}" "$(sops --version --disable-version-check 2>&1 || sops --version 2>&1)"
|
||||
[ -z "${drift:-}" ] || { echo "a tool does not report its pinned version"; exit 1; }
|
||||
echo "--- python can import what the renderer and the gate need ---"
|
||||
python3 -c "import yaml; print(\" PyYAML\", yaml.__version__)"
|
||||
'
|
||||
log "Verifying (Containerfile target: verify)"
|
||||
"$ENGINE" build "${ARG_FLAGS[@]}" --target verify -f "${HERE}/Containerfile" "${HERE}"
|
||||
log "Toolbox verified"
|
||||
fi
|
||||
|
||||
log "Building ${IMAGE}"
|
||||
"$ENGINE" build "${LABEL_ARGS[@]}" "${ARG_FLAGS[@]}" --target toolbox \
|
||||
-t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
|
||||
|
||||
log "Built ${IMAGE}"
|
||||
printf ' image id %s\n' "$("$ENGINE" image inspect --format '{{.Id}}' "$IMAGE")"
|
||||
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copy the image builder's CLIENT certificate from the cluster into the three repository Actions
|
||||
# secrets the image pipeline reads: BUILDKIT_CA, BUILDKIT_CLIENT_CERT, BUILDKIT_CLIENT_KEY.
|
||||
#
|
||||
# ./infra/gitea/buildkit-client-secret.sh # copy and verify
|
||||
# ./infra/gitea/buildkit-client-secret.sh --check # compare only; changes nothing
|
||||
#
|
||||
# RUNNING IT IS AN OPERATOR ACTION. It writes repository secrets.
|
||||
#
|
||||
# WHERE THE CERTIFICATE COMES FROM. platform/buildkit/certificates.yaml has cert-manager issue
|
||||
# `buildkit-client-ci` from a CA that exists only for the builder; buildkitd admits a connection
|
||||
# only with a certificate signed by that CA. CI jobs hold NO cluster credential (R-SEC-02), so they
|
||||
# cannot read the Secret themselves: this script is the one bridge, run by a human who has both
|
||||
# a kube context and the Gitea credential.
|
||||
#
|
||||
# WHEN TO RUN IT
|
||||
# 1. once, after Flux has applied platform/buildkit (Kustomization `buildkit` Ready, and
|
||||
# `kubectl -n buildkit get certificate buildkit-client-ci` READY=True);
|
||||
# 2. after every renewal of that certificate (duration 1y, renewed 30 days early). Until then
|
||||
# the image pipeline fails LOUDLY at its builder preflight — it never skips.
|
||||
# `--check` tells you whether the copy is stale.
|
||||
#
|
||||
# NOTHING SECRET IS PRINTED. The key goes from `kubectl get secret` into the Gitea secrets API
|
||||
# through shell variables and nowhere else. The script prints only names, sizes and the
|
||||
# certificate's subject and expiry (public data).
|
||||
#
|
||||
# NEEDS: kubectl with a context for the cluster (default k3s-prod), jq, curl, openssl, base64,
|
||||
# and the Gitea credential `git` already has stored for the host — the same requirements as
|
||||
# infra/gitea/conformance-trigger-kubeconfig.sh, run from the same place.
|
||||
set -euo pipefail
|
||||
HOST=gitea.home.keithjackson.dev; OWNER=Home-Network; REPO=home-network-k3s-impl
|
||||
NS=buildkit; SECRET=buildkit-client-ci; KCTX="${KCTX:-k3s-prod}"
|
||||
CHECK=0
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--check) CHECK=1 ;;
|
||||
-h|--help) sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown argument: $a" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
for t in kubectl jq curl openssl base64 git; do command -v "$t" >/dev/null || die "missing required tool: $t"; done
|
||||
|
||||
echo "==> reading Secret ${NS}/${SECRET} (context ${KCTX}; values not shown)"
|
||||
js="$(kubectl --context "$KCTX" -n "$NS" get secret "$SECRET" -o json)" \
|
||||
|| die "cannot read ${NS}/${SECRET} — is platform/buildkit deployed and the Certificate Ready?"
|
||||
CA="$(jq -r '.data["ca.crt"] // empty' <<<"$js" | base64 -d)"
|
||||
CRT="$(jq -r '.data["tls.crt"] // empty' <<<"$js" | base64 -d)"
|
||||
KEY="$(jq -r '.data["tls.key"] // empty' <<<"$js" | base64 -d)"
|
||||
unset js
|
||||
[ -n "$CA" ] && [ -n "$CRT" ] && [ -n "$KEY" ] || die "${NS}/${SECRET} lacks ca.crt/tls.crt/tls.key"
|
||||
|
||||
# Sanity before anything is written: the leaf is a CLIENT cert issued by this CA, and the key
|
||||
# belongs to it. A wrong pairing would otherwise surface only as a TLS failure in CI.
|
||||
openssl verify -CAfile <(printf '%s\n' "$CA") <(printf '%s\n' "$CRT") >/dev/null 2>&1 \
|
||||
|| die "tls.crt does not verify against ca.crt"
|
||||
openssl x509 -noout -ext extendedKeyUsage <<<"$CRT" 2>/dev/null | grep -q "TLS Web Client Authentication" \
|
||||
|| die "tls.crt is not a client-auth certificate"
|
||||
[ "$(openssl x509 -noout -pubkey <<<"$CRT" | openssl sha256)" = "$(openssl pkey -pubout <<<"$KEY" 2>/dev/null | openssl sha256)" ] \
|
||||
|| die "tls.key does not match tls.crt"
|
||||
echo " $(openssl x509 -noout -subject <<<"$CRT") / $(openssl x509 -noout -enddate <<<"$CRT")"
|
||||
|
||||
creds="$(printf 'protocol=https\nhost=%s\n\n' "$HOST" | git credential fill)"
|
||||
U="$(sed -n 's/^username=//p' <<<"$creds")"; P="$(sed -n 's/^password=//p' <<<"$creds")"; unset creds
|
||||
[ -n "$U" ] && [ -n "$P" ] || die "no stored credential for $HOST"
|
||||
echo "==> authenticating to Gitea as '${U}' (secret not shown)"
|
||||
api() { curl -sS --config <(printf 'user = "%s:%s"\n' "$U" "$P") "$@"; }
|
||||
API="https://${HOST}/api/v1/repos/${OWNER}/${REPO}/actions/secrets"
|
||||
|
||||
if [ "$CHECK" -eq 1 ]; then
|
||||
# Gitea never returns a secret's value, only its name and creation time. "Stale" therefore
|
||||
# means: the secret was created before the certificate's current notBefore.
|
||||
names="$(api "$API" | jq -c '[.[]? | {name, created_at}]')"
|
||||
nb="$(date -u -d "$(openssl x509 -noout -startdate <<<"$CRT" | cut -d= -f2)" +%s)"
|
||||
rc=0
|
||||
for n in BUILDKIT_CA BUILDKIT_CLIENT_CERT BUILDKIT_CLIENT_KEY; do
|
||||
c="$(jq -r --arg n "$n" '.[] | select(.name==$n) | .created_at' <<<"$names")"
|
||||
if [ -z "$c" ]; then echo " MISSING $n"; rc=1
|
||||
elif [ "$(date -u -d "$c" +%s)" -lt "$nb" ]; then echo " STALE $n (stored $c, certificate re-issued since)"; rc=1
|
||||
else echo " current $n (stored $c)"; fi
|
||||
done
|
||||
exit "$rc"
|
||||
fi
|
||||
|
||||
put() { # name value
|
||||
local code
|
||||
code="$(api -o /dev/null -w '%{http_code}' -X PUT -H 'Content-Type: application/json' \
|
||||
-d "$(jq -n --arg d "$2" '{data:$d}')" "${API}/$1")"
|
||||
[ "$code" = 201 ] || [ "$code" = 204 ] || die "PUT $1 failed: HTTP ${code}"
|
||||
echo " stored $1 (${#2} bytes, value not shown)"
|
||||
}
|
||||
echo "==> storing repository Actions secrets"
|
||||
put BUILDKIT_CA "$CA"
|
||||
put BUILDKIT_CLIENT_CERT "$CRT"
|
||||
put BUILDKIT_CLIENT_KEY "$KEY"
|
||||
unset CA CRT KEY
|
||||
|
||||
# VERIFY BY READING BACK (names only — Gitea never returns values). curl -sS exits 0 on an HTTP
|
||||
# error, which is how set-ci-token.sh once claimed a store that never happened.
|
||||
names="$(api "$API" | jq -r '[.[]?.name] | join(" ")')"
|
||||
for n in BUILDKIT_CA BUILDKIT_CLIENT_CERT BUILDKIT_CLIENT_KEY; do
|
||||
grep -qw "$n" <<<"$names" || die "stored $n but it is NOT listed (${names}) — refusing to claim success"
|
||||
done
|
||||
echo "==> stored and VERIFIED present. Re-run the images workflow (workflow_dispatch) to use them."
|
||||
Reference in new issue
Block a user