feat(images): build on rootless BuildKit over mTLS; verification moves into Containerfile verify stages
Operator decision on PR #81: no CI job may reach a privileged daemon. The act runner is untouched. - platform/buildkit (+ clusters/base/buildkit.yaml): moby/buildkit v0.34.1-rootless pinned by digest, uid 1000, not privileged; caps drop ALL + SETUID/SETGID, seccomp/AppArmor Unconfined, read-only rootfs (each measured under podman). cert-manager CA + server/client certs, ClusterIP :1234, default-deny with ingress only from the act-runner pod (DinD masquerades jobs to it). No Kyverno exception: every applied policy matches app-namespace=true only. - Each Containerfile names its shipped stage and ends in a `verify` stage holding every former `docker run` assertion (uid 65532, RUN --network=none, writable-path inventory standing in for the read-only rootfs). The old `&& echo` checks that could never fail are real assertions now. - images.sh drives buildkitd with a pinned buildx (sha256-checked) via bake; PRs build verify targets with agent-runner FROM the in-run toolbox (target: context); main verifies, then pushes. Fails loudly when the builder secrets are absent or buildkitd is unreachable. - infra/gitea/buildkit-client-secret.sh copies the client cert into Actions secrets. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed
1 parent
65a7769f55
commit
310deb0f7d
28 files changed
+1490
-608
No files matched your search
@@ -98,7 +98,10 @@ RUN set -eux; \
|
||||
|
||||
# --- runtime -----------------------------------------------------------------
|
||||
ARG FEDORA_VERSION=44
|
||||
FROM registry.fedoraproject.org/fedora-minimal:${FEDORA_VERSION}
|
||||
# Stage `conformance-runner` is the image that ships; stage `verify` (end of file) is a TEST stage
|
||||
# built FROM it that only runs assertions and is never pushed. build-image.sh and the CI
|
||||
# pipeline build `--target verify` to prove it and `--target conformance-runner` for the image.
|
||||
FROM registry.fedoraproject.org/fedora-minimal:${FEDORA_VERSION} AS conformance-runner
|
||||
|
||||
ARG CHAINSAW_VERSION=
|
||||
ARG KUBECTL_VERSION=
|
||||
@@ -198,3 +201,86 @@ WORKDIR /workspace
|
||||
# prepend something the policy cannot see. `bash` is the interactive fallback for a human
|
||||
# debugging the image locally.
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
# =============================================================================
|
||||
# verify — a TEST stage, never shipped. Every assertion conformance/incluster/build-image.sh used
|
||||
# to make with `run --user 65532 --read-only --tmpfs /workspace` now runs here, at build time,
|
||||
# because CI builds on rootless BuildKit where no container runtime exists.
|
||||
#
|
||||
# WHAT A BUILD STEP CAN AND CANNOT REPRODUCE:
|
||||
# * uid 65532 ............ exact (USER inherited from the conformance-runner stage).
|
||||
# * no network ........... exact: RUN --network=none, and the first check proves it.
|
||||
# * --read-only rootfs ... NOT reproducible (a RUN step always has a writable layer).
|
||||
# Approximated by permission: everything uid 65532 can write must lie
|
||||
# under /workspace, /tmp or /var/tmp. In the Job only /workspace (and
|
||||
# /results) are writable — the image points HOME and TMPDIR there — and
|
||||
# readOnlyRootFilesystem is enforced by
|
||||
# platform/kyverno/admission/conformance-trigger-job-shape.yaml.
|
||||
# =============================================================================
|
||||
FROM conformance-runner AS verify
|
||||
RUN --network=none /bin/bash -euo pipefail <<'VERIFY'
|
||||
echo "--- no network (the build-time stand-in for the Job's isolation) ---"
|
||||
if getent hosts gitea.home.keithjackson.dev >/dev/null 2>&1 || timeout 3 bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
|
||||
echo "UNEXPECTED: network reachable in the verify stage"; exit 1
|
||||
fi
|
||||
echo "confirmed: no network"
|
||||
echo "--- id ---"; id
|
||||
echo "--- HOME ---"; echo "HOME=${HOME}"
|
||||
echo "--- bash ---"; bash --version | head -1
|
||||
echo "--- python3 ---"; python3 --version
|
||||
echo "--- git ---"; git --version
|
||||
echo "--- kubectl ---"; kubectl version --client
|
||||
echo "--- chainsaw ---"; chainsaw version
|
||||
echo "--- baked versions ---"; cat /etc/conformance-runner-versions
|
||||
echo "--- conformance-clone: fixed path, root-owned, NOT writable by the runner ---"
|
||||
stat -c "%n %U:%G %a" /usr/local/bin/conformance-clone
|
||||
if [ -w /usr/local/bin/conformance-clone ]; then
|
||||
echo "UNEXPECTED: uid $(id -u) can write the one executable the policy will trust"; exit 1
|
||||
fi
|
||||
echo "confirmed: not writable by uid $(id -u)"
|
||||
# Each of these MUST fail. Assert the non-zero exit as well as the message: a check that only
|
||||
# greps stderr would pass a build where the guard had stopped rejecting.
|
||||
expect_fail() {
|
||||
local label="$1"; shift
|
||||
local out rc=0
|
||||
out="$(env "$@" 2>&1)" || rc=$?
|
||||
echo "--- ${label} ---"
|
||||
printf "%s\n" "${out}" | head -2
|
||||
if [ "${rc}" -eq 0 ]; then
|
||||
echo "UNEXPECTED: that should have failed, but it exited 0"; exit 1
|
||||
fi
|
||||
echo "(rejected, exit ${rc})"
|
||||
}
|
||||
expect_fail "conformance-clone rejects arguments (that is where an injection would live)" \
|
||||
conformance-clone --layer l2
|
||||
expect_fail "conformance-clone rejects a missing SHA" \
|
||||
conformance-clone
|
||||
expect_fail "conformance-clone rejects a branch name instead of a SHA (no silent run of the tip)" \
|
||||
CONFORMANCE_GIT_SHA=main conformance-clone
|
||||
expect_fail "conformance-clone rejects an UNRENDERED placeholder" \
|
||||
CONFORMANCE_GIT_SHA=@@CONFORMANCE_GIT_SHA@@ conformance-clone
|
||||
expect_fail "conformance-clone rejects a HOME off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
HOME=/etc conformance-clone
|
||||
expect_fail "conformance-clone refuses a REDIRECTED repository URL (the env-injection hole)" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_REPO_URL=https://evil.invalid/x.git conformance-clone
|
||||
expect_fail "conformance-clone refuses a clone target off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_SRC=/etc/src conformance-clone
|
||||
echo "--- the root filesystem is not writable by the runner (expected: a failure) ---"
|
||||
if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: / is writable by uid $(id -u)"; exit 1; fi
|
||||
echo "confirmed: / is not writable"
|
||||
echo "--- everything uid $(id -u) CAN write lies under /workspace, /tmp or /var/tmp ---"
|
||||
stray="$(find / -xdev \( -path /proc -o -path /sys -o -path /dev \) -prune -o -writable ! -type l -print 2>/dev/null \
|
||||
| grep -vE '^/(workspace|tmp|var/tmp)(/|$)' || true)"
|
||||
if [ -n "${stray}" ]; then echo "UNEXPECTED: writable outside the allowed paths:"; printf ' %s\n' ${stray} | head -20; exit 1; fi
|
||||
echo "confirmed: no writable path outside /workspace, /tmp, /var/tmp"
|
||||
echo "--- HOME is writable (the workspace emptyDir) ---"
|
||||
touch "${HOME}/.probe" || { echo "UNEXPECTED: ${HOME} not writable"; exit 1; }
|
||||
echo "confirmed: ${HOME} is writable"
|
||||
echo "--- uid is non-zero ---"
|
||||
[ "$(id -u)" -ne 0 ] || { echo "UNEXPECTED: running as root"; exit 1; }
|
||||
echo "confirmed: running as uid $(id -u), not root"
|
||||
echo "CONFORMANCE-RUNNER VERIFIED"
|
||||
VERIFY
|
||||
@@ -61,10 +61,12 @@ credentials — the Job's own scoped ServiceAccount does (R-SEC-02).
|
||||
> digest-pinned to the conformance-runner image, `command` exactly
|
||||
> `/usr/local/bin/conformance-clone`, and **no `args` at all**. `ephemeralContainers` and
|
||||
> `env`/`envFrom` were the same shape of hole and are closed in the same policy.
|
||||
- `build-image.sh` — builds the above and **verifies the toolchain by running it** inside the
|
||||
image as uid 65532 with `--read-only`. It **never pushes**, and neither does anyone from a
|
||||
workstation: CI (`.gitea/workflows/images.yaml`) runs this same script, and on a push to
|
||||
`main` publishes the verified image under `sha-<commit>` and `main`. Renovate then moves
|
||||
- `build-image.sh` — builds the above and **verifies the toolchain by running it**: the
|
||||
Containerfile's `verify` stage executes every tool and every `conformance-clone` refusal as
|
||||
uid 65532 with no network, at build time. It **never pushes**, and neither does anyone from a
|
||||
workstation: CI (`.gitea/workflows/images.yaml`) builds the same Containerfile and `verify`
|
||||
stage on rootless BuildKit (`platform/buildkit/`), and on a push to `main` publishes the
|
||||
verified image under `sha-<commit>` and `main`. Renovate then moves
|
||||
`CONFORMANCE_RUNNER_IMAGE_DIGEST` in `conformance/versions.env` onto the registry-returned
|
||||
digest in a pull request (that file is a sensitive path, so the bump needs an
|
||||
administrator's `/approve-override`).
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
# conformance/incluster/build-image.sh — build the pinned `conformance-runner`
|
||||
# image and prove it works. Closes docs/plans/l2-l3-conformance-ci-wiring.md §3 item 1.
|
||||
#
|
||||
# ./build-image.sh # build + verify + print the image id
|
||||
# ./build-image.sh # build the verify target, then the image
|
||||
# ./build-image.sh --print-build-args # print KEY=VALUE build args (what CI consumes)
|
||||
# ./build-image.sh --no-verify # build only (skip the in-image tool proof)
|
||||
# ./build-image.sh --verify-clone # ALSO really clone this repo with conformance-clone.
|
||||
# # Opt-in because it needs the Gitea host reachable; the
|
||||
@@ -13,13 +14,12 @@
|
||||
# IMAGE_TAG=foo ./build-image.sh # override the local tag (CI uses this)
|
||||
#
|
||||
# THIS SCRIPT NEVER PUSHES — and nobody pushes this image from a workstation either.
|
||||
# Publishing is done by CI: .gitea/workflows/images.yaml calls this script, and on a push to
|
||||
# `main` it pushes what this script built AND verified to the Gitea registry, records the
|
||||
# registry-returned digest in the job summary, and Renovate then opens the pull request that
|
||||
# moves conformance/versions.env onto that digest. So this script's job is to build and to
|
||||
# prove the toolchain works INSIDE the image as the non-root uid the policy demands — the same
|
||||
# proof locally and in CI. It runs under ENGINE=docker in CI (Docker-in-Docker), so nothing
|
||||
# below may be podman-only.
|
||||
# Publishing is done by CI: .gitea/workflows/images.yaml builds the SAME Containerfile on
|
||||
# rootless BuildKit (platform/buildkit/), taking its build args from `--print-build-args`, and
|
||||
# on a push to `main` pushes it to the Gitea registry and records the registry-returned digest
|
||||
# in the job summary; Renovate then opens the pull request that moves conformance/versions.env
|
||||
# onto that digest. The proof that the image works is the Containerfile's `verify` stage, which
|
||||
# this script (podman locally) and CI both build.
|
||||
#
|
||||
# VERSIONS ARE NOT DEFINED HERE. They are read from infra/devcontainer/versions.env, the
|
||||
# single source of truth (CHAINSAW_VERSION, KUBECTL_VERSION, FEDORA_VERSION) — the same
|
||||
@@ -39,9 +39,11 @@ TAG_OVERRIDE="${IMAGE_TAG:-}"
|
||||
|
||||
VERIFY=1
|
||||
VERIFY_CLONE=0
|
||||
PRINT_ARGS=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-verify) VERIFY=0 ;;
|
||||
--print-build-args) PRINT_ARGS=1 ;;
|
||||
--verify-clone) VERIFY_CLONE=1 ;;
|
||||
-h|--help) sed -n '2,/^####/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
||||
@@ -52,8 +54,6 @@ log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# --- pins: sourced, never invented -------------------------------------------
|
||||
TOOLCHAIN_ENV="${REPO_ROOT}/infra/devcontainer/versions.env"
|
||||
[ -f "${TOOLCHAIN_ENV}" ] || die "missing ${TOOLCHAIN_ENV} — the pins live there, not here"
|
||||
@@ -63,6 +63,13 @@ source "${TOOLCHAIN_ENV}"
|
||||
: "${KUBECTL_VERSION:?KUBECTL_VERSION not set in ${TOOLCHAIN_ENV}}"
|
||||
: "${FEDORA_VERSION:?FEDORA_VERSION not set in ${TOOLCHAIN_ENV}}"
|
||||
|
||||
# THE build args, in ONE place; `--print-build-args` hands them to the CI pipeline, which builds
|
||||
# this Containerfile on rootless BuildKit.
|
||||
BUILD_ARGS=("CHAINSAW_VERSION=${CHAINSAW_VERSION}" "KUBECTL_VERSION=${KUBECTL_VERSION}" "FEDORA_VERSION=${FEDORA_VERSION}")
|
||||
if [ "${PRINT_ARGS}" -eq 1 ]; then printf '%s\n' "${BUILD_ARGS[@]}"; exit 0; fi
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# --- image name: fixed by the admission policy -------------------------------
|
||||
# platform/kyverno/admission/conformance-trigger-job-shape.yaml admits ONLY
|
||||
# gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:*. The repository path
|
||||
@@ -84,86 +91,27 @@ info " FEDORA ${FEDORA_VERSION}"
|
||||
LABEL_ARGS=()
|
||||
[ -z "${BUILD_REVISION:-}" ] || LABEL_ARGS+=(--label "org.opencontainers.image.revision=${BUILD_REVISION}")
|
||||
|
||||
"${ENGINE}" build "${LABEL_ARGS[@]}" \
|
||||
--file "${HERE}/Containerfile" \
|
||||
--tag "${IMAGE_REF}" \
|
||||
--build-arg "CHAINSAW_VERSION=${CHAINSAW_VERSION}" \
|
||||
--build-arg "KUBECTL_VERSION=${KUBECTL_VERSION}" \
|
||||
--build-arg "FEDORA_VERSION=${FEDORA_VERSION}" \
|
||||
"${HERE}"
|
||||
ARG_FLAGS=()
|
||||
for a in "${BUILD_ARGS[@]}"; do ARG_FLAGS+=(--build-arg "$a"); done
|
||||
|
||||
# --- verify: the point of the whole exercise ---------------------------------
|
||||
# The stock chainsaw image "had" chainsaw and still could not run ./run.sh. So prove the
|
||||
# tools EXECUTE, and prove they execute as the NON-ROOT uid with a READ-ONLY root
|
||||
# filesystem — the shape the admission policy forces. A pass here is the only evidence
|
||||
# this image is usable; a build that merely succeeded is not.
|
||||
# The stock chainsaw image "had" chainsaw and still could not run ./run.sh. So the proof is that
|
||||
# the tools EXECUTE, as the NON-ROOT uid, with no network, that conformance-clone is not writable
|
||||
# by that uid and refuses every hostile input, and that nothing outside /workspace is writable.
|
||||
# That proof is the Containerfile's `verify` stage, built here and in CI alike (CI has no
|
||||
# container runtime — rootless BuildKit only). See the stage's header for what a build step can
|
||||
# and cannot reproduce of the Job's shape. A build that merely succeeded is not evidence.
|
||||
if [ "${VERIFY}" -eq 1 ]; then
|
||||
log "Verifying the toolchain inside the image (non-root, read-only rootfs)"
|
||||
"${ENGINE}" run --rm \
|
||||
--user 65532:65532 \
|
||||
--read-only \
|
||||
--tmpfs /workspace:rw,mode=1777 \
|
||||
--entrypoint "" \
|
||||
"${IMAGE_REF}" \
|
||||
/bin/bash -euo pipefail -c '
|
||||
echo "--- id ---"; id
|
||||
echo "--- HOME ---"; echo "HOME=${HOME}"
|
||||
echo "--- bash ---"; bash --version | head -1
|
||||
echo "--- python3 ---"; python3 --version
|
||||
echo "--- git ---"; git --version
|
||||
echo "--- kubectl ---"; kubectl version --client
|
||||
echo "--- chainsaw ---"; chainsaw version
|
||||
echo "--- baked versions ---"; cat /etc/conformance-runner-versions
|
||||
echo "--- conformance-clone: fixed path, root-owned, NOT writable by the runner ---"
|
||||
stat -c "%n %U:%G %a" /usr/local/bin/conformance-clone
|
||||
if [ -w /usr/local/bin/conformance-clone ]; then
|
||||
echo "UNEXPECTED: uid $(id -u) can write the one executable the policy will trust"; exit 1
|
||||
fi
|
||||
echo "confirmed: not writable by uid $(id -u)"
|
||||
# Each of these MUST fail. Assert the non-zero exit as well as the message: a check
|
||||
# that only greps stderr would pass a build where the guard had stopped rejecting.
|
||||
# (`|| true` inside the capture keeps `set -e`/`pipefail` from aborting on the
|
||||
# expected failure — without it the verify dies at the first negative case.)
|
||||
expect_fail() {
|
||||
local label="$1"; shift
|
||||
local out rc=0
|
||||
out="$(env "$@" 2>&1)" || rc=$?
|
||||
echo "--- ${label} ---"
|
||||
printf "%s\n" "${out}" | head -2
|
||||
if [ "${rc}" -eq 0 ]; then
|
||||
echo "UNEXPECTED: that should have failed, but it exited 0"; exit 1
|
||||
fi
|
||||
echo "(rejected, exit ${rc})"
|
||||
}
|
||||
expect_fail "conformance-clone rejects arguments (that is where an injection would live)" \
|
||||
conformance-clone --layer l2
|
||||
expect_fail "conformance-clone rejects a missing SHA" \
|
||||
conformance-clone
|
||||
expect_fail "conformance-clone rejects a branch name instead of a SHA (no silent run of the tip)" \
|
||||
CONFORMANCE_GIT_SHA=main conformance-clone
|
||||
expect_fail "conformance-clone rejects an UNRENDERED placeholder" \
|
||||
CONFORMANCE_GIT_SHA=@@CONFORMANCE_GIT_SHA@@ conformance-clone
|
||||
expect_fail "conformance-clone rejects a HOME off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
HOME=/etc conformance-clone
|
||||
expect_fail "conformance-clone refuses a REDIRECTED repository URL (the env-injection hole)" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_REPO_URL=https://evil.invalid/x.git conformance-clone
|
||||
expect_fail "conformance-clone refuses a clone target off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_SRC=/etc/src conformance-clone
|
||||
echo "--- rootfs is read-only (expected: a failure) ---"
|
||||
if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: rootfs is writable"; exit 1; fi
|
||||
echo "confirmed: / is read-only"
|
||||
echo "--- HOME is writable (the workspace emptyDir) ---"
|
||||
touch "${HOME}/.probe" && echo "confirmed: ${HOME} is writable"
|
||||
echo "--- uid is non-zero ---"
|
||||
[ "$(id -u)" -ne 0 ] && echo "confirmed: running as uid $(id -u), not root"
|
||||
'
|
||||
log "Verifying the toolchain (Containerfile target: verify)"
|
||||
"${ENGINE}" build "${ARG_FLAGS[@]}" --target verify --file "${HERE}/Containerfile" "${HERE}"
|
||||
log "Toolchain verified"
|
||||
fi
|
||||
|
||||
"${ENGINE}" build "${LABEL_ARGS[@]}" "${ARG_FLAGS[@]}" --target conformance-runner \
|
||||
--file "${HERE}/Containerfile" --tag "${IMAGE_REF}" "${HERE}"
|
||||
|
||||
# --- verify the clone helper against the REAL repository (opt-in: needs network) ----
|
||||
# LOCAL ONLY: this one `run`s the image, which needs a container engine, so CI does not do it.
|
||||
# The negative cases above prove conformance-clone refuses bad input. This proves the
|
||||
# positive case: it lands the EXACT commit asked for and can prove it did. Without this, the
|
||||
# rev-parse assertion is only an unexercised line of shell.
|
||||
|
||||
@@ -250,8 +250,8 @@ check('echo "promtool=${PROMTOOL_VERSION}"' in cf_text,
|
||||
# The build recipe moved from dev.sh into build-image.sh on 2026-10-29, so that `dev.sh build`
|
||||
# and the CI image pipeline (.gitea/workflows/images.yaml) share ONE recipe. Check the recipe.
|
||||
BUILDSH = ROOT / "infra/devcontainer/build-image.sh"
|
||||
check(BUILDSH.is_file() and "--build-arg PROMTOOL_VERSION=" in BUILDSH.read_text(),
|
||||
"infra/devcontainer/build-image.sh does not pass --build-arg PROMTOOL_VERSION — the ARG "
|
||||
check(BUILDSH.is_file() and '"PROMTOOL_VERSION=$PROMTOOL_VERSION"' in BUILDSH.read_text(),
|
||||
"infra/devcontainer/build-image.sh does not pass PROMTOOL_VERSION in its BUILD_ARGS — the ARG "
|
||||
"would be empty and the build would fetch a bogus URL")
|
||||
|
||||
INSTALL = ROOT / ".gitea/workflows/scripts/install-tools.sh"
|
||||
|
||||
Reference in new issue
Block a user