# ============================================================================= # conformance-runner — the image the in-cluster L2/L3 Chainsaw Job actually runs. # Closes docs/plans/l2-l3-conformance-ci-wiring.md §3 item 1. # # WHY THIS EXISTS. chainsaw-job.yaml pins `ghcr.io/kyverno/chainsaw:v0.2.12` and # `command: ["./run.sh"]`. That image carries the chainsaw binary and almost nothing else — # no bash, no python3, no kubectl, no git. So the committed Job could never have run: it # would have failed at exec of `./run.sh`. This image carries exactly what # `conformance/run.sh` needs: # # bash run.sh and every tests/**/check.sh are bash (arrays, mapfile, BASH_SOURCE) # python3 lib/{run_l1_offline,gen_coverage,merge_junit,emit_skipped_junit,status}.py # (stdlib only — conformance/versions.env PYTHON_MIN=3.9, so no pip layer) # chainsaw the L1/L2/L3 runner itself (run_chainsaw / lint_tests) # kubectl conf_cluster_reachable()'s `kubectl cluster-info` preflight + probe scripts # git the init-container's `clone --depth 1` + `checkout ` (§3 item 2) and # run.sh's `rev-parse --short HEAD` for the run id # tar/gzip/coreutils the `=== ARTIFACTS-TGZ-BASE64 ===` results channel (§3 item 3) # # It also carries `/usr/local/bin/conformance-clone` (0755 root:root) — the init container's # ONLY entrypoint, a fixed path with no argument surface. That exists to close an admission # hole, not for convenience: see the COPY below and the script's own header. # # VERSIONS COME FROM infra/devcontainer/versions.env — NOT from new pins invented here. # That file is the single source of truth for CHAINSAW_VERSION / KUBECTL_VERSION / # FEDORA_VERSION, and build-image.sh sources it and passes them in as --build-arg. The # ARG defaults below are deliberately EMPTY so a build that forgets to pass them fails # loudly (see the assertion in the first RUN) instead of silently baking a stale version. # # KNOWN LIMIT — READ BEFORE CHOOSING THE JOB'S `args`. # `run.sh` runs the selected OFFLINE L0/L1 checks (run_offline_checks) before it ever # touches the cluster, for ANY selection that is not offline-only. Those checks shell out # to flux / promtool / kyverno / kustomize, which this image deliberately does NOT carry — # baking the whole devcontainer toolchain into the runner would roughly double its size to # re-run work that ci/l0.sh + ci/l1.sh already did offline in the same CI run. So the Job # must select the cluster-gated layers only, e.g. `args: ["--layer", "l2"]` / # `["--layer", "l3"]`, where offline_selected_dirs() resolves to nothing and # run_offline_checks() returns 0. `--layer all` and a bare `--requirements ` would drag # the L0 checks in and fail on a missing tool. Flagged to the plan owner; choosing the Job's # args is T-6.1b's call, not this image's. # # THE ADMISSION CONTRACT THIS IMAGE MUST SATISFY. # platform/kyverno/admission/conformance-trigger-job-shape.yaml is a fail-closed Enforce # ClusterPolicy (R-SEC-02) and it is the security boundary. It requires the Job's container # to be `gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:*` — a DIGEST, # never a tag — with `runAsNonRoot: true` and `allowPrivilegeEscalation: false`. Two # consequences are baked in here: # # 1. `USER` is NUMERIC. With `runAsNonRoot: true` and no explicit `runAsUser`, the kubelet # REJECTS an image whose USER is a name it cannot resolve to a uid # ("container has runAsNonRoot and image has non-numeric user"). So: USER 65532:65532. # 2. The pod is `readOnlyRootFilesystem: true` and the policy admits only the `workspace` # and `results` volumes — there is no writable /tmp and one cannot be added without # changing the boundary. So HOME and every cache path point INTO /workspace (the # emptyDir), not at the read-only root. That is why HOME=/workspace below. # ============================================================================= # --- builder: fetch the pinned release binaries ------------------------------ # A full Fedora for the download stage (it has curl + tar); none of it ships in the runtime # image, so the runner does not carry a downloader it has no business having at runtime. ARG FEDORA_VERSION=44 FROM registry.fedoraproject.org/fedora:${FEDORA_VERSION} AS builder ARG CHAINSAW_VERSION= ARG KUBECTL_VERSION= ARG TARGETARCH=amd64 # Fail the BUILD — not the Job, three weeks later — if a pin was not passed in. RUN set -eux; \ : "${CHAINSAW_VERSION:?pass --build-arg CHAINSAW_VERSION from infra/devcontainer/versions.env}"; \ : "${KUBECTL_VERSION:?pass --build-arg KUBECTL_VERSION from infra/devcontainer/versions.env}" # Same URLs and the same arch handling as infra/devcontainer/Containerfile, so the chainsaw # and kubectl in this image are the same builds the toolbox gate runs with. # # INTEGRITY, MEASURED NOT ASSUMED. kubectl publishes a plain `.sha256`, so it is verified # below. chainsaw publishes NO checksums file — the v0.2.12 release assets are the tarballs plus # per-asset cosign `.sig`/`.pem` and an SBOM (listed via the GitHub releases API). So chainsaw's # integrity here rests on the tag + TLS, and afterwards on this image's own digest. # TODO, worth doing and deliberately not smuggled in here: verify chainsaw with # `cosign verify-blob --certificate chainsaw_linux_amd64.tar.gz.pem # --signature chainsaw_linux_amd64.tar.gz.sig ...`. That needs cosign in the builder stage and # the correct keyless identity/issuer for kyverno's release workflow — a supply-chain decision # that belongs in its own reviewed change, not as a silent addition to a runner image. # infra/devcontainer/Containerfile has the same gap for the same binary. RUN set -eux; \ mkdir -p /out; \ curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" \ -o /out/kubectl; \ curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl.sha256" \ -o /tmp/kubectl.sha256; \ echo "$(cat /tmp/kubectl.sha256) /out/kubectl" | sha256sum -c -; \ curl -fsSL "https://github.com/kyverno/chainsaw/releases/download/v${CHAINSAW_VERSION}/chainsaw_linux_${TARGETARCH}.tar.gz" \ | tar -xz -C /out chainsaw; \ chmod 0755 /out/kubectl /out/chainsaw; \ /out/kubectl version --client=true --output=yaml >/dev/null; \ /out/chainsaw version >/dev/null # --- runtime ----------------------------------------------------------------- ARG FEDORA_VERSION=44 # Stage `conformance-runner` is the image that ships; stage `verify` (end of file) is a TEST stage # built FROM it that only runs assertions and is never pushed. build-image.sh and the CI # pipeline build `--target verify` to prove it and `--target conformance-runner` for the image. FROM registry.fedoraproject.org/fedora-minimal:${FEDORA_VERSION} AS conformance-runner ARG CHAINSAW_VERSION= ARG KUBECTL_VERSION= ARG FEDORA_VERSION=44 # Nonroot uid/gid. 65532 is the distroless `nonroot` convention; the value is arbitrary but # it MUST be numeric and non-zero — see "THE ADMISSION CONTRACT" above. ARG RUNNER_UID=65532 ARG RUNNER_GID=65532 LABEL org.opencontainers.image.title="conformance-runner" \ org.opencontainers.image.description="bash + python3 + chainsaw + kubectl + git for the in-cluster L2/L3 conformance Job (conformance/run.sh)" \ org.opencontainers.image.source="https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl" \ org.opencontainers.image.documentation="conformance/incluster/README.md" \ homelab.keithjackson.dev/requirements="R-TEST-02,R-SEC-02" # bash: run.sh + every check.sh. python3: the stdlib report generators. # git-core: the init-container clone (git-core, not git — drops the perl/svn tail). # coreutils/findutils/grep/sed/gawk: find, sort, base64, xargs, sed -nE, awk in lib/*.sh. # tar+gzip: the base64-tar results channel. diffutils: `diff` in the L1 golden compare. # ca-certificates: TLS for the init clone and for kubectl's in-cluster API calls. RUN set -eux; \ microdnf -y install --setopt=install_weak_deps=0 --nodocs \ bash python3 git-core coreutils findutils grep sed gawk tar gzip diffutils \ ca-certificates; \ microdnf clean all; \ rm -rf /var/cache/* /var/log/dnf* COPY --from=builder /out/kubectl /out/chainsaw /usr/local/bin/ # The init container's ONLY entrypoint. Copied as a file with a fixed path and NO argument # surface, because the admission policy pins the main container's `command` but never reads # `initContainers` — so an init container with `command: ["/bin/sh","-c"]` and an arbitrary # script is currently ADMITTED and runs as the privileged `conformance-runner` SA (reproduced # by the policy's author; pass 3, fail 0). A 4th policy rule requiring # `command: ["/usr/local/bin/conformance-clone"]` with no `args` is what makes this file # REQUIRED rather than merely available; that rule belongs in its own reviewed PR against the # boundary, not here. See the script's header for the env-injection half of the same hole. # # 0755 root:root: the container runs as 65532 and MUST NOT be able to replace the one # executable the policy will trust. Ownership is asserted below rather than assumed. COPY conformance-clone /usr/local/bin/conformance-clone RUN set -eux; \ chown root:root /usr/local/bin/conformance-clone; \ chmod 0755 /usr/local/bin/conformance-clone; \ bash -n /usr/local/bin/conformance-clone; \ /usr/local/bin/conformance-clone --anything 2>&1 | grep -q 'takes no arguments'; \ test "$(stat -c '%U:%G %a' /usr/local/bin/conformance-clone)" = 'root:root 755' # The nonroot account is written directly rather than pulling in shadow-utils to run useradd # once. HOME is /workspace because at runtime that emptyDir is the only writable path. RUN set -eux; \ echo "runner:x:${RUNNER_UID}:${RUNNER_GID}:conformance runner:/workspace:/bin/bash" >> /etc/passwd; \ echo "runner:x:${RUNNER_GID}:" >> /etc/group; \ install -d -m 0775 -o "${RUNNER_UID}" -g "${RUNNER_GID}" /workspace # SYSTEM-level config, because HOME is an emptyDir that starts empty on every run and the # root filesystem is read-only — a runtime `--global` write would fail. # safe.directory=*: the init-container clones into the shared emptyDir and the main container # then runs `rev-parse` over it; any uid mismatch between the two would otherwise abort with # "detected dubious ownership". The repo is fetched fresh per run into an emptyDir, so there # is no pre-existing hostile checkout this relaxes a check against. RUN set -eux; \ printf '%s\n' \ '[safe]' \ ' directory = *' \ '[advice]' \ ' detachedHead = false' \ > /etc/gitconfig; \ chmod 0644 /etc/gitconfig # Provenance: what is actually in this layer, readable from a running pod. RUN set -eux; \ { echo "fedora=${FEDORA_VERSION}"; \ echo "chainsaw=${CHAINSAW_VERSION}"; \ echo "kubectl=${KUBECTL_VERSION}"; \ echo "bash=$(bash --version | head -1)"; \ echo "python3=$(python3 --version)"; \ echo "git=$(git --version)"; \ echo "conformance-clone=$(sha256sum /usr/local/bin/conformance-clone | cut -d' ' -f1)"; \ } > /etc/conformance-runner-versions; \ chmod 0444 /etc/conformance-runner-versions # Every writable path points into the `workspace` emptyDir — the root filesystem is # read-only and the admission policy forbids adding a third volume for scratch. ENV HOME=/workspace \ XDG_CACHE_HOME=/workspace/.cache \ XDG_CONFIG_HOME=/workspace/.config \ TMPDIR=/workspace/.tmp \ PYTHONDONTWRITEBYTECODE=1 \ CONF_ROOT=/workspace/conformance USER ${RUNNER_UID}:${RUNNER_GID} WORKDIR /workspace # No ENTRYPOINT on purpose. The admission policy pins `command: ["./run.sh"]` with # `workingDir: /workspace/conformance`; an ENTRYPOINT here would either be ignored or # prepend something the policy cannot see. `bash` is the interactive fallback for a human # debugging the image locally. CMD ["/bin/bash"] # ============================================================================= # verify — a TEST stage, never shipped. Every assertion conformance/incluster/build-image.sh used # to make with `run --user 65532 --read-only --tmpfs /workspace` now runs here, at build time, # because CI builds on rootless BuildKit where no container runtime exists. # # WHAT A BUILD STEP CAN AND CANNOT REPRODUCE: # * uid 65532 ............ exact (USER inherited from the conformance-runner stage). # * no network ........... exact: RUN --network=none, and the first check proves it. # * --read-only rootfs ... NOT reproducible (a RUN step always has a writable layer). # Approximated by permission: everything uid 65532 can write must lie # under /workspace, /tmp or /var/tmp. In the Job only /workspace (and # /results) are writable — the image points HOME and TMPDIR there — and # readOnlyRootFilesystem is enforced by # platform/kyverno/admission/conformance-trigger-job-shape.yaml. # ============================================================================= FROM conformance-runner AS verify RUN --network=none /bin/bash -euo pipefail <<'VERIFY' echo "--- no network (the build-time stand-in for the Job's isolation) ---" if getent hosts gitea.home.keithjackson.dev >/dev/null 2>&1 || timeout 3 bash -c '/dev/null; then echo "UNEXPECTED: network reachable in the verify stage"; exit 1 fi echo "confirmed: no network" echo "--- id ---"; id echo "--- HOME ---"; echo "HOME=${HOME}" echo "--- bash ---"; bash --version | head -1 echo "--- python3 ---"; python3 --version echo "--- git ---"; git --version echo "--- kubectl ---"; kubectl version --client echo "--- chainsaw ---"; chainsaw version echo "--- baked versions ---"; cat /etc/conformance-runner-versions echo "--- conformance-clone: fixed path, root-owned, NOT writable by the runner ---" stat -c "%n %U:%G %a" /usr/local/bin/conformance-clone if [ -w /usr/local/bin/conformance-clone ]; then echo "UNEXPECTED: uid $(id -u) can write the one executable the policy will trust"; exit 1 fi echo "confirmed: not writable by uid $(id -u)" # Each of these MUST fail. Assert the non-zero exit as well as the message: a check that only # greps stderr would pass a build where the guard had stopped rejecting. expect_fail() { local label="$1"; shift local out rc=0 out="$(env "$@" 2>&1)" || rc=$? echo "--- ${label} ---" printf "%s\n" "${out}" | head -2 if [ "${rc}" -eq 0 ]; then echo "UNEXPECTED: that should have failed, but it exited 0"; exit 1 fi echo "(rejected, exit ${rc})" } expect_fail "conformance-clone rejects arguments (that is where an injection would live)" \ conformance-clone --layer l2 expect_fail "conformance-clone rejects a missing SHA" \ conformance-clone expect_fail "conformance-clone rejects a branch name instead of a SHA (no silent run of the tip)" \ CONFORMANCE_GIT_SHA=main conformance-clone expect_fail "conformance-clone rejects an UNRENDERED placeholder" \ CONFORMANCE_GIT_SHA=@@CONFORMANCE_GIT_SHA@@ conformance-clone expect_fail "conformance-clone rejects a HOME off the workspace volume" \ CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \ HOME=/etc conformance-clone expect_fail "conformance-clone refuses a REDIRECTED repository URL (the env-injection hole)" \ CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \ CONFORMANCE_REPO_URL=https://evil.invalid/x.git conformance-clone expect_fail "conformance-clone refuses a clone target off the workspace volume" \ CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \ CONFORMANCE_SRC=/etc/src conformance-clone echo "--- the root filesystem is not writable by the runner (expected: a failure) ---" if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: / is writable by uid $(id -u)"; exit 1; fi echo "confirmed: / is not writable" echo "--- everything uid $(id -u) CAN write lies under /workspace, /tmp or /var/tmp ---" stray="$(find / -xdev \( -path /proc -o -path /sys -o -path /dev \) -prune -o -writable ! -type l -print 2>/dev/null \ | grep -vE '^/(workspace|tmp|var/tmp)(/|$)' || true)" if [ -n "${stray}" ]; then echo "UNEXPECTED: writable outside the allowed paths:"; printf ' %s\n' ${stray} | head -20; exit 1; fi echo "confirmed: no writable path outside /workspace, /tmp, /var/tmp" echo "--- HOME is writable (the workspace emptyDir) ---" touch "${HOME}/.probe" || { echo "UNEXPECTED: ${HOME} not writable"; exit 1; } echo "confirmed: ${HOME} is writable" echo "--- uid is non-zero ---" [ "$(id -u)" -ne 0 ] || { echo "UNEXPECTED: running as root"; exit 1; } echo "confirmed: running as uid $(id -u), not root" echo "CONFORMANCE-RUNNER VERIFIED" VERIFY