ci: bake pinned test toolchain into devcontainer + fix conformance layer/spec-immutability seams

Bake the conformance harness toolchain into infra/devcontainer so `make -C ci gate`
runs identically here and in CI:
- Containerfile: install kyverno CLI (1.13.4), kyverno-json, chainsaw (0.2.12),
  yamllint (1.35.1), check-jsonschema (0.29.4), yq (4.44.6) via new pinned RUN block;
  add diffutils + python3-pip; drop stale LIBVIRT_DEFAULT_URI; refresh header for the
  Proxmox(bpg)+conformance toolbox; extend /etc/dev-toolchain-versions provenance.
- versions.env: add KYVERNO/KYVERNO_JSON/CHAINSAW/YAMLLINT/YQ/CHECK_JSONSCHEMA;
  align KUBECTL 1.32.0 -> 1.31.4. dev.sh build(): pass the new --build-args.
- ci/versions.env: pin CHECK_JSONSCHEMA_VERSION=0.29.4 (single source of truth).

Fix harness seams so the gate can run:
- dev.sh run(): mount the sibling spec repo (home-network-k3s) read-only at
  /home-network-k3s for ci/spec-immutability.sh (R-TEST-06); conditional so the
  container still runs without it.
- conformance/lib/common.sh conf_dirs_for_layer(): accept a comma/space list of
  layers and union the dirs, so ci/conformance.sh `run.sh --layer l2,l3` selects
  both layers in one pass (backward compatible for single layers).

NOTE: ci/versions.env pins KYVERNO_JSON_VERSION=0.0.5, which has no upstream release
(max tag is v0.0.3; platform/kyverno/scripts/run-l0.sh already defaults to v0.0.3).
Baked v0.0.3 and flagged for PM to correct ci/versions.env. kyverno-json is not used
by any static gate stage.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

ci/conformance.sh

@@ -42,8 +42,10 @@ fi
 # NOT accept --junit/--coverage. This stage is CLUSTER-GATED (only runs when
 # CLUSTER_AVAILABLE=true, Phase 1+), so its trigger path is confirmed against a real
 # cluster in Phase 1 — TODO(Phase 1): drive the in-cluster Chainsaw Job
-# (conformance/incluster/chainsaw-job.yaml) under the narrow CONFORMANCE_TRIGGER_KUBECONFIG,
-# and confirm multi-layer (l2 + l3) selection syntax against run.sh. Artifacts are then
+# (conformance/incluster/chainsaw-job.yaml) under the narrow CONFORMANCE_TRIGGER_KUBECONFIG.
+# Multi-layer selection: `--layer l2,l3` selects BOTH layers in one run — conf_dirs_for_layer
+# (conformance/lib/common.sh) now unions a comma/space list of layers, so this is a single
+# invocation with one aggregated report (no per-layer looping needed). Artifacts are then
 # COLLECTED from the fixed location below.
 "$CONFORMANCE_CMD" --layer l2,l3 "${req_args[@]}"
 

ci/versions.env

@@ -20,3 +20,7 @@ CHAINSAW_VERSION="0.2.12"
 YAMLLINT_VERSION="1.35.1"
 KUBECTL_VERSION="1.31.4"
 YQ_VERSION="4.44.6"
+
+# ServiceDefinition JSON-Schema validation (schemas/validate.sh). Pip-installed;
+# pinned here so CI and a laptop validate against the exact same validator.
+CHECK_JSONSCHEMA_VERSION="0.29.4"

conformance/lib/common.sh

@@ -47,20 +47,26 @@ conf_dirs_for_requirements() {
   done < <(conf_all_test_files) | sort -u
 }
 
-# Directories for a given layer subtree: l1 | l2 | l3 | meta | all
+# Directories for a given layer (subtree): l1 | l2 | l3 | meta | all.
+# Accepts a comma/space separated LIST of layers (e.g. "l2,l3") and returns the UNION of
+# their test dirs, deduplicated. A single layer keeps its prior behavior (backward compatible);
+# this is what lets `run.sh --layer l2,l3` (the ci/conformance.sh L2/L3 stage) select both
+# layers in one pass instead of erroring on an unknown "l2,l3" layer.
 conf_dirs_for_layer() {
-  local layer="$1" sub
-  case "$layer" in
-    all) find "${CONF_ROOT}/tests" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; | sort -u ;;
-    l1)  sub="l1-rendering" ;;
-    l2)  sub="l2-behavioral" ;;
-    l3)  sub="l3-fault-injection" ;;
-    meta) sub="meta" ;;
-    *) echo "unknown layer: $layer" >&2; return 2 ;;
-  esac
-  if [ "$layer" != all ]; then
-    find "${CONF_ROOT}/tests/${sub}" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; 2>/dev/null | sort -u
-  fi
+  local layers="$1" layer
+  local -a roots=()
+  for layer in ${layers//,/ }; do
+    case "$layer" in
+      all)  roots=("${CONF_ROOT}/tests"); break ;;
+      l1)   roots+=("${CONF_ROOT}/tests/l1-rendering") ;;
+      l2)   roots+=("${CONF_ROOT}/tests/l2-behavioral") ;;
+      l3)   roots+=("${CONF_ROOT}/tests/l3-fault-injection") ;;
+      meta) roots+=("${CONF_ROOT}/tests/meta") ;;
+      *) echo "unknown layer: $layer" >&2; return 2 ;;
+    esac
+  done
+  [ "${#roots[@]}" -gt 0 ] || return 0
+  find "${roots[@]}" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; 2>/dev/null | sort -u
 }
 
 # Normalize a comma/space separated requirement list to space-separated, uppercased.

infra/devcontainer/Containerfile

@@ -1,9 +1,12 @@
 # Reproducible dev toolbox for the home-network-k3s platform (Phase 1+).
 # Bakes the NON-Python prereqs (OpenTofu, age, sops, flux) plus ansible + kube tooling
-# and the libvirt client libraries into a pinned baseline image. Run it with the host
-# libvirt socket mounted (see dev.sh) so OpenTofu (dmacvicar/libvirt provider) and
-# Ansible can drive the local KVM cluster. Fedora base matches the host libvirt major
-# version (host is fc44 / libvirt 12) for client<->daemon compatibility.
+# AND the pinned conformance-harness toolchain (kyverno CLI, kyverno-json, chainsaw,
+# yamllint, check-jsonschema, yq) into a single pinned baseline image, so `make -C ci gate`
+# runs identically here and in CI. This is the Proxmox (bpg/proxmox) toolbox + conformance
+# harness: OpenTofu drives the Proxmox-hosted k3s cluster via the bpg provider (dev.sh opens
+# an SSH tunnel to the kube-API), NOT a local libvirt/KVM host. The libvirt-client/qemu-img
+# packages are retained as harmless disk-image utilities. Fedora base is pinned for
+# reproducibility.
 ARG FEDORA_VERSION=44
 FROM registry.fedoraproject.org/fedora:${FEDORA_VERSION}
 
@@ -13,13 +16,22 @@ ARG SOPS_VERSION
 ARG FLUX_VERSION
 ARG KUBECTL_VERSION
 ARG HELM_VERSION
+# Conformance test toolchain (pinned to match ci/versions.env).
+ARG KYVERNO_VERSION
+ARG KYVERNO_JSON_VERSION
+ARG CHAINSAW_VERSION
+ARG YAMLLINT_VERSION
+ARG YQ_VERSION
+ARG CHECK_JSONSCHEMA_VERSION
 ARG TARGETARCH=amd64
 
-# Distro packages: libvirt client libs (required by the tofu libvirt provider + virsh),
-# ansible, ssh, disk-image + cloud-init seed tooling, and common utilities.
+# Distro packages: libvirt client libs (harmless disk-image utilities), ansible, ssh,
+# disk-image + cloud-init seed tooling, python3-pip (for the pip-installed test tools),
+# diffutils (provides `diff`, used by L1 golden render checks), and common utilities.
 RUN dnf -y install --setopt=install_weak_deps=False \
       libvirt-client qemu-img cloud-utils genisoimage openssh-clients \
-      ansible-core python3-libvirt git make jq unzip tar gzip xz which ca-certificates curl \
+      ansible-core python3-libvirt python3-pip diffutils \
+      git make jq unzip tar gzip xz which ca-certificates curl \
  && dnf clean all
 
 # Pinned Go binaries from official releases.
@@ -35,11 +47,28 @@ RUN set -eux; \
       install -m0755 "/tmp/linux-${TARGETARCH}/helm" /usr/local/bin/helm; \
     rm -rf /tmp/*
 
+# Pinned conformance test toolchain (L0/L1 static gate + the L2/L3 Chainsaw runner).
+# Arch strings differ per tool: yq/chainsaw/kyverno-json use ${TARGETARCH} (amd64), the
+# kyverno CLI tarball uses x86_64 — map amd64 -> x86_64 for it only.
+RUN set -eux; \
+    python3 -m pip install --no-cache-dir --break-system-packages \
+      "yamllint==${YAMLLINT_VERSION}" "check-jsonschema==${CHECK_JSONSCHEMA_VERSION}"; \
+    curl -fsSL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${TARGETARCH}" -o /usr/local/bin/yq; \
+      chmod +x /usr/local/bin/yq; \
+    curl -fsSL "https://github.com/kyverno/chainsaw/releases/download/v${CHAINSAW_VERSION}/chainsaw_linux_${TARGETARCH}.tar.gz" | tar -xz -C /usr/local/bin chainsaw; \
+    KYVERNO_CLI_ARCH="$([ "${TARGETARCH}" = amd64 ] && echo x86_64 || echo "${TARGETARCH}")"; \
+    curl -fsSL "https://github.com/kyverno/kyverno/releases/download/v${KYVERNO_VERSION}/kyverno-cli_v${KYVERNO_VERSION}_linux_${KYVERNO_CLI_ARCH}.tar.gz" | tar -xz -C /usr/local/bin kyverno; \
+    curl -fsSL "https://github.com/kyverno/kyverno-json/releases/download/v${KYVERNO_JSON_VERSION}/kyverno-json_linux_${TARGETARCH}.tar.gz" | tar -xz -C /usr/local/bin kyverno-json; \
+    chmod +x /usr/local/bin/yq /usr/local/bin/chainsaw /usr/local/bin/kyverno /usr/local/bin/kyverno-json; \
+    rm -rf /tmp/*
+
 # Record the baked toolchain for provenance.
 RUN { echo "opentofu=${OPENTOFU_VERSION}"; echo "age=${AGE_VERSION}"; echo "sops=${SOPS_VERSION}"; \
       echo "flux=${FLUX_VERSION}"; echo "kubectl=${KUBECTL_VERSION}"; echo "helm=${HELM_VERSION}"; \
+      echo "kyverno=${KYVERNO_VERSION}"; echo "kyverno-json=${KYVERNO_JSON_VERSION}"; \
+      echo "chainsaw=${CHAINSAW_VERSION}"; echo "yamllint=${YAMLLINT_VERSION}"; \
+      echo "yq=${YQ_VERSION}"; echo "check-jsonschema=${CHECK_JSONSCHEMA_VERSION}"; \
     } > /etc/dev-toolchain-versions
 
-ENV LIBVIRT_DEFAULT_URI=qemu:///system
 WORKDIR /work
 CMD ["/bin/bash"]

infra/devcontainer/dev.sh

@@ -37,6 +37,12 @@ build() {
     --build-arg FLUX_VERSION="$FLUX_VERSION" \
     --build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
     --build-arg HELM_VERSION="$HELM_VERSION" \
+    --build-arg KYVERNO_VERSION="$KYVERNO_VERSION" \
+    --build-arg KYVERNO_JSON_VERSION="$KYVERNO_JSON_VERSION" \
+    --build-arg CHAINSAW_VERSION="$CHAINSAW_VERSION" \
+    --build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \
+    --build-arg YQ_VERSION="$YQ_VERSION" \
+    --build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \
     -t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
 }
 
@@ -54,10 +60,22 @@ run() {
   if [ "${DEV_NO_TUNNEL:-0}" != "1" ]; then
     prelude="if [ -f /root/.ssh/home_network ]; then ssh -fN -L ${K3S_API_LOCAL_PORT}:127.0.0.1:6443 -p ${K3S_SSH_PORT} -i /root/.ssh/home_network -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ExitOnForwardFailure=yes -o ConnectTimeout=5 -o ServerAliveInterval=30 ${K3S_SSH_USER}@${K3S_SSH_HOST} 2>/dev/null || echo '[dev.sh] k3s API tunnel not up (cluster down or NAT changed?) — kubectl/flux/helm will not reach the cluster' >&2; fi"
   fi
+  # Spec repo (home-network-k3s) mount, read-only, for ci/spec-immutability.sh (R-TEST-06).
+  # That script reads SPEC_REPO (default ${REPO_ROOT}/../home-network-k3s); inside the
+  # container REPO_ROOT is /work, so the default resolves to /home-network-k3s. Mount the
+  # sibling spec repo there RO when it exists; skip the mount otherwise so the container
+  # still runs (the gate then honestly SKIPS immutability — exit 3, never faked green).
+  local -a spec_mount=()
+  local spec_host
+  spec_host="$(cd "${REPO_ROOT}/../home-network-k3s" 2>/dev/null && pwd || true)"
+  if [ -n "${spec_host}" ] && [ -d "${spec_host}/.git" ]; then
+    spec_mount=(-v "${spec_host}:/home-network-k3s:ro" -e SPEC_REPO=/home-network-k3s)
+  fi
   "$ENGINE" run --rm -it \
     --network host \
     --security-opt label=disable \
     -v "${REPO_ROOT}:/work" -w /work \
+    "${spec_mount[@]}" \
     -v "${HOME}/.ssh:/root/.ssh:ro" \
     -v "${HOME}/.config/sops:/root/.config/sops" \
     -v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \

infra/devcontainer/versions.env

@@ -1,13 +1,31 @@
 # Pinned toolchain for the home-network-k3s dev container (the reproducible baseline).
 # The non-Python prereqs (OpenTofu, age, sops, flux) are Go binaries, so — per the
-# operator's rule — they are baked into a container image instead of a venv.
+# operator's rule — they are baked into a container image instead of a venv. The
+# conformance test toolchain (kyverno CLI, kyverno-json, chainsaw, yamllint,
+# check-jsonschema, yq) is likewise baked in so `make -C ci gate` runs here == in CI.
 #
 # NOTE: these are best-effort pins; bump to current stable if a download 404s at build.
+# The conformance-tool versions below MUST stay in lockstep with ci/versions.env.
 FEDORA_VERSION=44
 OPENTOFU_VERSION=1.9.0
 AGE_VERSION=1.2.1
 SOPS_VERSION=3.9.4
 FLUX_VERSION=2.4.0
-KUBECTL_VERSION=1.32.0
+# Aligned with ci/versions.env (KUBECTL=1.31.4) and the k3s 1.31.5 cluster.
+KUBECTL_VERSION=1.31.4
 HELM_VERSION=3.16.0
+
+# --- Conformance test toolchain (match ci/versions.env) ----------------------
+KYVERNO_VERSION=1.13.4
+# ci/versions.env pins KYVERNO_JSON_VERSION=0.0.5, but NO such release exists:
+# kyverno/kyverno-json's highest tag is v0.0.3 (git ls-remote --tags), and the
+# platform's own platform/kyverno/scripts/run-l0.sh already defaults to v0.0.3.
+# 0.0.5 is an erroneous pin in ci/versions.env — flagged to PM to correct there.
+# We bake the real, platform-aligned v0.0.3 so the image builds. kyverno-json is
+# NOT invoked by any static gate stage (ci/l0.sh uses `kyverno test` + check-jsonschema).
+KYVERNO_JSON_VERSION=0.0.3
+CHAINSAW_VERSION=0.2.12
+YAMLLINT_VERSION=1.35.1
+YQ_VERSION=4.44.6
+CHECK_JSONSCHEMA_VERSION=0.29.4
 # Proxmox provider (bpg/proxmox) is pinned in the tofu module (infra/tofu/versions.tf), not here.