diff --git a/conformance/tests/l0-static/admission-policies-actually-applied/chainsaw-test.yaml b/conformance/tests/l0-static/admission-policies-actually-applied/chainsaw-test.yaml new file mode 100644 index 0000000..440935b --- /dev/null +++ b/conformance/tests/l0-static/admission-policies-actually-applied/chainsaw-test.yaml @@ -0,0 +1,42 @@ +# ============================================================================= +# L0 STATIC — every authored admission policy is actually APPLIED (R-SEC-01, R-SEC-02) +# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster. +# +# WHY THIS EXISTS. `kyverno test` proves a policy is CORRECT. Nothing proved it was DEPLOYED. +# platform/kyverno/ holds three lists and only platform-config/kustomization.yaml applies +# anything; the other two are a reference copy and a namespace+HelmRelease render root. On +# 2026-09-29 the R-SEC-02 boundary policy was added to the reference copy: it parsed, its +# kyverno test passed, the gate passed, Flux reported the tier Ready at the new revision — and +# the cluster ran the conformance-trigger ServiceAccount with nothing bounding it. +# +# WHOLE-REPO SCOPE, DELIBERATELY. The failure mode is a policy that is present but wired +# nowhere, which by definition touches no requirement the diff attributes. A path-scoped run +# would have skipped this check on the very commit that introduced the defect. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: admission-policies-actually-applied + labels: + conformance.home.k3s/R-SEC-01: "true" + conformance.home.k3s/R-SEC-02: "true" + annotations: + conformance.home.k3s/requirements: "R-SEC-01,R-SEC-02" + conformance.home.k3s/layer: "L0" + conformance.home.k3s/scope: "whole-repo" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Every ClusterPolicy authored under platform/kyverno/ is either referenced by + platform-config/kustomization.yaml (so Flux applies it) or is a documented CI-only + exemption that is genuinely not applied — so a policy cannot be authored, tested, + green, and absent from the cluster. +spec: + steps: + - name: every-authored-policy-is-applied-or-documented-ci-only + description: >- + Enumerate the ClusterPolicy files under platform/kyverno/ and assert each is wired + into the Kustomization that actually applies Kyverno CRs, or exempted with a reason. + try: + - script: + content: bash ./check.sh diff --git a/conformance/tests/l0-static/admission-policies-actually-applied/check.sh b/conformance/tests/l0-static/admission-policies-actually-applied/check.sh new file mode 100755 index 0000000..3d697bd --- /dev/null +++ b/conformance/tests/l0-static/admission-policies-actually-applied/check.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# ============================================================================= +# L0 static check (offline, no cluster) — admission-policies-actually-applied +# Serves: R-SEC-01, R-SEC-02 (admission enforcement is only real if it is APPLIED) +# +# WHY THIS EXISTS — a policy that ships green and never reaches the cluster. +# +# platform/kyverno/ holds THREE lists, and only one of them applies anything: +# 1. platform/kyverno/admission/kustomization.yaml — reference only. Its own sibling says +# "kept as reference; THIS file is the buildable render root". +# 2. platform/kyverno/kustomization.yaml — that render root: namespace + +# HelmRelease ONLY. It explicitly notes the ClusterPolicies are applied elsewhere. +# 3. platform-config/kustomization.yaml — THE ONE. Kyverno CRs go here because +# they need the CRDs from the HelmRelease in (2), so they reconcile a tier later. +# +# On 2026-09-29 a ClusterPolicy (conformance-trigger-job-shape, the R-SEC-02 boundary bounding +# what the conformance CI trigger may create) was added to (1). Everything went green: the file +# parsed, `kyverno test` exercised it BY PATH and passed, the gate passed, Flux reported +# `platform` Ready at the new revision — and the policy was never in any applied set. The +# cluster ended up running the conformance-trigger ServiceAccount with NOTHING bounding it, +# which is precisely the state that identity's own RBAC header says must never exist. +# +# Nothing could have caught it. `kyverno test` proves a policy is CORRECT, not that it is +# DEPLOYED. This check asserts the second thing. +# +# GIVEN the repository as git tracks it, +# WHEN every file under platform/kyverno/ that declares a ClusterPolicy is enumerated, +# THEN each one is either referenced by platform-config/kustomization.yaml (so Flux applies +# it), or named in CI_ONLY below with a reason — and CI_ONLY itself may not name a +# policy that IS applied, so the exemption list cannot rot into a lie either way. +# ============================================================================= +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root +exec python3 - "$ROOT" <<'PY' +import sys, subprocess +from pathlib import Path + +ROOT = Path(sys.argv[1]) +FAILURES, CHECKS = [], 0 + +def check(cond, desc): + global CHECKS + CHECKS += 1 + if not cond: + FAILURES.append(desc) + +# Policies authored for the CI L0 gate that are deliberately NOT applied in-cluster. Each needs +# a reason, and each is asserted BELOW to really be absent from the applied set — an exemption +# that silently starts being applied is as much a lie as one that silently stops. +CI_ONLY = { + "disallow-plaintext-secrets": + "CI-only by design: it requires a top-level `sops` block, which is IMPOSSIBLE at " + "admission because Flux decrypts before the Secret reaches the API server. Its " + "in-cluster counterpart is admission/secret-provenance.yaml.", +} + +tracked = subprocess.run(["git", "ls-files", "-z", "--", "platform/kyverno/**/*.yaml", + "platform/kyverno/*.yaml"], + cwd=ROOT, capture_output=True, text=True, check=True) +files = [ROOT / p for p in tracked.stdout.split("\0") if p] + +# A ClusterPolicy file, identified without a YAML parse of every doc: cheap and sufficient. +policies = {} +for f in files: + try: + txt = f.read_text() + except Exception: + continue + if "kind: ClusterPolicy" not in txt: + continue + for line in txt.splitlines(): + st = line.strip() + if st.startswith("name:") and " name:" in line: + policies[f.relative_to(ROOT).as_posix()] = st.split("name:", 1)[1].strip() + break + +check(len(policies) >= 6, + f"(0) the scan found {len(policies)} ClusterPolicy file(s) under platform/kyverno/ — " + f"too few means the scan broke, and every assertion below would be vacuous") + +applied_txt = (ROOT / "platform-config" / "kustomization.yaml").read_text() + +unapplied, wrongly_exempt = [], [] +for rel, name in sorted(policies.items()): + # platform-config references these as ../platform/kyverno/... + referenced = rel in applied_txt or rel.replace("platform/", "../platform/", 1) in applied_txt + if name in CI_ONLY: + if referenced: + wrongly_exempt.append(f"{name} ({rel}) is listed CI_ONLY but IS applied by platform-config") + elif not referenced: + unapplied.append(f"{name} ({rel})") + +check(not unapplied, + f"(1) every in-cluster ClusterPolicy is referenced by platform-config/kustomization.yaml " + f"— an unreferenced policy is authored, tested, green, and NEVER APPLIED {unapplied if unapplied else ''}") + +check(not wrongly_exempt, + f"(2) no CI_ONLY exemption names a policy that IS applied — a stale exemption hides a " + f"real deployment {wrongly_exempt if wrongly_exempt else ''}") + +for name, reason in CI_ONLY.items(): + check(bool(reason.strip()), + f"(3) CI_ONLY['{name}'] carries a reason — an unexplained exemption is how this " + f"list stops being reviewable") + +check(CHECKS >= 4, + f"only {CHECKS} assertions ran — the check degenerated to a near-no-op") + +if FAILURES: + print(f"CHECK FAIL (admission-policies-actually-applied): {len(FAILURES)} of {CHECKS} assertions failed", + file=sys.stderr) + for f in FAILURES: + print(f" - {f}", file=sys.stderr) + raise SystemExit(1) +print(f"check OK: admission-policies-actually-applied — {CHECKS} assertions; " + f"{len(policies)} ClusterPolicy file(s), {len(policies) - len(CI_ONLY)} applied by " + f"platform-config, {len(CI_ONLY)} CI-only with reasons. Proves policies are DEPLOYED, " + f"not that they are correct — `kyverno test` is what proves correctness.") +PY diff --git a/platform-config/kustomization.yaml b/platform-config/kustomization.yaml index 3916e94..ddb94ae 100644 --- a/platform-config/kustomization.yaml +++ b/platform-config/kustomization.yaml @@ -34,6 +34,14 @@ resources: # policies, the next import is blocked in the gap. - ../platform/kyverno/policies/label-cdi-worker-pods.yaml - ../platform/kyverno/admission/secret-provenance.yaml + # R-SEC-02 — bounds what the conformance CI trigger may create. MUST be listed HERE: this + # Kustomization is what actually applies ClusterPolicies. Adding a policy only to + # platform/kyverno/admission/kustomization.yaml is a SILENT NO-OP — that file is reference + # only ("kept as reference; THIS file is the buildable render root" in + # platform/kyverno/kustomization.yaml, which itself ships only the namespace + HelmRelease). + # That is exactly how this policy shipped green and never reached the cluster on 2026-09-29, + # leaving the conformance-trigger identity live with nothing bounding it. + - ../platform/kyverno/admission/conformance-trigger-job-shape.yaml # cilium: L2 LB IP pool + announcement policy (cilium.io CRDs already registered by Cilium) - cilium/l2-ip-pool.yaml - cilium/l2-announcement-policy.yaml