diff --git a/ci/l1.sh b/ci/l1.sh index 0aa85b0..d19e9f7 100755 --- a/ci/l1.sh +++ b/ci/l1.sh @@ -30,11 +30,13 @@ req_args=(${REQ_ARGS[@]+"${REQ_ARGS[@]}"}) # (conformance/_artifacts/) and does NOT accept --junit/--coverage. Invoke by layer + # touched subset, then COLLECT its artifacts into the CI artifact dirs. # --include-scope: union the touched subset with every L1 test DECLARING -# `conformance.home.k3s/scope: whole-repo`. No L1 test declares it today, so this is a no-op -# now — it is wired HERE, once, so the NEXT whole-repo invariant (at any layer) is picked up by -# declaring the annotation, instead of needing someone to remember to hand-write a stage. That -# "someone must remember" is the failure mode this whole change removes; see -# conformance/lib/common.sh. rc 3 = a scope-declared check SKIPPED = UNWIRED, never a pass. +# `conformance.home.k3s/scope: whole-repo`. As of 2026-09-18 ALL THREE L1 tests declare it, so this +# is load-bearing rather than a no-op: every L1 golden is checked on every gate run, whatever the +# touched paths attribute. It was wired HERE first, ahead of any declaration, so picking up a new +# whole-repo invariant needs only the annotation instead of someone remembering to hand-write a +# stage — and that "someone must remember" is exactly how the 2026-09-04 external-dns merge gated +# GREEN while leaving all three goldens stale (see the rationale in the Tests themselves). +# rc 3 = a scope-declared check SKIPPED = UNWIRED, never a pass. : "${WHOLE_REPO_SCOPE:=whole-repo}" l1_rc=0 "$CONFORMANCE_CMD" --layer l1 --include-scope "$WHOLE_REPO_SCOPE" \ diff --git a/conformance/tests/l0-static/whole-repo-invariants-wired/check.sh b/conformance/tests/l0-static/whole-repo-invariants-wired/check.sh index 4f22a3a..b0c7d6b 100755 --- a/conformance/tests/l0-static/whole-repo-invariants-wired/check.sh +++ b/conformance/tests/l0-static/whole-repo-invariants-wired/check.sh @@ -39,20 +39,20 @@ if [ "$FAILED" -ne 0 ]; then fi grep -q 'CONF_SCOPE_ANNOTATION' "$COMMON_SH" \ - && grep -q 'conf_dirs_for_scope' "$COMMON_SH" -chk $? "conformance/lib/common.sh resolves the scope annotation (conf_dirs_for_scope)" \ + && grep -q 'conf_dirs_for_scope' "$COMMON_SH" && arc=0 || arc=$? +chk $arc "conformance/lib/common.sh resolves the scope annotation (conf_dirs_for_scope)" \ "conformance/lib/common.sh no longer resolves the scope annotation — no test can be selected BY declaring a scope, so every whole-repo invariant is back to being selectable away by the requirement filter" -grep -q -- '--include-scope' "$RUN_SH" && grep -q 'conf_dirs_for_scopes' "$RUN_SH" -chk $? "conformance/run.sh --include-scope unions scope-declared tests into select_dirs" \ +grep -q -- '--include-scope' "$RUN_SH" && grep -q 'conf_dirs_for_scopes' "$RUN_SH" && arc=0 || arc=$? +chk $arc "conformance/run.sh --include-scope unions scope-declared tests into select_dirs" \ "conformance/run.sh does not union scope-declared tests (--include-scope / conf_dirs_for_scopes missing) — the annotation would be inert decoration" grep -Eq -- '--include-scope[[:space:]]+"?\$\{?WHOLE_REPO_SCOPE' "$L0_SH" \ - || grep -Eq -- "--include-scope[[:space:]]+['\"]?${SCOPE}" "$L0_SH" -chk $? "ci/l0.sh passes --include-scope ${SCOPE} to the conformance runner" \ + || grep -Eq -- "--include-scope[[:space:]]+['\"]?${SCOPE}" "$L0_SH" && arc=0 || arc=$? +chk $arc "ci/l0.sh passes --include-scope ${SCOPE} to the conformance runner" \ "ci/l0.sh does not pass --include-scope ${SCOPE}: the L0 merge-gate stage is back to a purely requirement-scoped run, and every whole-repo invariant is filtered out of exactly the change that breaks it" @@ -63,16 +63,16 @@ while IFS= read -r d; do [ -n "$d" ] && declared_dirs+=("$d"); done < <( CONF_ROOT="$CONF" bash -c 'source "${CONF_ROOT}/lib/common.sh"; conf_dirs_for_scope "$1"' _ "$SCOPE" ) MIN_DECLARED="${MIN_DECLARED:-5}" -[ "${#declared_dirs[@]}" -ge "$MIN_DECLARED" ] -chk $? "${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED})" \ +[ "${#declared_dirs[@]}" -ge "$MIN_DECLARED" ] && arc=0 || arc=$? +chk $arc "${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED})" \ "only ${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED}) — either the annotation was stripped from the whole-repo checks, or the resolver stopped matching it. Every assertion below would then pass while checking nothing" for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do rel="${d#"$ROOT"/}" - [ -f "$d/check.sh" ] || [ -f "$d/render-check.sh" ] - chk $? "${rel}: declares the scope AND carries an executable offline check" \ + [ -f "$d/check.sh" ] || [ -f "$d/render-check.sh" ] && arc=0 || arc=$? + chk $arc "${rel}: declares the scope AND carries an executable offline check" \ "${rel} declares scope='${SCOPE}' but carries neither check.sh nor render-check.sh — an unconditional run of a test with nothing to run is a guaranteed SKIP, i.e. a permanent not-green, not a guard" @@ -91,25 +91,58 @@ BOGUS_REQ="r-nosuchreq-99" sel_without="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --print-selection 2>/dev/null || true)" sel_with="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --include-scope "$SCOPE" --print-selection 2>/dev/null || true)" -[ -z "$sel_without" ] -chk $? "a requirement filter matching no test resolves to an EMPTY L0 selection (control)" \ +[ -z "$sel_without" ] && arc=0 || arc=$? +chk $arc "a requirement filter matching no test resolves to an EMPTY L0 selection (control)" \ "the control case is broken: --requirements ${BOGUS_REQ} still selected tests ($(printf '%s' "$sel_without" | tr '\n' ' ')), so the assertion below proves nothing about --include-scope" +# PARTITIONED BY LAYER, because the union is layer-BOUNDED by design: `--layer l0` must select the +# scope-declaring L0 tests and NOT the L1 ones (asserted separately below). This loop compared EVERY +# scope-declaring dir against an `--layer l0` selection, which silently assumed only L0 tests would +# ever declare the scope — and ci/l1.sh was written from the opposite premise, that a whole-repo +# invariant could appear "at any layer". The moment the three L1 rendering tests declared it +# (2026-09-18) this check failed, blaming the union for correctly excluding another layer. missing="" for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do + case "$d" in */tests/l0-static/*) ;; *) continue ;; esac printf '%s\n' "$sel_with" | grep -qxF "$d" || missing="${missing} ${d#"$ROOT"/}" done -[ -z "$missing" ] -chk $? "every scope-declared L0 test is selected DESPITE a requirement filter matching none" \ +[ -z "$missing" ] && arc=0 || arc=$? +chk $arc "every scope-declared L0 test is selected DESPITE a requirement filter matching none" \ "the scope union does not actually select:${missing}. This is the blind spot itself: a change that attributes no requirement would not run these checks" +# The SAME property one layer up. This is what makes the L1 declaration load-bearing rather than +# decorative: ci/l1.sh has always passed --include-scope, but until an L1 test declared the scope +# nothing proved the union reached L1 at all. The 2026-09-04 external-dns merge is the case in +# point — it attributed no rendering requirement, selected ZERO L1 tests, gated GREEN, and left all +# three rendering goldens stale. +sel_l1="$(bash "$RUN_SH" --layer l1 --requirements "$BOGUS_REQ" --include-scope "$SCOPE" \ + --print-selection 2>/dev/null || true)" +missing_l1="" +declared_l1=0 +for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do + case "$d" in */tests/l1-rendering/*) ;; *) continue ;; esac + declared_l1=$((declared_l1+1)) + printf '%s\n' "$sel_l1" | grep -qxF "$d" || missing_l1="${missing_l1} ${d#"$ROOT"/}" +done +[ "$declared_l1" -gt 0 ] && [ -z "$missing_l1" ] && arc=0 || arc=$? +chk $arc "every scope-declared L1 test is selected under --layer l1 DESPITE a requirement filter matching none" \ + "scope-declared L1 tests: ${declared_l1}; not selected by --layer l1 --include-scope ${SCOPE}: + ${missing_l1:-}. If the count is 0 the rendering goldens are back to being checked only + when a touched path happens to attribute R-PLAT-02/R-NET-01/R-NET-05/06/07" + +# And the L1 union must not leak the other way either. +strays_l1="$(printf '%s\n' "$sel_l1" | grep -v '/tests/l1-rendering/' || true)" +[ -z "$strays_l1" ] && arc=0 || arc=$? +chk $arc "the scope union stays inside --layer l1 (no cross-layer leakage)" \ + "--layer l1 --include-scope ${SCOPE} selected non-L1 test(s): $(printf '%s' "$strays_l1" | tr '\n' ' ')" + # The union must not smuggle in another layer's tests when --layer bounds the run. strays="$(printf '%s\n' "$sel_with" | grep -v '/tests/l0-static/' || true)" -[ -z "$strays" ] -chk $? "the scope union stays inside --layer l0 (no cross-layer leakage)" \ +[ -z "$strays" ] && arc=0 || arc=$? +chk $arc "the scope union stays inside --layer l0 (no cross-layer leakage)" \ "--layer l0 --include-scope ${SCOPE} selected non-L0 test(s): $(printf '%s' "$strays" | tr '\n' ' ')" # ---- (3) a SKIP of a scope-declared check is UNWIRED, NOT GREEN ------------------------- @@ -143,23 +176,23 @@ run_offline() { # run_offline -> echoes rc rc_declared="$(run_offline "${WORK}/l0-static/synth-declared")" rc_undeclared="$(run_offline "${WORK}/l0-static/synth-undeclared")" -[ "$rc_declared" = 3 ] -chk $? "a SKIPPED check that DECLARES scope='${SCOPE}' makes the runner exit 3 (UNWIRED, not green)" \ +[ "$rc_declared" = 3 ] && arc=0 || arc=$? +chk $arc "a SKIPPED check that DECLARES scope='${SCOPE}' makes the runner exit 3 (UNWIRED, not green)" \ "a SKIPPED scope-declared check made the offline runner exit ${rc_declared}, not 3. A whole-repo invariant reporting 'nothing to audit' means the tree it audits has VANISHED; the deleted ci/l0.sh stages called that 'not a pass' and set the gate's rc=3. Exit 0 here turns that back into a silent green" -[ "$rc_undeclared" = 0 ] -chk $? "a SKIPPED check with NO scope declaration still exits 0 (unchanged for scaffolds)" \ +[ "$rc_undeclared" = 0 ] && arc=0 || arc=$? +chk $arc "a SKIPPED check with NO scope declaration still exits 0 (unchanged for scaffolds)" \ "a SKIPPED undeclared check made the runner exit ${rc_undeclared}, not 0 — the strict rule leaked onto ordinary scaffolds, which would make every unfinished test block" # ...and ci/l0.sh must map that rc 3 to the GATE's rc 3, never to a pass. l0_static_stage="$(sed -n '/Stage 3b: L0 STATIC/,/^case "\$rc" in/p' "$L0_SH")" printf '%s\n' "$l0_static_stage" | grep -Eq '^[[:space:]]*3\)' \ - && printf '%s\n' "$l0_static_stage" | grep -Eq 'rc=3' -chk $? "ci/l0.sh maps the runner's rc 3 to the gate's own rc 3 (SKIPPED, not a pass)" \ + && printf '%s\n' "$l0_static_stage" | grep -Eq 'rc=3' && arc=0 || arc=$? +chk $arc "ci/l0.sh maps the runner's rc 3 to the gate's own rc 3 (SKIPPED, not a pass)" \ "ci/l0.sh's L0-static stage has no rc-3 arm setting the gate's rc=3, so a scope-declared check that SKIPPED would be reported as a PASS — the exact false green the deleted stages were written to prevent" @@ -193,8 +226,8 @@ done # ---- (5) the gate still BLOCKS on an l0 failure ----------------------------------------- # Carried over from the three deleted stages' own checks — asserted once, here. -grep -Eq '^for s in .*\bl0\b.*; do' "$GATE_SH" && grep -Eq 'fail\).*BLOCK=1' "$GATE_SH" -chk $? "ci/gate.sh sets the merge block when the l0 stage fails" \ +grep -Eq '^for s in .*\bl0\b.*; do' "$GATE_SH" && grep -Eq 'fail\).*BLOCK=1' "$GATE_SH" && arc=0 || arc=$? +chk $arc "ci/gate.sh sets the merge block when the l0 stage fails" \ "ci/gate.sh does not set the merge block on an l0 stage failure — every assertion above would hold while a red L0 stage merged anyway" diff --git a/conformance/tests/l1-rendering/container-workload-render/chainsaw-test.yaml b/conformance/tests/l1-rendering/container-workload-render/chainsaw-test.yaml index 05418e5..ffd2276 100644 --- a/conformance/tests/l1-rendering/container-workload-render/chainsaw-test.yaml +++ b/conformance/tests/l1-rendering/container-workload-render/chainsaw-test.yaml @@ -23,6 +23,17 @@ metadata: conformance.home.k3s/requirements: "R-PLAT-02" conformance.home.k3s/layer: "L1" conformance.home.k3s/safety-critical: "false" + # WHY SCOPE, NOT REQUIREMENTS ALONE: every L1 test exercises ONE shared code path — + # catalog/render.py against schemas/servicedefinition.schema.json — so anything that changes + # rendered output breaks these goldens regardless of which requirement the touched path maps to. + # Requirement-scoped selection cannot see that coupling. Concretely (2026-09-04 external-dns + # merge): `platform/externaldns/helmrelease.yaml` attributes R-ENV-02, R-OBS-01/02/04, + # R-STATE-03 and R-TEST-01/02/03/04 via ci/touched-requirements.sh — NOT R-PLAT-02, R-NET-01 or + # R-NET-05/06/07 — yet it added `external-dns.alpha.kubernetes.io/controller: none` to the + # rendered output of ALL THREE L1 goldens. A requirement-scoped gate selected ZERO L1 tests, went + # GREEN, and left every golden stale; the staleness surfaced only on a later full run. + # ci/l1.sh has passed --include-scope since it was written, waiting for this annotation. + conformance.home.k3s/scope: "whole-repo" conformance.home.k3s/intent: >- A kind: container ServiceDefinition renders a Deployment + Service (the PaaS path). spec: diff --git a/conformance/tests/l1-rendering/default-deny-baseline-render/chainsaw-test.yaml b/conformance/tests/l1-rendering/default-deny-baseline-render/chainsaw-test.yaml index ea46220..7ce99cb 100644 --- a/conformance/tests/l1-rendering/default-deny-baseline-render/chainsaw-test.yaml +++ b/conformance/tests/l1-rendering/default-deny-baseline-render/chainsaw-test.yaml @@ -27,6 +27,17 @@ metadata: conformance.home.k3s/requirements: "R-NET-01" conformance.home.k3s/layer: "L1" conformance.home.k3s/safety-critical: "false" + # WHY SCOPE, NOT REQUIREMENTS ALONE: every L1 test exercises ONE shared code path — + # catalog/render.py against schemas/servicedefinition.schema.json — so anything that changes + # rendered output breaks these goldens regardless of which requirement the touched path maps to. + # Requirement-scoped selection cannot see that coupling. Concretely (2026-09-04 external-dns + # merge): `platform/externaldns/helmrelease.yaml` attributes R-ENV-02, R-OBS-01/02/04, + # R-STATE-03 and R-TEST-01/02/03/04 via ci/touched-requirements.sh — NOT R-PLAT-02, R-NET-01 or + # R-NET-05/06/07 — yet it added `external-dns.alpha.kubernetes.io/controller: none` to the + # rendered output of ALL THREE L1 goldens. A requirement-scoped gate selected ZERO L1 tests, went + # GREEN, and left every golden stale; the staleness surfaced only on a later full run. + # ci/l1.sh has passed --include-scope since it was written, waiting for this annotation. + conformance.home.k3s/scope: "whole-repo" conformance.home.k3s/intent: >- Rendering any service's namespace yields a default-deny baseline policy. spec: diff --git a/conformance/tests/l1-rendering/egress-class-render/chainsaw-test.yaml b/conformance/tests/l1-rendering/egress-class-render/chainsaw-test.yaml index 6ad2275..ab9d0a9 100644 --- a/conformance/tests/l1-rendering/egress-class-render/chainsaw-test.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/chainsaw-test.yaml @@ -32,6 +32,17 @@ metadata: conformance.home.k3s/requirements: "R-NET-05, R-NET-06, R-NET-07" conformance.home.k3s/layer: "L1" conformance.home.k3s/safety-critical: "false" + # WHY SCOPE, NOT REQUIREMENTS ALONE: every L1 test exercises ONE shared code path — + # catalog/render.py against schemas/servicedefinition.schema.json — so anything that changes + # rendered output breaks these goldens regardless of which requirement the touched path maps to. + # Requirement-scoped selection cannot see that coupling. Concretely (2026-09-04 external-dns + # merge): `platform/externaldns/helmrelease.yaml` attributes R-ENV-02, R-OBS-01/02/04, + # R-STATE-03 and R-TEST-01/02/03/04 via ci/touched-requirements.sh — NOT R-PLAT-02, R-NET-01 or + # R-NET-05/06/07 — yet it added `external-dns.alpha.kubernetes.io/controller: none` to the + # rendered output of ALL THREE L1 goldens. A requirement-scoped gate selected ZERO L1 tests, went + # GREEN, and left every golden stale; the staleness surfaced only on a later full run. + # ci/l1.sh has passed --include-scope since it was written, waiting for this annotation. + conformance.home.k3s/scope: "whole-repo" conformance.home.k3s/intent: >- egress: vpn renders the egress-gateway + kill-switch path; internet does not. spec: