home-network-k3s-impl/conformance/tests/l0-static/whole-repo-invariants-wired/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 STATIC CHECK — the whole-repo scope mechanism is wired, works, and stays honest.
# See chainsaw-test.yaml's header for WHY this test exists and what it replaced.
#
# Exit 0 = all assertions pass. Exit 1 = a failure. This check has NO exit-3 path: the
# mechanism it audits is source in this repo, so "nothing to audit" is not a state it can
# legitimately be in.
#
# IT MUST NEVER EXECUTE THE SUITE IT IS PART OF. `run.sh --print-selection` resolves the
# selection and exits without running a single check, which is what makes assertion (2)
# possible without infinite recursion.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
CONF="${ROOT}/conformance"
SCOPE="${WHOLE_REPO_SCOPE:-whole-repo}"
PASS=0
FAILED=0

note() { printf '  ok   %s\n' "$1"; PASS=$((PASS+1)); }
bad()  { printf '  FAIL %s\n' "$1" >&2; FAILED=$((FAILED+1)); }
chk()  { if [ "$1" = 0 ]; then note "$2"; else bad "$3"; fi }

L0_SH="${ROOT}/ci/l0.sh"
GATE_SH="${ROOT}/ci/gate.sh"
RUN_SH="${CONF}/run.sh"
COMMON_SH="${CONF}/lib/common.sh"
OFFLINE_PY="${CONF}/lib/run_l1_offline.py"

# ---- (1) the mechanism exists, end to end -----------------------------------------------
for f in "$L0_SH" "$GATE_SH" "$RUN_SH" "$COMMON_SH" "$OFFLINE_PY"; do
  [ -f "$f" ] || { bad "missing ${f#"$ROOT"/} — the mechanism cannot be audited"; }
done
if [ "$FAILED" -ne 0 ]; then
  echo "CHECK FAIL (whole-repo-invariants-wired): the gate's own sources are missing" >&2
  exit 1
fi

grep -q 'CONF_SCOPE_ANNOTATION' "$COMMON_SH" \
  && grep -q 'conf_dirs_for_scope' "$COMMON_SH"
chk $? "conformance/lib/common.sh resolves the scope annotation (conf_dirs_for_scope)" \
       "conformance/lib/common.sh no longer resolves the scope annotation — no test can be
       selected BY declaring a scope, so every whole-repo invariant is back to being
       selectable away by the requirement filter"

grep -q -- '--include-scope' "$RUN_SH" && grep -q 'conf_dirs_for_scopes' "$RUN_SH"
chk $? "conformance/run.sh --include-scope unions scope-declared tests into select_dirs" \
       "conformance/run.sh does not union scope-declared tests (--include-scope /
       conf_dirs_for_scopes missing) — the annotation would be inert decoration"

grep -Eq -- '--include-scope[[:space:]]+"?\$\{?WHOLE_REPO_SCOPE' "$L0_SH" \
  || grep -Eq -- "--include-scope[[:space:]]+['\"]?${SCOPE}" "$L0_SH"
chk $? "ci/l0.sh passes --include-scope ${SCOPE} to the conformance runner" \
       "ci/l0.sh does not pass --include-scope ${SCOPE}: the L0 merge-gate stage is back to a
       purely requirement-scoped run, and every whole-repo invariant is filtered out of
       exactly the change that breaks it"

# ---- (6) the declared set is non-empty and executable -----------------------------------
declared_dirs=()
while IFS= read -r d; do [ -n "$d" ] && declared_dirs+=("$d"); done < <(
  CONF_ROOT="$CONF" bash -c 'source "${CONF_ROOT}/lib/common.sh"; conf_dirs_for_scope "$1"' _ "$SCOPE"
)
MIN_DECLARED="${MIN_DECLARED:-5}"
[ "${#declared_dirs[@]}" -ge "$MIN_DECLARED" ]
chk $? "${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED})" \
       "only ${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED}) — either the
       annotation was stripped from the whole-repo checks, or the resolver stopped matching
       it. Every assertion below would then pass while checking nothing"

for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do
  rel="${d#"$ROOT"/}"
  [ -f "$d/check.sh" ] || [ -f "$d/render-check.sh" ]
  chk $? "${rel}: declares the scope AND carries an executable offline check" \
         "${rel} declares scope='${SCOPE}' but carries neither check.sh nor render-check.sh — an
         unconditional run of a test with nothing to run is a guaranteed SKIP, i.e. a
         permanent not-green, not a guard"
done

# ---- (2) IT WORKS: prove the selection cannot filter a declaring test out ---------------
# A requirement ID no test carries. With it alone the L0 selection MUST be empty; adding
# --include-scope MUST bring back every declaring test. Both halves are asserted, so this is
# falsifiable: if --include-scope became a no-op, the first half would still pass and the
# second would fail.
# Deliberately LOWERCASE: conf_normalize_reqs() upper-cases it, so the selection behaves
# identically — but the literal here does not match ci/touched-requirements.sh's content-scan
# pattern `R-[A-Z]+-[0-9]+`, so editing this file does not attribute a nonexistent requirement
# to the change. (Same reason the synthetic fixture below carries no requirement label at all.)
BOGUS_REQ="r-nosuchreq-99"
sel_without="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --print-selection 2>/dev/null || true)"
sel_with="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --include-scope "$SCOPE" --print-selection 2>/dev/null || true)"

[ -z "$sel_without" ]
chk $? "a requirement filter matching no test resolves to an EMPTY L0 selection (control)" \
       "the control case is broken: --requirements ${BOGUS_REQ} still selected tests
       ($(printf '%s' "$sel_without" | tr '\n' ' ')), so the assertion below proves nothing
       about --include-scope"

missing=""
for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do
  printf '%s\n' "$sel_with" | grep -qxF "$d" || missing="${missing} ${d#"$ROOT"/}"
done
[ -z "$missing" ]
chk $? "every scope-declared L0 test is selected DESPITE a requirement filter matching none" \
       "the scope union does not actually select:${missing}. This is the blind spot itself: a
       change that attributes no requirement would not run these checks"

# The union must not smuggle in another layer's tests when --layer bounds the run.
strays="$(printf '%s\n' "$sel_with" | grep -v '/tests/l0-static/' || true)"
[ -z "$strays" ]
chk $? "the scope union stays inside --layer l0 (no cross-layer leakage)" \
       "--layer l0 --include-scope ${SCOPE} selected non-L0 test(s): $(printf '%s' "$strays" | tr '\n' ' ')"

# ---- (3) a SKIP of a scope-declared check is UNWIRED, NOT GREEN -------------------------
# Proven by running the real offline runner over two synthetic test dirs: one DECLARING the
# scope, one not. Both checks exit 3. The declaring one must make the runner exit 3; the
# other must leave it at 0. This is the half a refactor loses most easily, because nothing
# else in the suite notices when "skip" quietly starts meaning "pass".
WORK="$(mktemp -d)"
trap 'rm -rf "${WORK}"' EXIT
mk_synth() {  # mk_synth <dir> <declare-scope: yes|no>
  local d="$1" declare="$2"
  mkdir -p "$d"
  {
    printf 'apiVersion: chainsaw.kyverno.io/v1alpha1\nkind: Test\nmetadata:\n'
    printf '  name: %s\n  annotations:\n' "$(basename "$d")"
    [ "$declare" = yes ] && printf '    conformance.home.k3s/scope: "%s"\n' "$SCOPE"
    printf '    conformance.home.k3s/layer: "L0"\nspec: {}\n'
  } > "$d/chainsaw-test.yaml"
  printf '#!/usr/bin/env bash\necho "synthetic: nothing to audit" >&2\nexit 3\n' > "$d/check.sh"
}
mk_synth "${WORK}/l0-static/synth-declared" yes
mk_synth "${WORK}/l0-static/synth-undeclared" no

run_offline() {  # run_offline <dir...> -> echoes rc
  local rc=0
  python3 "$OFFLINE_PY" --junit "${WORK}/j.xml" \
    --scope-annotation "conformance.home.k3s/scope" \
    --unwired-scope "$SCOPE" "$@" >/dev/null 2>&1 || rc=$?
  echo "$rc"
}
rc_declared="$(run_offline "${WORK}/l0-static/synth-declared")"
rc_undeclared="$(run_offline "${WORK}/l0-static/synth-undeclared")"

[ "$rc_declared" = 3 ]
chk $? "a SKIPPED check that DECLARES scope='${SCOPE}' makes the runner exit 3 (UNWIRED, not green)" \
       "a SKIPPED scope-declared check made the offline runner exit ${rc_declared}, not 3. A
       whole-repo invariant reporting 'nothing to audit' means the tree it audits has
       VANISHED; the deleted ci/l0.sh stages called that 'not a pass' and set the gate's
       rc=3. Exit 0 here turns that back into a silent green"

[ "$rc_undeclared" = 0 ]
chk $? "a SKIPPED check with NO scope declaration still exits 0 (unchanged for scaffolds)" \
       "a SKIPPED undeclared check made the runner exit ${rc_undeclared}, not 0 — the strict
       rule leaked onto ordinary scaffolds, which would make every unfinished test block"

# ...and ci/l0.sh must map that rc 3 to the GATE's rc 3, never to a pass.
l0_static_stage="$(sed -n '/Stage 3b: L0 STATIC/,/^case "\$rc" in/p' "$L0_SH")"
printf '%s\n' "$l0_static_stage" | grep -Eq '^[[:space:]]*3\)' \
  && printf '%s\n' "$l0_static_stage" | grep -Eq 'rc=3'
chk $? "ci/l0.sh maps the runner's rc 3 to the gate's own rc 3 (SKIPPED, not a pass)" \
       "ci/l0.sh's L0-static stage has no rc-3 arm setting the gate's rc=3, so a scope-declared
       check that SKIPPED would be reported as a PASS — the exact false green the deleted
       stages were written to prevent"

# ---- (4) the bespoke per-check stage pattern has not crept back -------------------------
# Not forbidden outright: a ci/*.sh MAY hardcode a test's check.sh. But if it does, that test
# must ALSO declare the scope, so a hand-written stage can never be the only thing running a
# whole-repo check (which is how the mechanism silently stops covering a new invariant).
# Two exclusions, both the repo's existing "reference, not declaration" rule (see
# ci/touched-requirements.sh changed_diff() rules 2-3):
#   * whole-line `#` comments — a path NAMED in prose is documentation, not an invocation;
#   * ci/tests/ — those files embed example test paths as unit-test FIXTURE DATA.
hardcoded="$(find "${ROOT}/ci" -name '*.sh' -not -path '*/ci/tests/*' -print0 2>/dev/null \
              | xargs -0 -r grep -hE -v '^[[:space:]]*#' \
              | grep -hoE 'conformance/tests/[a-z0-9-]+/[a-zA-Z0-9._-]+/(check|render-check)\.sh' \
              | sort -u || true)"
for rel in $hardcoded; do
  tdir="${ROOT}/$(dirname "$rel")"
  y="${tdir}/chainsaw-test.yaml"
  if [ -f "$y" ] && grep -Eq "^[[:space:]]*conformance\.home\.k3s/scope:[[:space:]]*[\"']?${SCOPE}" "$y"; then
    note "ci/ hardcodes ${rel}, and that test also DECLARES the scope (belt and braces, fine)"
  else
    bad "ci/ hardcodes ${rel} as a bespoke stage, but that test does NOT declare
       conformance.home.k3s/scope: \"${SCOPE}\". That is the pattern this mechanism replaced: the
       stage is then the ONLY thing running the check, it must be hand-maintained, and the
       runner's own selection still does not know the invariant is whole-repo. Add the
       annotation to ${rel%/*}/chainsaw-test.yaml and delete the stage."
  fi
done
[ -z "$hardcoded" ] && note "no ci/*.sh hardcodes a conformance check path (fully declarative)"

# ---- (5) the gate still BLOCKS on an l0 failure -----------------------------------------
# Carried over from the three deleted stages' own checks — asserted once, here.
grep -Eq '^for s in .*\bl0\b.*; do' "$GATE_SH" && grep -Eq 'fail\).*BLOCK=1' "$GATE_SH"
chk $? "ci/gate.sh sets the merge block when the l0 stage fails" \
       "ci/gate.sh does not set the merge block on an l0 stage failure — every assertion above
       would hold while a red L0 stage merged anyway"

echo
echo "whole-repo scope: ${#declared_dirs[@]} declaring test(s): $(for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do printf '%s ' "$(basename "$d")"; done)"
if [ "$FAILED" -eq 0 ]; then
  echo "check OK: whole-repo-invariants-wired — ${PASS} assertions (R-TEST-03)"
  exit 0
fi
echo "CHECK FAIL (whole-repo-invariants-wired): ${FAILED} failed, ${PASS} passed" >&2
exit 1