#!/usr/bin/env bash # ============================================================================= # L0 STATIC CHECK — the whole-repo scope mechanism is wired, works, and stays honest. # See chainsaw-test.yaml's header for WHY this test exists and what it replaced. # # Exit 0 = all assertions pass. Exit 1 = a failure. This check has NO exit-3 path: the # mechanism it audits is source in this repo, so "nothing to audit" is not a state it can # legitimately be in. # # IT MUST NEVER EXECUTE THE SUITE IT IS PART OF. `run.sh --print-selection` resolves the # selection and exits without running a single check, which is what makes assertion (2) # possible without infinite recursion. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root CONF="${ROOT}/conformance" SCOPE="${WHOLE_REPO_SCOPE:-whole-repo}" PASS=0 FAILED=0 note() { printf ' ok %s\n' "$1"; PASS=$((PASS+1)); } bad() { printf ' FAIL %s\n' "$1" >&2; FAILED=$((FAILED+1)); } chk() { if [ "$1" = 0 ]; then note "$2"; else bad "$3"; fi } L0_SH="${ROOT}/ci/l0.sh" GATE_SH="${ROOT}/ci/gate.sh" RUN_SH="${CONF}/run.sh" COMMON_SH="${CONF}/lib/common.sh" OFFLINE_PY="${CONF}/lib/run_l1_offline.py" # ---- (1) the mechanism exists, end to end ----------------------------------------------- for f in "$L0_SH" "$GATE_SH" "$RUN_SH" "$COMMON_SH" "$OFFLINE_PY"; do [ -f "$f" ] || { bad "missing ${f#"$ROOT"/} — the mechanism cannot be audited"; } done if [ "$FAILED" -ne 0 ]; then echo "CHECK FAIL (whole-repo-invariants-wired): the gate's own sources are missing" >&2 exit 1 fi grep -q 'CONF_SCOPE_ANNOTATION' "$COMMON_SH" \ && grep -q 'conf_dirs_for_scope' "$COMMON_SH" chk $? "conformance/lib/common.sh resolves the scope annotation (conf_dirs_for_scope)" \ "conformance/lib/common.sh no longer resolves the scope annotation — no test can be selected BY declaring a scope, so every whole-repo invariant is back to being selectable away by the requirement filter" grep -q -- '--include-scope' "$RUN_SH" && grep -q 'conf_dirs_for_scopes' "$RUN_SH" chk $? "conformance/run.sh --include-scope unions scope-declared tests into select_dirs" \ "conformance/run.sh does not union scope-declared tests (--include-scope / conf_dirs_for_scopes missing) — the annotation would be inert decoration" grep -Eq -- '--include-scope[[:space:]]+"?\$\{?WHOLE_REPO_SCOPE' "$L0_SH" \ || grep -Eq -- "--include-scope[[:space:]]+['\"]?${SCOPE}" "$L0_SH" chk $? "ci/l0.sh passes --include-scope ${SCOPE} to the conformance runner" \ "ci/l0.sh does not pass --include-scope ${SCOPE}: the L0 merge-gate stage is back to a purely requirement-scoped run, and every whole-repo invariant is filtered out of exactly the change that breaks it" # ---- (6) the declared set is non-empty and executable ----------------------------------- declared_dirs=() while IFS= read -r d; do [ -n "$d" ] && declared_dirs+=("$d"); done < <( CONF_ROOT="$CONF" bash -c 'source "${CONF_ROOT}/lib/common.sh"; conf_dirs_for_scope "$1"' _ "$SCOPE" ) MIN_DECLARED="${MIN_DECLARED:-5}" [ "${#declared_dirs[@]}" -ge "$MIN_DECLARED" ] chk $? "${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED})" \ "only ${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED}) — either the annotation was stripped from the whole-repo checks, or the resolver stopped matching it. Every assertion below would then pass while checking nothing" for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do rel="${d#"$ROOT"/}" [ -f "$d/check.sh" ] || [ -f "$d/render-check.sh" ] chk $? "${rel}: declares the scope AND carries an executable offline check" \ "${rel} declares scope='${SCOPE}' but carries neither check.sh nor render-check.sh — an unconditional run of a test with nothing to run is a guaranteed SKIP, i.e. a permanent not-green, not a guard" done # ---- (2) IT WORKS: prove the selection cannot filter a declaring test out --------------- # A requirement ID no test carries. With it alone the L0 selection MUST be empty; adding # --include-scope MUST bring back every declaring test. Both halves are asserted, so this is # falsifiable: if --include-scope became a no-op, the first half would still pass and the # second would fail. # Deliberately LOWERCASE: conf_normalize_reqs() upper-cases it, so the selection behaves # identically — but the literal here does not match ci/touched-requirements.sh's content-scan # pattern `R-[A-Z0-9]+-[0-9]+`, so editing this file does not attribute a nonexistent requirement # to the change. (Same reason the synthetic fixture below carries no requirement label at all.) BOGUS_REQ="r-nosuchreq-99" sel_without="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --print-selection 2>/dev/null || true)" sel_with="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --include-scope "$SCOPE" --print-selection 2>/dev/null || true)" [ -z "$sel_without" ] chk $? "a requirement filter matching no test resolves to an EMPTY L0 selection (control)" \ "the control case is broken: --requirements ${BOGUS_REQ} still selected tests ($(printf '%s' "$sel_without" | tr '\n' ' ')), so the assertion below proves nothing about --include-scope" missing="" for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do printf '%s\n' "$sel_with" | grep -qxF "$d" || missing="${missing} ${d#"$ROOT"/}" done [ -z "$missing" ] chk $? "every scope-declared L0 test is selected DESPITE a requirement filter matching none" \ "the scope union does not actually select:${missing}. This is the blind spot itself: a change that attributes no requirement would not run these checks" # The union must not smuggle in another layer's tests when --layer bounds the run. strays="$(printf '%s\n' "$sel_with" | grep -v '/tests/l0-static/' || true)" [ -z "$strays" ] chk $? "the scope union stays inside --layer l0 (no cross-layer leakage)" \ "--layer l0 --include-scope ${SCOPE} selected non-L0 test(s): $(printf '%s' "$strays" | tr '\n' ' ')" # ---- (3) a SKIP of a scope-declared check is UNWIRED, NOT GREEN ------------------------- # Proven by running the real offline runner over two synthetic test dirs: one DECLARING the # scope, one not. Both checks exit 3. The declaring one must make the runner exit 3; the # other must leave it at 0. This is the half a refactor loses most easily, because nothing # else in the suite notices when "skip" quietly starts meaning "pass". WORK="$(mktemp -d)" trap 'rm -rf "${WORK}"' EXIT mk_synth() { # mk_synth