home-network-k3s-impl/ci/tests/gate-decision.test.sh

#!/usr/bin/env bash
# ci/tests/gate-decision.test.sh — the merge gate's OWN decision table.
#
# WHY THIS SUITE EXISTS (2026-09-13). ci/gate.sh used to set BLOCK=1 for a FAILED stage but
# only `warn` for a SKIPPED one, so it printed the words "— not green" and then concluded
# `MERGE GATE: PASS` and exited 0. Observed live twice: `prom-rules` skipped for a missing
# promtool, and `l0` skipped for a missing pinned kyverno CLI — both read by automation as
# merge-approved. That is fail-open in the one component whose entire job is to fail closed,
# and nothing tested it: every other ci/tests suite covers the INPUT (touched-requirements),
# none covered the DECISION.
#
# WHAT IS UNDER TEST: the real ci/gate.sh and ci/lib.sh, copied into a throwaway CI_DIR beside
# STUB stage scripts whose exit codes this suite dictates (STAGE_RC_<stage>) and a stub
# touched-requirements.sh whose rc/reason it dictates (TR_RC / TR_REASON / TR_OUT). Stubbing the
# stages is the point: this asserts how the gate REASONS about pass/fail/skip, independently of
# what any stage happens to do on this machine — so it runs anywhere, with no tools, no cluster,
# no network, in milliseconds.
#
# THE INVARIANT: `MERGE GATE: PASS` + exit 0 means every stage that was supposed to run RAN and
# was GREEN. A stage that could not run is never a pass, and the exit code says so too.
#
# Run:  ci/tests/gate-decision.test.sh     (or `make -C ci test`)
set -uo pipefail

REAL_CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

PASS=0 FAIL=0
ok()  { printf '  ok   %s\n' "$*"; PASS=$((PASS+1)); }
bad() { printf '  FAIL %s\n' "$*"; FAIL=$((FAIL+1)); }

TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT
CI="${TMP}/ci"; mkdir -p "$CI"
REPO="${TMP}/repo"; mkdir -p "$REPO"

cp "${REAL_CI_DIR}/gate.sh" "${REAL_CI_DIR}/lib.sh" "${REAL_CI_DIR}/versions.env" "$CI/"

# One stub per stage. It exits with STAGE_RC_<stage-with-dashes-as-underscores>, default 0.
for stage in spec-immutability lint l0 render-golden l1 conformance; do
  cat > "${CI}/${stage}.sh" <<'STUB'
#!/usr/bin/env bash
n="$(basename "$0" .sh)"; v="STAGE_RC_${n//-/_}"
exit "${!v:-0}"
STUB
  chmod +x "${CI}/${stage}.sh"
done

# Stub decision input. TR_RC (default 0), TR_REASON (written to $TOUCHED_REASON_FILE like the
# real script does), TR_OUT (the touched requirement list on stdout).
cat > "${CI}/touched-requirements.sh" <<'STUB'
#!/usr/bin/env bash
if [ -n "${TOUCHED_REASON_FILE:-}" ]; then printf '%s\n' "${TR_REASON:-}" > "$TOUCHED_REASON_FILE"; fi
[ -n "${TR_OUT:-}" ] && printf '%s\n' "$TR_OUT"
exit "${TR_RC:-0}"
STUB
chmod +x "${CI}/touched-requirements.sh"

# run_gate <VAR=VAL>... — run the gate with a clean, explicit environment. Sets OUT and RC.
# GATE_STRICT_SKIPS is pinned false unless a case says otherwise, so the suite never depends on
# whether the machine running it happens to look like CI.
OUT="" RC=0
run_gate() {
  local summary_file="${TMP}/summary.md"; : > "$summary_file"
  OUT="$(env -i PATH="$PATH" HOME="$HOME" \
        REPO_ROOT="$REPO" GITHUB_STEP_SUMMARY="$summary_file" \
        GATE_STRICT_SKIPS=false TR_OUT="R-TEST-03" \
        "$@" bash "${CI}/gate.sh" 2>&1)"
  RC=$?
  SUMMARY="$(cat "$summary_file")"
}

# assert_verdict <headline> <expected-rc> <label>
assert_verdict() {
  local want="$1" want_rc="$2" label="$3"
  if ! grep -qF "MERGE GATE: ${want}" <<<"$OUT"; then
    bad "${label}: headline should be '${want}' — got: $(grep -oE 'MERGE GATE: [A-Z ]+' <<<"$OUT" | tail -1)"
  elif [ "$RC" -ne "$want_rc" ]; then
    bad "${label}: headline '${want}' correct but exit code ${RC} != ${want_rc}"
  else
    ok "${label} -> ${want}, exit ${want_rc}"
  fi
}
# The load-bearing negative: nothing may say PASS, and nothing may exit 0.
assert_not_pass() {
  local label="$1"
  if grep -qF "MERGE GATE: PASS" <<<"$OUT"; then bad "${label}: printed 'MERGE GATE: PASS'"
  elif [ "$RC" -eq 0 ]; then bad "${label}: exited 0"
  else ok "${label}: not a PASS, non-zero exit (${RC})"; fi
}

echo "== 1. the happy path: every stage green"
run_gate
assert_verdict PASS 0 "all stages green + a touched subset"
grep -qF "nothing was declared out of scope" <<<"$SUMMARY" \
  && ok "PASS summary states nothing was declared out of scope" \
  || bad "PASS summary does not state the absence of declared gaps"

echo
echo "== 2. a FAILED stage blocks (unchanged behaviour, locked down)"
for s in lint l0 render-golden l1 conformance spec-immutability; do
  run_gate "STAGE_RC_${s//-/_}=1"
  assert_verdict BLOCKED 1 "${s} FAILED"
done

echo
echo "== 3. THE REGRESSION: a SKIPPED stage is not a PASS and not exit 0"
for s in lint l0 render-golden l1; do
  run_gate "STAGE_RC_${s//-/_}=3"
  assert_not_pass "${s} SKIPPED (local)"
  assert_verdict UNDETERMINED 3 "${s} SKIPPED (local)"
  grep -qF "$s" <<<"$SUMMARY" && ok "${s}: the step summary names the stage that did not run" \
                              || bad "${s}: step summary does not name the skipped stage"
done

echo
echo "== 4. under STRICT skips (CI declares it) a could-not-run skip BLOCKS, never merely 3"
for s in lint l0 render-golden l1; do
  run_gate GATE_STRICT_SKIPS=true "STAGE_RC_${s//-/_}=3"
  assert_verdict BLOCKED 1 "${s} SKIPPED with GATE_STRICT_SKIPS=true"
done
# ...which is what makes gate.yml's `|| [ $? -eq 3 ]` safe: in CI, exit 3 cannot be a missing
# tool or an unwired seam. Only the declared-environmental skip below can produce it.
run_gate GATE_STRICT_SKIPS=true STAGE_RC_l0=3 STAGE_RC_lint=3
assert_verdict BLOCKED 1 "several stages SKIPPED under strict skips"

echo
echo "== 5. spec-immutability: unverified is never green, but is not upgraded to a block by CI"
run_gate STAGE_RC_spec_immutability=3
assert_not_pass "spec-immutability SKIPPED"
assert_verdict UNDETERMINED 3 "spec-immutability SKIPPED (local)"
run_gate GATE_STRICT_SKIPS=true STAGE_RC_spec_immutability=3
assert_verdict UNDETERMINED 3 "spec-immutability SKIPPED under strict skips (deliberate exception)"
run_gate REQUIRE_SPEC_VERIFICATION=true STAGE_RC_spec_immutability=3
assert_verdict BLOCKED 1 "spec-immutability SKIPPED + REQUIRE_SPEC_VERIFICATION=true"

echo
echo "== 6. conformance L2/L3: a DECLARED-absent capability may still pass — and is NAMED"
run_gate STAGE_RC_conformance=3
assert_verdict PASS 0 "conformance SKIPPED, no cluster declared"
grep -qF "conformance-l2-l3" <<<"$OUT" \
  && ok "the PASS headline names the declared L2/L3 gap" \
  || bad "the PASS headline hides the declared L2/L3 gap"
grep -qF "conformance-l2-l3" <<<"$SUMMARY" \
  && ok "the step summary names the declared L2/L3 gap" \
  || bad "the step summary hides the declared L2/L3 gap"
run_gate CLUSTER_AVAILABLE=true STAGE_RC_conformance=3
assert_verdict BLOCKED 1 "conformance SKIPPED despite CLUSTER_AVAILABLE=true"
run_gate REQUIRE_CLUSTER_TESTS=true STAGE_RC_conformance=3
assert_verdict BLOCKED 1 "conformance SKIPPED + REQUIRE_CLUSTER_TESTS=true with a touched subset"
# A misspelled switch must leave the guard OFF (truthy()), never half-on.
run_gate CLUSTER_AVAILABLE=ture STAGE_RC_conformance=3
assert_verdict PASS 0 "CLUSTER_AVAILABLE=ture is NOT 'available' (no cluster was declared)"
run_gate CLUSTER_AVAILABLE=TRUE STAGE_RC_conformance=3
assert_verdict BLOCKED 1 "CLUSTER_AVAILABLE=TRUE IS a declaration (skip then blocks)"

echo
echo "== 7. the undetermined touched subset: unchanged (2026-08-31 false-green paths)"
run_gate TR_RC=3 TR_REASON=on-base TR_OUT=
assert_verdict "NOT APPLICABLE" 0 "subset undeterminable, reason on-base, all stages green"
run_gate TR_RC=3 TR_REASON=no-base TR_OUT=
assert_verdict BLOCKED 1 "subset undeterminable, reason no-base"
run_gate TR_RC=3 TR_REASON=diff-failed TR_OUT=
assert_verdict BLOCKED 1 "subset undeterminable, reason diff-failed"
# Precedence: a stage that did not run outranks "nothing to gate" — the stronger, non-zero
# headline must win, or a push-to-main run could hide a missing tool behind NOT APPLICABLE.
run_gate TR_RC=3 TR_REASON=on-base TR_OUT= STAGE_RC_l1=3
assert_not_pass "on-base AND l1 SKIPPED"
assert_verdict UNDETERMINED 3 "on-base AND l1 SKIPPED (UNDETERMINED outranks NOT APPLICABLE)"

echo
echo "== 8. an UNUSABLE subset blocks (carried over from 47068ee): any other non-zero exit"
for rc in 1 2 4; do
  run_gate "TR_RC=${rc}" TR_REASON=unclassified-path TR_OUT=
  assert_verdict BLOCKED 1 "touched-requirements.sh exited ${rc} (subset unusable, not empty)"
done
run_gate TR_RC=1 TR_REASON=unclassified-path TR_OUT=
grep -qF "UNUSABLE" <<<"$SUMMARY" && ok "the step summary calls the subset UNUSABLE" \
                                  || bad "the step summary does not report an unusable subset"

echo
echo "== 9. combinations: a FAILURE outranks a non-run, and the headline is the worst news"
run_gate STAGE_RC_l0=1 STAGE_RC_l1=3
assert_verdict BLOCKED 1 "l0 FAILED + l1 SKIPPED"
run_gate STAGE_RC_spec_immutability=3 STAGE_RC_conformance=3
assert_verdict UNDETERMINED 3 "spec-immutability SKIPPED + conformance declared-absent"

echo
if [ "$FAIL" -eq 0 ]; then echo "gate-decision tests: PASS ($PASS assertions)"; exit 0
else echo "gate-decision tests: FAIL ($FAIL failed, $PASS passed)"; exit 1; fi