jakman85andClaude Opus 4.8
4a017c590e
Implements the two L1 rendering tests that were failing scaffolds: - egress-class-render (R-NET-05/06/07): renders vpn + internet fixtures, diffs each against a committed golden, asserts the vpn steering label is present on vpn and absent on internet (fail-closed), and that flipping the class changes ONLY the egress path (R-NET-07). This fixes the --requirements R-NET-05 merge gate. - default-deny-baseline-render (R-NET-01): asserts every render carries the namespaced default-deny baseline, shaped deny-all + both directions. Each test carries a pure-offline render-check.sh (renderer + golden diff, no cluster). run.sh runs L1 render checks offline and emits a REAL pass/fail JUnit (docs/14: L1 rendering assertions run in CI without a cluster) instead of skipping L1 with the cluster-gated layers; the chainsaw-test.yaml wrappers invoke the same render-check.sh so an in-cluster run executes the identical assertion. Co-Authored-By: Claude Opus 4.8 <[email protected]>
250 lines
11 KiB
Bash
Executable File
250 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# conformance/run.sh — the single-command conformance runner (R-TEST-02/03/04)
|
|
#
|
|
# ONE command runs the suite (or a requirement-/layer-scoped subset) and always
|
|
# emits JUnit + a requirement-coverage report. Local runs and the in-cluster CI
|
|
# Chainsaw Job invoke this same script so results are identical (reproducibility).
|
|
#
|
|
# ./run.sh # full suite; skips L1/L2/L3 if no cluster
|
|
# ./run.sh --requirements R-NET-01,R-NET-06 # merge-gate subset (tagged)
|
|
# ./run.sh --layer l2 # one layer
|
|
# ./run.sh --requirements R-NET-01,R-NET-02 --layer l2 # intersect: tagged AND in layer
|
|
# ./run.sh --coverage-only # regenerate coverage report only (no cluster)
|
|
# ./run.sh --lint # structural-validate test defs (no cluster)
|
|
# ./run.sh --strict-coverage # hygiene gate: fail on tag/annotation drift
|
|
# ./run.sh --requirements R-NET-01 # ALSO asserts those IDs are covered (merge gate)
|
|
# ./run.sh --require-all # phase-exit: every requirement must be covered
|
|
#
|
|
# Artifacts (always written under conformance/_artifacts/, gitignored):
|
|
# _artifacts/report.junit.xml JUnit (Chainsaw output, or skipped-scaffold shim)
|
|
# _artifacts/coverage/coverage.md human coverage report
|
|
# _artifacts/coverage/coverage.json machine coverage report
|
|
#
|
|
# Phase 0: no cluster exists. The behavior tests are valid scaffolds that cannot
|
|
# execute yet, so run.sh emits a *skipped* JUnit and a coverage report, and exits
|
|
# green — the harness is proven reproducible while later phases fill in behavior.
|
|
###############################################################################
|
|
set -euo pipefail
|
|
|
|
CONF_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
export CONF_ROOT
|
|
# shellcheck source=lib/common.sh
|
|
source "${CONF_ROOT}/lib/common.sh"
|
|
# shellcheck source=versions.env
|
|
source "${CONF_ROOT}/versions.env"
|
|
|
|
ARTIFACTS="${CONF_ROOT}/_artifacts"
|
|
COVERAGE_OUT="${ARTIFACTS}/coverage"
|
|
JUNIT_OUT="${ARTIFACTS}/report.junit.xml"
|
|
PYTHON="${PYTHON:-python3}"
|
|
|
|
LAYER="all"
|
|
REQS=""
|
|
MODE="run" # run | coverage-only | lint
|
|
STRICT_COVERAGE=0
|
|
REQUIRE_ALL=0
|
|
FORCE_NO_CLUSTER=0
|
|
|
|
# Documented, out-of-scope coverage gaps exempt from the completeness gate: requirements
|
|
# owned by another workstream that WS-TEST does not yet scaffold. Listed here so the
|
|
# exemption is explicit and reviewable, not silent.
|
|
# R-STATE-02 (node-loss) — delivered with Longhorn/infra (WS-STORAGE).
|
|
# R-ENV-02 (dev->prod promote) — owned by WS-INFRA/WS-GITOPS; unscaffolded (a GAP in
|
|
# coverage.md). The coarse path->requirement map keys
|
|
# clusters/* AND ci/* to it and the R-ENV-02 doc token
|
|
# appears in platform/kustomization.yaml's header, so ANY
|
|
# infra/overlay/CI edit (e.g. enabling a component, bumping
|
|
# SPEC_PIN) would otherwise false-block on it.
|
|
# R-GITOPS-01 (state in git/Flux) — same: owned by WS-GITOPS, unscaffolded, and the ci/*
|
|
# path-map keys to it, so editing any ci/ file (SPEC_PIN
|
|
# adoption included) would false-block.
|
|
# These are visible GAPs, not hidden — removing them from this list is the right move the
|
|
# moment WS-TEST adds their verifying tests. FOLLOW-UP (WS-TEST): the deeper fix is to make
|
|
# the path->requirement matcher stop keying on doc-header tokens so real requirements need
|
|
# not be exempted; tracked as a harness debt item, not resolved here.
|
|
DOCUMENTED_GAPS="R-STATE-02 R-ENV-02 R-GITOPS-01"
|
|
|
|
usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--layer) LAYER="$2"; shift 2 ;;
|
|
--requirements) REQS="$(conf_normalize_reqs "$2")"; shift 2 ;;
|
|
--coverage-only) MODE="coverage-only"; shift ;;
|
|
--lint) MODE="lint"; shift ;;
|
|
--strict-coverage) STRICT_COVERAGE=1; shift ;;
|
|
--require-all) REQUIRE_ALL=1; shift ;;
|
|
--no-cluster) FORCE_NO_CLUSTER=1; shift ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "unknown arg: $1" >&2; usage; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p "${ARTIFACTS}" "${COVERAGE_OUT}"
|
|
|
|
log() { printf '\033[1m[run.sh]\033[0m %s\n' "$*"; }
|
|
|
|
# --- Coverage report is always produced (cluster-independent, R-TEST-04) --------------
|
|
gen_coverage() {
|
|
local flags=(--allow-gap "${DOCUMENTED_GAPS}")
|
|
[ "${STRICT_COVERAGE}" = 1 ] && flags+=(--strict)
|
|
# A tagged subset run is also the merge gate: the touched requirements MUST be covered.
|
|
[ -n "${REQS}" ] && flags+=(--require "${REQS}")
|
|
[ "${REQUIRE_ALL}" = 1 ] && flags+=(--require-all)
|
|
"${PYTHON}" "${CONF_ROOT}/lib/coverage.py" \
|
|
--registry "${CONF_ROOT}/requirements/registry.yaml" \
|
|
--tests "${CONF_ROOT}/tests" \
|
|
--out "${COVERAGE_OUT}" \
|
|
"${flags[@]}"
|
|
}
|
|
|
|
# --- Structural lint (no cluster): parse each test def and check the tag convention ----
|
|
lint_tests() {
|
|
log "Linting test definitions (structural, no cluster)"
|
|
local rc=0 f
|
|
if command -v chainsaw >/dev/null 2>&1; then
|
|
# Prefer chainsaw's own linter when the pinned binary is available.
|
|
while IFS= read -r f; do
|
|
chainsaw lint test --file "$f" >/dev/null || { echo "LINT FAIL: $f" >&2; rc=1; }
|
|
done < <(conf_all_test_files)
|
|
else
|
|
log "chainsaw not on PATH; falling back to structural check"
|
|
# Minimal YAML-shape check: kind Test + at least one requirement label per file.
|
|
while IFS= read -r f; do
|
|
grep -Eq '^kind:\s*Test' "$f" || { echo "LINT FAIL (no kind: Test): $f" >&2; rc=1; }
|
|
grep -Eq "^[[:space:]]*${CONF_LABEL_PREFIX}/R-[A-Z0-9]+-[0-9]+:" "$f" \
|
|
|| { echo "LINT FAIL (no requirement label): $f" >&2; rc=1; }
|
|
done < <(conf_all_test_files)
|
|
fi
|
|
return $rc
|
|
}
|
|
|
|
# --- Select which test directories Chainsaw should run --------------------------------
|
|
select_dirs() {
|
|
# --requirements AND --layer together => intersection (tests carrying the tag
|
|
# AND living in that layer subtree). Either alone keeps its prior meaning, so
|
|
# this is backward compatible (LAYER defaults to "all"). Both helpers emit
|
|
# sorted, de-duplicated dirs, so `comm -12` yields the intersection.
|
|
if [ -n "${REQS}" ] && [ "${LAYER}" != all ]; then
|
|
comm -12 \
|
|
<(conf_dirs_for_requirements "${REQS}") \
|
|
<(conf_dirs_for_layer "${LAYER}")
|
|
elif [ -n "${REQS}" ]; then
|
|
conf_dirs_for_requirements "${REQS}"
|
|
else
|
|
conf_dirs_for_layer "${LAYER}"
|
|
fi
|
|
}
|
|
|
|
# --- Execute the Chainsaw suite against a live cluster ---------------------------------
|
|
run_chainsaw() {
|
|
local -a dirs
|
|
mapfile -t dirs < <(select_dirs)
|
|
if [ "${#dirs[@]}" -eq 0 ]; then
|
|
log "No tests match the selection (layer=${LAYER} reqs='${REQS}')."
|
|
return 0
|
|
fi
|
|
local -a test_dir_args=()
|
|
for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done
|
|
|
|
log "Running Chainsaw over ${#dirs[@]} test dir(s) with pinned ${CHAINSAW_VERSION}"
|
|
# Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts).
|
|
# --report-path is passed ABSOLUTE (overriding the config's relative `_artifacts`) so the report
|
|
# write succeeds regardless of the caller's CWD — otherwise chainsaw's relative path resolves
|
|
# against $PWD and fails ("open _artifacts/chainsaw-report.xml: no such file") when run.sh is
|
|
# invoked from the repo root instead of conformance/.
|
|
mkdir -p "${ARTIFACTS}"
|
|
chainsaw test \
|
|
--config "${CONF_ROOT}/config/chainsaw-config.yaml" \
|
|
--report-path "${ARTIFACTS}" \
|
|
"${test_dir_args[@]}"
|
|
|
|
# Normalize whatever chainsaw named the report to a stable path CI collects.
|
|
local produced
|
|
produced="$(find "${ARTIFACTS}" -maxdepth 1 -name 'chainsaw-report*.xml' -o \
|
|
-name 'report*.xml' 2>/dev/null | grep -v report.junit.xml | head -1 || true)"
|
|
if [ -n "${produced}" ] && [ "${produced}" != "${JUNIT_OUT}" ]; then
|
|
cp "${produced}" "${JUNIT_OUT}"
|
|
fi
|
|
}
|
|
|
|
# --- L1 rendering: cluster-INDEPENDENT render checks (docs/14) --------------------------
|
|
# L1 verifies ServiceDefinition -> rendered-resources against golden expectations; it needs
|
|
# no live cluster. Each l1-rendering test dir carries a `render-check.sh` (renderer + golden
|
|
# diff + structural asserts). We run those directly and emit a REAL pass/fail JUnit, instead
|
|
# of skipping L1 with the cluster-gated layers (T-0.3). Honors the same --layer/--requirements
|
|
# selection as the rest of the suite (select_dirs), filtered to the l1-rendering subtree.
|
|
l1_selected_dirs() {
|
|
select_dirs | grep '/l1-rendering/' || true
|
|
}
|
|
|
|
run_l1_offline() {
|
|
local -a l1dirs
|
|
mapfile -t l1dirs < <(l1_selected_dirs)
|
|
if [ "${#l1dirs[@]}" -eq 0 ]; then
|
|
log "No L1 render tests match the selection (layer=${LAYER} reqs='${REQS}')."
|
|
return 0
|
|
fi
|
|
log "Running ${#l1dirs[@]} L1 render check(s) OFFLINE (no cluster; docs/14)."
|
|
"${PYTHON}" "${CONF_ROOT}/lib/run_l1_offline.py" \
|
|
--junit "${JUNIT_OUT}" --label-prefix "${CONF_LABEL_PREFIX}" "${l1dirs[@]}"
|
|
}
|
|
|
|
###############################################################################
|
|
# Main
|
|
###############################################################################
|
|
log "Coverage report first (always)"
|
|
gen_coverage
|
|
|
|
case "${MODE}" in
|
|
coverage-only)
|
|
log "coverage-only: done. See ${COVERAGE_OUT}/coverage.md"
|
|
exit 0
|
|
;;
|
|
lint)
|
|
lint_tests
|
|
log "lint: done."
|
|
exit 0
|
|
;;
|
|
esac
|
|
|
|
# L1 rendering is cluster-independent (docs/14): run the selected L1 render checks offline
|
|
# and treat their result as authoritative REGARDLESS of cluster availability. A render-check
|
|
# failure is a real failure (nothing cluster-gated about it), so it always blocks.
|
|
L1_RC=0
|
|
run_l1_offline || L1_RC=$?
|
|
if [ "${LAYER}" = l1 ]; then
|
|
# L1-only run (the ci/l1.sh merge-gate path): the offline render result IS the result.
|
|
if [ "${L1_RC}" -eq 0 ]; then
|
|
log "Done (L1 render). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
|
|
else
|
|
log "L1 render checks FAILED (rc=${L1_RC}). JUnit: ${JUNIT_OUT}"
|
|
fi
|
|
exit "${L1_RC}"
|
|
fi
|
|
if [ "${L1_RC}" -ne 0 ]; then
|
|
log "L1 render checks FAILED (rc=${L1_RC}) — blocking (render failure is not cluster-gated)."
|
|
exit "${L1_RC}"
|
|
fi
|
|
|
|
# run mode: need a cluster for L2/L3. Without one (Phase 0), emit skipped JUnit for the
|
|
# cluster-gated layers. (L1 already ran offline above and passed.)
|
|
if [ "${FORCE_NO_CLUSTER}" = 1 ] || ! conf_cluster_reachable; then
|
|
log "No reachable cluster — emitting SKIPPED JUnit scaffold for L2/L3 (Phase 0 behavior)."
|
|
log "Also running structural lint so scaffolds stay valid."
|
|
lint_tests || { log "lint failed"; exit 1; }
|
|
"${PYTHON}" "${CONF_ROOT}/lib/emit_skipped_junit.py" \
|
|
--tests "${CONF_ROOT}/tests" --out "${JUNIT_OUT}"
|
|
log "Done (scaffold mode). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
|
|
exit 0
|
|
fi
|
|
|
|
command -v chainsaw >/dev/null 2>&1 || {
|
|
echo "ERROR: cluster reachable but 'chainsaw' (${CHAINSAW_VERSION}) not on PATH." >&2
|
|
echo "Install the pinned binary or run via the in-cluster Job (incluster/)." >&2
|
|
exit 3
|
|
}
|
|
run_chainsaw
|
|
log "Done. JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
|