home-network-k3s-impl/conformance/run.sh

#!/usr/bin/env bash
###############################################################################
# conformance/run.sh — the single-command conformance runner (R-TEST-02/03/04)
#
# ONE command runs the suite (or a requirement-/layer-scoped subset) and always
# emits JUnit + a requirement-coverage report. Local runs and the in-cluster CI
# Chainsaw Job invoke this same script so results are identical (reproducibility).
#
#   ./run.sh                          # full suite; skips L1/L2/L3 if no cluster
#   ./run.sh --requirements R-NET-01,R-NET-06   # merge-gate subset (tagged)
#   ./run.sh --layer l2               # one layer
#   ./run.sh --requirements R-NET-01,R-NET-02 --layer l2   # intersect: tagged AND in layer
#   ./run.sh --coverage-only          # regenerate coverage report only (no cluster)
#   ./run.sh --lint                   # structural-validate test defs (no cluster)
#   ./run.sh --strict-coverage        # hygiene gate: fail on tag/annotation drift
#   ./run.sh --requirements R-NET-01  # ALSO asserts those IDs are covered (merge gate)
#   ./run.sh --require-all            # phase-exit: every requirement must be covered
#
# Artifacts (always written under conformance/_artifacts/, gitignored):
#   _artifacts/report.junit.xml        JUnit (Chainsaw output, or skipped-scaffold shim)
#   _artifacts/coverage/coverage.md    human coverage report
#   _artifacts/coverage/coverage.json  machine coverage report
#
# Phase 0: no cluster exists. The behavior tests are valid scaffolds that cannot
# execute yet, so run.sh emits a *skipped* JUnit and a coverage report, and exits
# green — the harness is proven reproducible while later phases fill in behavior.
###############################################################################
set -euo pipefail

CONF_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export CONF_ROOT
# shellcheck source=lib/common.sh
source "${CONF_ROOT}/lib/common.sh"
# shellcheck source=versions.env
source "${CONF_ROOT}/versions.env"

ARTIFACTS="${CONF_ROOT}/_artifacts"
COVERAGE_OUT="${ARTIFACTS}/coverage"
JUNIT_OUT="${ARTIFACTS}/report.junit.xml"
PYTHON="${PYTHON:-python3}"

LAYER="all"
REQS=""
MODE="run"          # run | coverage-only | lint
STRICT_COVERAGE=0
REQUIRE_ALL=0
FORCE_NO_CLUSTER=0

# Documented, out-of-scope coverage gaps exempt from the completeness gate: requirements
# owned by another workstream that WS-TEST does not yet scaffold. Listed here so the
# exemption is explicit and reviewable, not silent.
#   R-ENV-02   (dev->prod promote) — owned by WS-INFRA/WS-GITOPS; unscaffolded (a GAP in
#                                    coverage.md). The coarse path->requirement map keys
#                                    clusters/* AND ci/* to it and the R-ENV-02 doc token
#                                    appears in platform/kustomization.yaml's header, so ANY
#                                    infra/overlay/CI edit (e.g. enabling a component, bumping
#                                    SPEC_PIN) would otherwise false-block on it.
#   R-GITOPS-01 (state in git/Flux) — same: owned by WS-GITOPS, unscaffolded, and the ci/*
#                                    path-map keys to it, so editing any ci/ file (SPEC_PIN
#                                    adoption included) would false-block.
# These are visible GAPs, not hidden — removing them from this list is the right move the
# moment WS-TEST adds their verifying tests.
#
# FOLLOW-UP (WS-TEST) — HALF DONE. The debt was "make the matcher stop keying on doc-header
# tokens so real requirements need not be exempted". The CONTENT-SCAN half is now fixed in
# ci/touched-requirements.sh: it scans the added/removed lines of the diff, and skips *.md, so
# a requirement named in a header or in prose no longer counts as a declaration. Consequence
# to be aware of: a manifest whose HEADER declares the requirements it serves no longer
# contributes those IDs unless the change actually touches a line carrying them — the
# authoritative signal is the `conformance.home.k3s/R-*` label on a test, which still is.
# The PATH-MAP half remains: ci/touched-requirements.sh keys clusters/* and ci/* to
# R-ENV-02 + R-GITOPS-01 wholesale, which is why those two must stay exempted here. Tighten
# the path map (or scaffold the two requirements) and this list empties.
#
# R-STATE-02 (node-loss) WAS listed here (owned by WS-STORAGE, "delivered with Longhorn/infra")
# but now has a real verifying L3 test — tests/l3-fault-injection/node-loss — so it is no
# longer a gap and has been removed from this list; the coverage report counts it as covered.
DOCUMENTED_GAPS="R-ENV-02 R-GITOPS-01"

usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }

while [ $# -gt 0 ]; do
  case "$1" in
    --layer)          LAYER="$2"; shift 2 ;;
    --requirements)   REQS="$(conf_normalize_reqs "$2")"; shift 2 ;;
    --coverage-only)  MODE="coverage-only"; shift ;;
    --lint)           MODE="lint"; shift ;;
    --strict-coverage) STRICT_COVERAGE=1; shift ;;
    --require-all)    REQUIRE_ALL=1; shift ;;
    --no-cluster)     FORCE_NO_CLUSTER=1; shift ;;
    -h|--help)        usage; exit 0 ;;
    *) echo "unknown arg: $1" >&2; usage; exit 2 ;;
  esac
done

mkdir -p "${ARTIFACTS}" "${COVERAGE_OUT}"

log() { printf '\033[1m[run.sh]\033[0m %s\n' "$*"; }

# --- Coverage report is always produced (cluster-independent, R-TEST-04) --------------
gen_coverage() {
  local flags=(--allow-gap "${DOCUMENTED_GAPS}")
  [ "${STRICT_COVERAGE}" = 1 ] && flags+=(--strict)
  # A tagged subset run is also the merge gate: the touched requirements MUST be covered.
  [ -n "${REQS}" ] && flags+=(--require "${REQS}")
  [ "${REQUIRE_ALL}" = 1 ] && flags+=(--require-all)
  "${PYTHON}" "${CONF_ROOT}/lib/coverage.py" \
    --registry "${CONF_ROOT}/requirements/registry.yaml" \
    --tests "${CONF_ROOT}/tests" \
    --out "${COVERAGE_OUT}" \
    "${flags[@]}"
}

# --- Structural lint (no cluster): parse each test def and check the tag convention ----
lint_tests() {
  log "Linting test definitions (structural, no cluster)"
  local rc=0 f
  if command -v chainsaw >/dev/null 2>&1; then
    # Prefer chainsaw's own linter when the pinned binary is available.
    while IFS= read -r f; do
      chainsaw lint test --file "$f" >/dev/null || { echo "LINT FAIL: $f" >&2; rc=1; }
    done < <(conf_all_test_files)
  else
    log "chainsaw not on PATH; falling back to structural check"
    # Minimal YAML-shape check: kind Test + at least one requirement label per file.
    while IFS= read -r f; do
      grep -Eq '^kind:\s*Test' "$f" || { echo "LINT FAIL (no kind: Test): $f" >&2; rc=1; }
      grep -Eq "^[[:space:]]*${CONF_LABEL_PREFIX}/R-[A-Z0-9]+-[0-9]+:" "$f" \
        || { echo "LINT FAIL (no requirement label): $f" >&2; rc=1; }
    done < <(conf_all_test_files)
  fi
  return $rc
}

# --- Select which test directories Chainsaw should run --------------------------------
select_dirs() {
  # --requirements AND --layer together => intersection (tests carrying the tag
  # AND living in that layer subtree). Either alone keeps its prior meaning, so
  # this is backward compatible (LAYER defaults to "all"). Both helpers emit
  # sorted, de-duplicated dirs, so `comm -12` yields the intersection.
  if [ -n "${REQS}" ] && [ "${LAYER}" != all ]; then
    comm -12 \
      <(conf_dirs_for_requirements "${REQS}") \
      <(conf_dirs_for_layer "${LAYER}")
  elif [ -n "${REQS}" ]; then
    conf_dirs_for_requirements "${REQS}"
  else
    conf_dirs_for_layer "${LAYER}"
  fi
}

# --- HEAVY tests: the storage/VM/DR tests whose KubeVirt-CDI imports + Longhorn volume ops
#     spike embedded-etcd I/O on the small dev VMs. Run SERIALLY (parallel=1) in their own
#     pass so their etcd spikes never coincide with ANOTHER test's setup — the coincidence
#     that made a single heavy test's apiserver blip fail-close Kyverno admission and abort
#     every in-flight test's 0.01s setup (see config/chainsaw-config.yaml). Keyed by the test
#     DIRECTORY basename (stable); override via CONF_HEAVY_TESTS to add/remove without code
#     changes. Listed here even for tests still scaffolded (vm-lifecycle, dr-restore, ...) so
#     the serialization is already correct the moment they grow real KubeVirt/Longhorn bodies.
CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore node-loss}"

conf_is_heavy_dir() {
  # $1 = test dir; true if its basename is in the heavy set.
  local base t
  base="$(basename "$1")"
  for t in ${CONF_HEAVY_TESTS}; do [ "$base" = "$t" ] && return 0; done
  return 1
}

# One Chainsaw invocation over a specific set of dirs, at a given parallelism, writing to a
# named report file. Returns chainsaw's rc (0=all passed) WITHOUT aborting run.sh (callers
# collect both passes' rc so a light-pass failure still lets the heavy pass run, and vice
# versa). failFast stays false (config) so every test in the pass runs regardless.
run_chainsaw_pass() {
  local report_name="$1" parallel="$2"; shift 2
  local -a dirs=("$@")
  local -a test_dir_args=()
  local d
  for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done
  # Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts).
  # --report-path is passed ABSOLUTE (overriding the config's relative `_artifacts`) so the
  # report write succeeds regardless of the caller's CWD. --report-name/--parallel override
  # the config per-pass so the two passes don't clobber each other's report and the heavy
  # pass forces serial execution.
  local rc=0
  chainsaw test \
    --config "${CONF_ROOT}/config/chainsaw-config.yaml" \
    --report-path "${ARTIFACTS}" \
    --report-name "${report_name}" \
    --parallel "${parallel}" \
    "${test_dir_args[@]}" || rc=$?
  return $rc
}

# --- Execute the Chainsaw suite against a live cluster ---------------------------------
# Split into a LIGHT pass (concurrent, config parallelism) and a HEAVY pass (serial,
# parallel=1). Robust single-pass green: the heavy etcd spikes are isolated in time from the
# light tests' setup, so a heavy test's transient apiserver blip can no longer cascade into
# mass fail-closed setup-aborts of unrelated tests. The two JUnit reports are stitched back
# into the one stable artifact CI collects (report.junit.xml) via lib/merge_junit.py, so the
# split is invisible to everything downstream.
run_chainsaw() {
  local -a dirs
  mapfile -t dirs < <(select_dirs)
  if [ "${#dirs[@]}" -eq 0 ]; then
    log "No tests match the selection (layer=${LAYER} reqs='${REQS}')."
    return 0
  fi

  local -a light=() heavy=()
  local d
  for d in "${dirs[@]}"; do
    if conf_is_heavy_dir "$d"; then heavy+=("$d"); else light+=("$d"); fi
  done

  mkdir -p "${ARTIFACTS}"
  local light_report="${ARTIFACTS}/chainsaw-report-light.xml"
  local heavy_report="${ARTIFACTS}/chainsaw-report-heavy.xml"
  rm -f "${light_report}" "${heavy_report}"
  local light_rc=0 heavy_rc=0

  # Determine the light pass's parallelism from the shared config (single source of truth),
  # falling back to 2 if unreadable — so bumping the config on prod-class hw needs no run.sh edit.
  local light_parallel
  light_parallel="$(sed -nE 's/^[[:space:]]*parallel:[[:space:]]*([0-9]+).*/\1/p' \
                      "${CONF_ROOT}/config/chainsaw-config.yaml" | head -1)"
  [ -n "${light_parallel}" ] || light_parallel=2

  if [ "${#light[@]}" -gt 0 ]; then
    log "LIGHT pass: ${#light[@]} test(s), parallel=${light_parallel} (pinned ${CHAINSAW_VERSION})"
    run_chainsaw_pass "chainsaw-report-light" "${light_parallel}" "${light[@]}" || light_rc=$?
  else
    log "LIGHT pass: no tests selected."
  fi

  if [ "${#heavy[@]}" -gt 0 ]; then
    log "HEAVY pass: ${#heavy[@]} test(s) SERIAL (parallel=1) — ${heavy[*]##*/}"
    run_chainsaw_pass "chainsaw-report-heavy" 1 "${heavy[@]}" || heavy_rc=$?
  else
    log "HEAVY pass: no heavy tests selected."
  fi

  # Stitch both passes' JUnit into the one stable artifact CI collects.
  "${PYTHON}" "${CONF_ROOT}/lib/merge_junit.py" \
    --out "${JUNIT_OUT}" --name chainsaw-report \
    "${light_report}" "${heavy_report}"

  # Overall result is the union: fail if EITHER pass had a failing/erroring test. Real
  # failures still surface (nothing is masked); the split only changes WHEN heavy tests run.
  if [ "${light_rc}" -ne 0 ] || [ "${heavy_rc}" -ne 0 ]; then
    log "Chainsaw reported failures (light rc=${light_rc}, heavy rc=${heavy_rc})."
    return 1
  fi
  return 0
}

# --- L1 rendering: cluster-INDEPENDENT render checks (docs/14) --------------------------
# L1 verifies ServiceDefinition -> rendered-resources against golden expectations; it needs
# no live cluster. Each l1-rendering test dir carries a `render-check.sh` (renderer + golden
# diff + structural asserts). We run those directly and emit a REAL pass/fail JUnit, instead
# of skipping L1 with the cluster-gated layers (T-0.3). Honors the same --layer/--requirements
# selection as the rest of the suite (select_dirs), filtered to the l1-rendering subtree.
l1_selected_dirs() {
  select_dirs | grep '/l1-rendering/' || true
}

run_l1_offline() {
  local -a l1dirs
  mapfile -t l1dirs < <(l1_selected_dirs)
  if [ "${#l1dirs[@]}" -eq 0 ]; then
    log "No L1 render tests match the selection (layer=${LAYER} reqs='${REQS}')."
    return 0
  fi
  log "Running ${#l1dirs[@]} L1 render check(s) OFFLINE (no cluster; docs/14)."
  "${PYTHON}" "${CONF_ROOT}/lib/run_l1_offline.py" \
    --junit "${JUNIT_OUT}" --label-prefix "${CONF_LABEL_PREFIX}" "${l1dirs[@]}"
}

###############################################################################
# Main
###############################################################################
log "Coverage report first (always)"
gen_coverage

case "${MODE}" in
  coverage-only)
    log "coverage-only: done. See ${COVERAGE_OUT}/coverage.md"
    exit 0
    ;;
  lint)
    lint_tests
    log "lint: done."
    exit 0
    ;;
esac

# L1 rendering is cluster-independent (docs/14): run the selected L1 render checks offline
# and treat their result as authoritative REGARDLESS of cluster availability. A render-check
# failure is a real failure (nothing cluster-gated about it), so it always blocks.
L1_RC=0
run_l1_offline || L1_RC=$?
if [ "${LAYER}" = l1 ]; then
  # L1-only run (the ci/l1.sh merge-gate path): the offline render result IS the result.
  if [ "${L1_RC}" -eq 0 ]; then
    log "Done (L1 render). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
  else
    log "L1 render checks FAILED (rc=${L1_RC}). JUnit: ${JUNIT_OUT}"
  fi
  exit "${L1_RC}"
fi
if [ "${L1_RC}" -ne 0 ]; then
  log "L1 render checks FAILED (rc=${L1_RC}) — blocking (render failure is not cluster-gated)."
  exit "${L1_RC}"
fi

# run mode: need a cluster for L2/L3. Without one (Phase 0), emit skipped JUnit for the
# cluster-gated layers. (L1 already ran offline above and passed.)
if [ "${FORCE_NO_CLUSTER}" = 1 ] || ! conf_cluster_reachable; then
  log "No reachable cluster — emitting SKIPPED JUnit scaffold for L2/L3 (Phase 0 behavior)."
  log "Also running structural lint so scaffolds stay valid."
  lint_tests || { log "lint failed"; exit 1; }
  "${PYTHON}" "${CONF_ROOT}/lib/emit_skipped_junit.py" \
    --tests "${CONF_ROOT}/tests" --out "${JUNIT_OUT}"
  log "Done (scaffold mode). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
  exit 0
fi

command -v chainsaw >/dev/null 2>&1 || {
  echo "ERROR: cluster reachable but 'chainsaw' (${CHAINSAW_VERSION}) not on PATH." >&2
  echo "Install the pinned binary or run via the in-cluster Job (incluster/)." >&2
  exit 3
}
run_chainsaw
log "Done. JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"