home-network-k3s-impl/conformance/run.sh

#!/usr/bin/env bash
###############################################################################
# conformance/run.sh — the single-command conformance runner (R-TEST-02/03/04)
#
# ONE command runs the suite (or a requirement-/layer-scoped subset) and always
# emits JUnit + a requirement-coverage report. Local runs and the in-cluster CI
# Chainsaw Job invoke this same script so results are identical (reproducibility).
#
#   ./run.sh                          # full suite; skips L1/L2/L3 if no cluster
#   ./run.sh --requirements R-NET-01,R-NET-06   # merge-gate subset (tagged)
#   ./run.sh --layer l2               # one layer
#   ./run.sh --requirements R-NET-01,R-NET-02 --layer l2   # intersect: tagged AND in layer
#   ./run.sh --coverage-only          # regenerate coverage report only (no cluster)
#   ./run.sh --lint                   # structural-validate test defs (no cluster)
#   ./run.sh --strict-coverage        # hygiene gate: fail on tag/annotation drift
#   ./run.sh --requirements R-NET-01  # ALSO asserts those IDs are covered (merge gate)
#   ./run.sh --require-all            # phase-exit: every requirement must be covered
#
# Artifacts (always written under conformance/_artifacts/, gitignored):
#   _artifacts/report.junit.xml        JUnit (Chainsaw output, or skipped-scaffold shim)
#   _artifacts/coverage/coverage.md    human coverage report
#   _artifacts/coverage/coverage.json  machine coverage report
#
# Phase 0: no cluster exists. The behavior tests are valid scaffolds that cannot
# execute yet, so run.sh emits a *skipped* JUnit and a coverage report, and exits
# green — the harness is proven reproducible while later phases fill in behavior.
###############################################################################
set -euo pipefail

CONF_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export CONF_ROOT
# shellcheck source=lib/common.sh
source "${CONF_ROOT}/lib/common.sh"
# shellcheck source=versions.env
source "${CONF_ROOT}/versions.env"

ARTIFACTS="${CONF_ROOT}/_artifacts"
COVERAGE_OUT="${ARTIFACTS}/coverage"
JUNIT_OUT="${ARTIFACTS}/report.junit.xml"
PYTHON="${PYTHON:-python3}"

LAYER="all"
REQS=""
MODE="run"          # run | coverage-only | lint
STRICT_COVERAGE=0
REQUIRE_ALL=0
FORCE_NO_CLUSTER=0

# Documented, out-of-scope coverage gaps exempt from the completeness gate: a
# safety-critical requirement owned by another workstream that WS-TEST does not yet
# scaffold. R-STATE-02 (node-loss) is delivered with Longhorn/infra (WS-STORAGE).
# Listed here so the exemption is explicit and reviewable, not silent.
DOCUMENTED_GAPS="R-STATE-02"

usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }

while [ $# -gt 0 ]; do
  case "$1" in
    --layer)          LAYER="$2"; shift 2 ;;
    --requirements)   REQS="$(conf_normalize_reqs "$2")"; shift 2 ;;
    --coverage-only)  MODE="coverage-only"; shift ;;
    --lint)           MODE="lint"; shift ;;
    --strict-coverage) STRICT_COVERAGE=1; shift ;;
    --require-all)    REQUIRE_ALL=1; shift ;;
    --no-cluster)     FORCE_NO_CLUSTER=1; shift ;;
    -h|--help)        usage; exit 0 ;;
    *) echo "unknown arg: $1" >&2; usage; exit 2 ;;
  esac
done

mkdir -p "${ARTIFACTS}" "${COVERAGE_OUT}"

log() { printf '\033[1m[run.sh]\033[0m %s\n' "$*"; }

# --- Coverage report is always produced (cluster-independent, R-TEST-04) --------------
gen_coverage() {
  local flags=(--allow-gap "${DOCUMENTED_GAPS}")
  [ "${STRICT_COVERAGE}" = 1 ] && flags+=(--strict)
  # A tagged subset run is also the merge gate: the touched requirements MUST be covered.
  [ -n "${REQS}" ] && flags+=(--require "${REQS}")
  [ "${REQUIRE_ALL}" = 1 ] && flags+=(--require-all)
  "${PYTHON}" "${CONF_ROOT}/lib/coverage.py" \
    --registry "${CONF_ROOT}/requirements/registry.yaml" \
    --tests "${CONF_ROOT}/tests" \
    --out "${COVERAGE_OUT}" \
    "${flags[@]}"
}

# --- Structural lint (no cluster): parse each test def and check the tag convention ----
lint_tests() {
  log "Linting test definitions (structural, no cluster)"
  local rc=0 f
  if command -v chainsaw >/dev/null 2>&1; then
    # Prefer chainsaw's own linter when the pinned binary is available.
    while IFS= read -r f; do
      chainsaw lint test --file "$f" >/dev/null || { echo "LINT FAIL: $f" >&2; rc=1; }
    done < <(conf_all_test_files)
  else
    log "chainsaw not on PATH; falling back to structural check"
    # Minimal YAML-shape check: kind Test + at least one requirement label per file.
    while IFS= read -r f; do
      grep -Eq '^kind:\s*Test' "$f" || { echo "LINT FAIL (no kind: Test): $f" >&2; rc=1; }
      grep -Eq "^[[:space:]]*${CONF_LABEL_PREFIX}/R-[A-Z0-9]+-[0-9]+:" "$f" \
        || { echo "LINT FAIL (no requirement label): $f" >&2; rc=1; }
    done < <(conf_all_test_files)
  fi
  return $rc
}

# --- Select which test directories Chainsaw should run --------------------------------
select_dirs() {
  # --requirements AND --layer together => intersection (tests carrying the tag
  # AND living in that layer subtree). Either alone keeps its prior meaning, so
  # this is backward compatible (LAYER defaults to "all"). Both helpers emit
  # sorted, de-duplicated dirs, so `comm -12` yields the intersection.
  if [ -n "${REQS}" ] && [ "${LAYER}" != all ]; then
    comm -12 \
      <(conf_dirs_for_requirements "${REQS}") \
      <(conf_dirs_for_layer "${LAYER}")
  elif [ -n "${REQS}" ]; then
    conf_dirs_for_requirements "${REQS}"
  else
    conf_dirs_for_layer "${LAYER}"
  fi
}

# --- Execute the Chainsaw suite against a live cluster ---------------------------------
run_chainsaw() {
  local -a dirs
  mapfile -t dirs < <(select_dirs)
  if [ "${#dirs[@]}" -eq 0 ]; then
    log "No tests match the selection (layer=${LAYER} reqs='${REQS}')."
    return 0
  fi
  local -a test_dir_args=()
  for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done

  log "Running Chainsaw over ${#dirs[@]} test dir(s) with pinned ${CHAINSAW_VERSION}"
  # Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts).
  chainsaw test \
    --config "${CONF_ROOT}/config/chainsaw-config.yaml" \
    "${test_dir_args[@]}"

  # Normalize whatever chainsaw named the report to a stable path CI collects.
  local produced
  produced="$(find "${ARTIFACTS}" -maxdepth 1 -name 'chainsaw-report*.xml' -o \
                   -name 'report*.xml' 2>/dev/null | grep -v report.junit.xml | head -1 || true)"
  if [ -n "${produced}" ] && [ "${produced}" != "${JUNIT_OUT}" ]; then
    cp "${produced}" "${JUNIT_OUT}"
  fi
}

###############################################################################
# Main
###############################################################################
log "Coverage report first (always)"
gen_coverage

case "${MODE}" in
  coverage-only)
    log "coverage-only: done. See ${COVERAGE_OUT}/coverage.md"
    exit 0
    ;;
  lint)
    lint_tests
    log "lint: done."
    exit 0
    ;;
esac

# run mode: need a cluster for L1/L2/L3. Without one (Phase 0), emit skipped JUnit.
if [ "${FORCE_NO_CLUSTER}" = 1 ] || ! conf_cluster_reachable; then
  log "No reachable cluster — emitting SKIPPED JUnit scaffold (Phase 0 behavior)."
  log "Also running structural lint so scaffolds stay valid."
  lint_tests || { log "lint failed"; exit 1; }
  "${PYTHON}" "${CONF_ROOT}/lib/emit_skipped_junit.py" \
    --tests "${CONF_ROOT}/tests" --out "${JUNIT_OUT}"
  log "Done (scaffold mode). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
  exit 0
fi

command -v chainsaw >/dev/null 2>&1 || {
  echo "ERROR: cluster reachable but 'chainsaw' (${CHAINSAW_VERSION}) not on PATH." >&2
  echo "Install the pinned binary or run via the in-cluster Job (incluster/)." >&2
  exit 3
}
run_chainsaw
log "Done. JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"