home-network-k3s-impl/conformance/lib/common.sh

#!/usr/bin/env bash
# Shared helpers for the conformance harness. Sourced by run.sh; not executed directly.
# shellcheck shell=bash

# --- Requirement-tagging convention (single source of truth) --------------------------
# A Chainsaw Test declares the requirement(s) it verifies with a per-requirement label:
#
#     metadata:
#       labels:
#         conformance.home.k3s/R-NET-01: "true"     # one label per verified R-* ID
#       annotations:
#         conformance.home.k3s/requirements: "R-NET-01, R-NET-02"   # human summary
#         conformance.home.k3s/layer: "L2"
#         conformance.home.k3s/safety-critical: "false"
#
# The LABEL is authoritative and does double duty:
#   * `chainsaw test --selector conformance.home.k3s/R-NET-01=true` runs the subset.
#   * lib/coverage.py reads the same labels to map requirement -> verifying test(s).
# The annotation is a convenience for humans/reports and MUST agree with the labels
# (coverage.py flags a mismatch). This is the convention referenced in README.md.
export CONF_LABEL_PREFIX="conformance.home.k3s"

# Absolute path to the conformance root (dir containing run.sh). Set by run.sh.
: "${CONF_ROOT:?CONF_ROOT must be set before sourcing common.sh}"

# --- Test discovery -------------------------------------------------------------------
# Every leaf test is a directory containing a `chainsaw-test.yaml`.
conf_all_test_files() {
  find "${CONF_ROOT}/tests" -type f -name 'chainsaw-test.yaml' 2>/dev/null | sort
}

# Directories of tests that carry ANY of the given requirement labels (space-separated
# R-IDs). Used to translate `--requirements R-NET-01,R-NET-06` into a set of test dirs
# so we run each matching test exactly once (union, deduplicated).
conf_dirs_for_requirements() {
  local reqs="$1" f dir rid matched
  while IFS= read -r f; do
    [ -n "$f" ] || continue
    matched=0
    for rid in $reqs; do
      # Match a label line:  conformance.home.k3s/R-NET-01: "true"
      if grep -Eq "^[[:space:]]*${CONF_LABEL_PREFIX}/${rid}:[[:space:]]*[\"']?true" "$f"; then
        matched=1; break
      fi
    done
    [ "$matched" = 1 ] && dirname "$f"
  done < <(conf_all_test_files) | sort -u
}

# --- DECLARED SCOPE: whole-repo invariants -------------------------------------------
# A test may also declare the SCOPE of the invariant it asserts:
#
#     metadata:
#       annotations:
#         conformance.home.k3s/scope: "whole-repo"
#
# WHY THIS EXISTS — the requirement-scoped-selection blind spot (R-TEST-03)
# ------------------------------------------------------------------------
# The merge gate runs the conformance subset tagged with the requirements a change
# TOUCHES (ci/touched-requirements.sh -> `--requirements`). That is right for an invariant
# whose input surface is the subtree its requirements map to. It is WRONG — silently, and in
# the worst direction — for an invariant that quantifies over the WHOLE tracked tree, or over
# subtrees (apps/, platform/<most>, infra/) that the coarse path->requirement map keys to
# NOTHING. For those, the change most likely to break the invariant attributes no requirement
# at all, so the requirement-scoped selection filters the check out of EXACTLY that change.
# Three real incidents came out of that blind spot; each is recorded in the header of the
# test that now guards it (flux-postbuild-substitution, keycloak-client-secret-reconciler,
# bulk-tier-distinct-from-replicated).
#
# The repo's previous answer was a hand-written "UNCONDITIONAL" stage per such test in
# ci/l0.sh — which does not scale and fails silently in the worst way: whoever adds the NEXT
# whole-repo invariant must REMEMBER to hand-write a stage, and if they forget, the check runs
# only when some unrelated requirement happens to be touched. The scope annotation replaces
# that with a DECLARATION the runner enforces: `run.sh --include-scope whole-repo` unions
# every test carrying it into the selection, so adding the annotation is the only step.
#
# TWO things follow from the annotation, and both are deliberate:
#   1. the test is ALWAYS selected (union with the requirement-scoped subset), and
#   2. a SKIP of that test is UNWIRED, NOT GREEN (lib/run_l1_offline.py --unwired-scope).
#      A whole-repo invariant that reports "nothing to audit" is reporting that the subtree
#      it audits has VANISHED, which is not a pass. This is the rc-3 semantics the bespoke
#      stages encoded ("check SKIPPED (clusters/ absent) — not a pass"), preserved verbatim.
export CONF_SCOPE_ANNOTATION="${CONF_LABEL_PREFIX}/scope"

# Directories of tests DECLARING the given scope (e.g. "whole-repo"). The annotation is
# matched on a real YAML mapping line only: a `#` comment cannot start with the key, so a
# scope named in a test's header prose is documentation, not a declaration (the same
# "reference, not declaration" rule ci/touched-requirements.sh applies to R-IDs).
conf_dirs_for_scope() {
  local scope="$1" f
  [ -n "$scope" ] || return 0
  while IFS= read -r f; do
    [ -n "$f" ] || continue
    if grep -Eq "^[[:space:]]*${CONF_SCOPE_ANNOTATION}:[[:space:]]*[\"']?${scope}[\"']?[[:space:]]*$" "$f"; then
      dirname "$f"
    fi
  done < <(conf_all_test_files) | sort -u
}

# Union of conf_dirs_for_scope over a comma/space separated list of scopes.
conf_dirs_for_scopes() {
  local scopes="$1" s
  for s in ${scopes//,/ }; do conf_dirs_for_scope "$s"; done | sort -u
}

# Directories for a given layer (subtree): l0 | l1 | l2 | l3 | meta | all.
# Accepts a comma/space separated LIST of layers (e.g. "l2,l3") and returns the UNION of
# their test dirs, deduplicated. A single layer keeps its prior behavior (backward compatible);
# this is what lets `run.sh --layer l2,l3` (the ci/conformance.sh L2/L3 stage) select both
# layers in one pass instead of erroring on an unknown "l2,l3" layer.
conf_dirs_for_layer() {
  local layers="$1" layer
  local -a roots=()
  for layer in ${layers//,/ }; do
    case "$layer" in
      all)  roots=("${CONF_ROOT}/tests"); break ;;
      l0)   roots+=("${CONF_ROOT}/tests/l0-static") ;;
      l1)   roots+=("${CONF_ROOT}/tests/l1-rendering") ;;
      l2)   roots+=("${CONF_ROOT}/tests/l2-behavioral") ;;
      l3)   roots+=("${CONF_ROOT}/tests/l3-fault-injection") ;;
      meta) roots+=("${CONF_ROOT}/tests/meta") ;;
      *) echo "unknown layer: $layer" >&2; return 2 ;;
    esac
  done
  [ "${#roots[@]}" -gt 0 ] || return 0
  find "${roots[@]}" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; 2>/dev/null | sort -u
}

# Normalize a comma/space separated requirement list to space-separated, uppercased.
conf_normalize_reqs() {
  echo "$1" | tr ',' ' ' | tr '[:lower:]' '[:upper:]' | xargs
}

# Is a live cluster reachable? (Preflight for L1/L2/L3 execution.)
conf_cluster_reachable() {
  command -v kubectl >/dev/null 2>&1 || return 1
  kubectl cluster-info >/dev/null 2>&1
}