#!/usr/bin/env bash # Shared helpers for the conformance harness. Sourced by run.sh; not executed directly. # shellcheck shell=bash # --- Requirement-tagging convention (single source of truth) -------------------------- # A Chainsaw Test declares the requirement(s) it verifies with a per-requirement label: # # metadata: # labels: # conformance.home.k3s/R-NET-01: "true" # one label per verified R-* ID # annotations: # conformance.home.k3s/requirements: "R-NET-01, R-NET-02" # human summary # conformance.home.k3s/layer: "L2" # conformance.home.k3s/safety-critical: "false" # # The LABEL is authoritative and does double duty: # * `chainsaw test --selector conformance.home.k3s/R-NET-01=true` runs the subset. # * lib/coverage.py reads the same labels to map requirement -> verifying test(s). # The annotation is a convenience for humans/reports and MUST agree with the labels # (coverage.py flags a mismatch). This is the convention referenced in README.md. export CONF_LABEL_PREFIX="conformance.home.k3s" # Absolute path to the conformance root (dir containing run.sh). Set by run.sh. : "${CONF_ROOT:?CONF_ROOT must be set before sourcing common.sh}" # --- Test discovery ------------------------------------------------------------------- # Every leaf test is a directory containing a `chainsaw-test.yaml`. conf_all_test_files() { find "${CONF_ROOT}/tests" -type f -name 'chainsaw-test.yaml' 2>/dev/null | sort } # Directories of tests that carry ANY of the given requirement labels (space-separated # R-IDs). Used to translate `--requirements R-NET-01,R-NET-06` into a set of test dirs # so we run each matching test exactly once (union, deduplicated). conf_dirs_for_requirements() { local reqs="$1" f dir rid matched while IFS= read -r f; do [ -n "$f" ] || continue matched=0 for rid in $reqs; do # Match a label line: conformance.home.k3s/R-NET-01: "true" if grep -Eq "^[[:space:]]*${CONF_LABEL_PREFIX}/${rid}:[[:space:]]*[\"']?true" "$f"; then matched=1; break fi done [ "$matched" = 1 ] && dirname "$f" done < <(conf_all_test_files) | sort -u } # --- DECLARED SCOPE: whole-repo invariants ------------------------------------------- # A test may also declare the SCOPE of the invariant it asserts: # # metadata: # annotations: # conformance.home.k3s/scope: "whole-repo" # # WHY THIS EXISTS — the requirement-scoped-selection blind spot (R-TEST-03) # ------------------------------------------------------------------------ # The merge gate runs the conformance subset tagged with the requirements a change # TOUCHES (ci/touched-requirements.sh -> `--requirements`). That is right for an invariant # whose input surface is the subtree its requirements map to. It is WRONG — silently, and in # the worst direction — for an invariant that quantifies over the WHOLE tracked tree, or over # subtrees (apps/, platform/, infra/) that the coarse path->requirement map keys to # NOTHING. For those, the change most likely to break the invariant attributes no requirement # at all, so the requirement-scoped selection filters the check out of EXACTLY that change. # Three real incidents came out of that blind spot; each is recorded in the header of the # test that now guards it (flux-postbuild-substitution, keycloak-client-secret-reconciler, # bulk-tier-distinct-from-replicated). # # The repo's previous answer was a hand-written "UNCONDITIONAL" stage per such test in # ci/l0.sh — which does not scale and fails silently in the worst way: whoever adds the NEXT # whole-repo invariant must REMEMBER to hand-write a stage, and if they forget, the check runs # only when some unrelated requirement happens to be touched. The scope annotation replaces # that with a DECLARATION the runner enforces: `run.sh --include-scope whole-repo` unions # every test carrying it into the selection, so adding the annotation is the only step. # # TWO things follow from the annotation, and both are deliberate: # 1. the test is ALWAYS selected (union with the requirement-scoped subset), and # 2. a SKIP of that test is UNWIRED, NOT GREEN (lib/run_l1_offline.py --unwired-scope). # A whole-repo invariant that reports "nothing to audit" is reporting that the subtree # it audits has VANISHED, which is not a pass. This is the rc-3 semantics the bespoke # stages encoded ("check SKIPPED (clusters/ absent) — not a pass"), preserved verbatim. export CONF_SCOPE_ANNOTATION="${CONF_LABEL_PREFIX}/scope" # Directories of tests DECLARING the given scope (e.g. "whole-repo"). The annotation is # matched on a real YAML mapping line only: a `#` comment cannot start with the key, so a # scope named in a test's header prose is documentation, not a declaration (the same # "reference, not declaration" rule ci/touched-requirements.sh applies to R-IDs). conf_dirs_for_scope() { local scope="$1" f [ -n "$scope" ] || return 0 while IFS= read -r f; do [ -n "$f" ] || continue if grep -Eq "^[[:space:]]*${CONF_SCOPE_ANNOTATION}:[[:space:]]*[\"']?${scope}[\"']?[[:space:]]*$" "$f"; then dirname "$f" fi done < <(conf_all_test_files) | sort -u } # Union of conf_dirs_for_scope over a comma/space separated list of scopes. conf_dirs_for_scopes() { local scopes="$1" s for s in ${scopes//,/ }; do conf_dirs_for_scope "$s"; done | sort -u } # Directories for a given layer (subtree): l0 | l1 | l2 | l3 | meta | all. # Accepts a comma/space separated LIST of layers (e.g. "l2,l3") and returns the UNION of # their test dirs, deduplicated. A single layer keeps its prior behavior (backward compatible); # this is what lets `run.sh --layer l2,l3` (the ci/conformance.sh L2/L3 stage) select both # layers in one pass instead of erroring on an unknown "l2,l3" layer. conf_dirs_for_layer() { local layers="$1" layer local -a roots=() for layer in ${layers//,/ }; do case "$layer" in all) roots=("${CONF_ROOT}/tests"); break ;; l0) roots+=("${CONF_ROOT}/tests/l0-static") ;; l1) roots+=("${CONF_ROOT}/tests/l1-rendering") ;; l2) roots+=("${CONF_ROOT}/tests/l2-behavioral") ;; l3) roots+=("${CONF_ROOT}/tests/l3-fault-injection") ;; meta) roots+=("${CONF_ROOT}/tests/meta") ;; *) echo "unknown layer: $layer" >&2; return 2 ;; esac done [ "${#roots[@]}" -gt 0 ] || return 0 find "${roots[@]}" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; 2>/dev/null | sort -u } # Normalize a comma/space separated requirement list to space-separated, uppercased. conf_normalize_reqs() { echo "$1" | tr ',' ' ' | tr '[:lower:]' '[:upper:]' | xargs } # Is a live cluster reachable? (Preflight for L1/L2/L3 execution.) conf_cluster_reachable() { command -v kubectl >/dev/null 2>&1 || return 1 kubectl cluster-info >/dev/null 2>&1 }