home-network-k3s-impl/platform/monitoring/tests/run.sh

#!/usr/bin/env bash
# platform/monitoring/tests/run.sh — the repo's Prometheus RULE-UNIT-TEST harness (no cluster).
#
# WHY THIS EXISTS
# This repo has now shipped both halves of the alerting failure space to prod:
#   * a PERMANENTLY-FIRING rule (the CNPG alert fixed 2026-09-12), and
#   * a STRUCTURALLY INERT rule (VeleroNoRecentBackup: `time() - <absent series>` yields no
#     series, so a schedule that had NEVER backed up could not raise it — prod ran with no
#     volume backups at all and nothing alerted).
# Neither is visible by reading the expression, by `kubectl apply`, or by looking at a healthy
# live cluster: an inert rule and a correct rule are indistinguishable when everything is fine.
# `promtool test rules` is the tool that separates them, because it lets the BAD state be
# constructed on purpose and asserted on, offline and in CI. Every rule here is therefore
# pinned twice: quiet on the good state, firing on the bad one.
#
# The rules under test are the COMMITTED manifests (extract_rules.py flattens the
# PrometheusRule CRs into a rule file) — never a copy, so a fixture cannot drift green while
# the deployed rule rots.
#
# ONE HARNESS, REPO-WIDE (consolidated 2026-09-12). This started as the monitoring-only
# runner and there was a second, cnpg-only script (`platform/cnpg/run-alert-tests.sh`) doing
# the same three steps by hand. Two harnesses means two behaviours, and the one nothing ran
# rots — so the cnpg script is gone and this one covers EVERY PrometheusRule in the repo that
# has unit tests. It still lives under platform/monitoring/ because observability owns alert
# testing; the tests themselves live next to the rules they test.
#
# TWO TEST-FILE CONVENTIONS, both discovered automatically:
#   1. COMPANION — `<manifest>.promtool-test.yaml` beside `<manifest>.yaml`. The manifest under
#      test is that sibling. This is how a platform component ships tests with its own alerts
#      (platform/cnpg/alerts.promtool-test.yaml).
#   2. SUITE — `<dir>/tests/*.test.yaml`. The manifests under test are every PrometheusRule
#      file in `<dir>` (platform/monitoring/tests/backups.test.yaml covers alerts.yaml +
#      pvc-capacity-alerts.yaml, so one suite can assert across rule groups).
#
# WHAT THE ADAPTER DOES, AND WHY (inherited from the cnpg script's header, which was right):
#   * A PrometheusRule is a Kubernetes CR; promtool wants a bare rules file with `groups:` at
#     the top level. So `.spec.groups` is extracted (extract_rules.py), never re-typed.
#   * The extracted file is written under whatever name the test file's own `rule_files:`
#     declares, so a test file needs no knowledge of this harness.
#   * `promtool check rules` runs FIRST. A rules file that does not parse would make every
#     unit test "pass" vacuously by finding no rules at all.
#   * Everything generated goes to a temp dir. A generated file sitting next to its source is
#     how a stale copy becomes the thing under test — and it is also why leaving Flux's
#     `${BASE_DOMAIN}` placeholders verbatim (see extract_rules.py) is safe here.
#
# Usage:  ./run.sh                       # every test file in the repo (both conventions)
#         ./run.sh platform/cnpg/alerts.promtool-test.yaml   # one file
#         ./run.sh --list                # print `<test-file>\t<manifest>` pairs, run nothing
#
# `--list` exists so the L0 check can ask THIS script which manifests are under test instead of
# re-deriving the conventions itself. Two implementations of "which rules does this test cover"
# would eventually disagree, and the disagreement would show up as a file the gate believes is
# tested and the harness never touches. It needs no promtool, so it works on any host.
#
# Exit codes follow ci/lib.sh: 0 pass, 1 fail, 3 SKIPPED (toolchain unavailable — never green).
#
# PROMTOOL COMES FROM THE PINNED TOOLCHAIN, AND NOWHERE ELSE. `PROMTOOL_VERSION` in
# infra/devcontainer/versions.env (baked into the image) and ci/versions.env (installed on the
# CI runner), pinned to the Prometheus version prod actually runs — promtool IS the rule
# engine, so a test green under one version and red under another is worse than no test.
# The two run-time acquisition paths this harness and the cnpg script used to have — a
# `curl | tar -xz` of a GitHub release, and a `podman run` of quay.io/prometheus/prometheus —
# are both REMOVED. Not because a pinned download is unusual here (the Containerfile downloads
# every tool it bakes) but because those decided the version, the network and the machine
# SILENTLY, at test time, on whatever host ran the tests instead of once at reviewed
# image-build time; and each one quietly turned "the toolchain has no promtool" into a green
# run, which is how the toolchain stayed without one. Missing promtool is now exit 3, never a
# pass. Override with PROMTOOL=/path/to/promtool.
#
# ENFORCEMENT. This script is the developer entrypoint. The merge gate runs it through
# conformance/tests/l0-static/alert-rules-unit-tested, which additionally reports every
# PrometheusRule in the repo that has NO unit tests at all.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${HERE}/../../.." && pwd)"

LIST_ONLY=0
if [ "${1:-}" = "--list" ]; then LIST_ONLY=1; shift; fi

# ---------------------------------------------------------------- toolchain resolution
# Skipped for --list: that mode runs no rules, and the L0 check must be able to resolve the
# test->manifest mapping (and report the gaps) even on a host with no promtool.
PROMTOOL_BIN=""
if [ "$LIST_ONLY" = 0 ]; then
PROMTOOL_BIN="${PROMTOOL:-$(command -v promtool || true)}"
if [ -z "${PROMTOOL_BIN}" ]; then
  pinned="$(sed -n 's/^PROMTOOL_VERSION="\{0,1\}\([^"]*\)"\{0,1\}.*/\1/p' \
              "${ROOT}/ci/versions.env" 2>/dev/null | head -1)"
  {
    echo "run.sh: SKIP (exit 3, NOT a pass) — promtool is not on PATH, and this harness no"
    echo "  longer fetches one (no GitHub release download, no container-image pull)."
    echo "  promtool is pinned at v${pinned:-<see ci/versions.env>} — the Prometheus version prod runs:"
    echo "    * infra/devcontainer/versions.env  PROMTOOL_VERSION   (baked into the image)"
    echo "    * ci/versions.env                  PROMTOOL_VERSION   (installed on the CI runner)"
    echo "  Run it in the pinned toolbox:"
    echo "    ./infra/devcontainer/dev.sh build     # if the image predates the promtool pin"
    echo "    ./infra/devcontainer/dev.sh run bash -c 'platform/monitoring/tests/run.sh'"
    echo "  Or point at a local binary of the same version: PROMTOOL=/path/to/promtool ./run.sh"
  } >&2
  exit 3
fi
# A PROMTOOL= that does not resolve is an OPERATOR MISTAKE, not a missing toolchain: exit 2,
# so a typo'd override cannot masquerade as the honest "no promtool here" skip above.
if [ ! -x "${PROMTOOL_BIN}" ]; then
  echo "run.sh: PROMTOOL='${PROMTOOL_BIN}' is not an executable file — fix the override" >&2
  exit 2
fi
# Absolutised: each test runs with cwd inside a temp dir, where a relative PROMTOOL= breaks
# (it did, during development).
PROMTOOL_BIN="$(cd "$(dirname "${PROMTOOL_BIN}")" && pwd)/$(basename "${PROMTOOL_BIN}")"
fi

# ---------------------------------------------------------------- which test files to run
# NOT `git ls-files`: the gate runs inside a container that mounts only the worktree, whose
# `.git` file points outside the mount, so git cannot read the index there — and with
# `pipefail` that failure silently emptied the discovery, i.e. a green run over ZERO tests
# (observed 2026-09-12). `find` needs no repo metadata, so discovery cannot be disabled by
# how the tree happens to be mounted. .git/ is pruned so a stale object copy is never picked up,
# and .claude/ with it: .claude/worktrees/agent-*/ holds full repo copies for isolated agent
# runs, so an unpruned find reported the SAME manifest under test once per worktree (up to 18
# times), and the L0 check that compares this listing against the tree failed on paths that
# are not authored surface at all (.claude/ is gitignored, .gitignore:43).
discover() {
  # Companion convention, anywhere in the tree.
  find "${ROOT}" \( -path "${ROOT}/.git" -o -path "${ROOT}/.claude" \) -prune -o -name '*.promtool-test.yaml' -print 2>/dev/null
  # Suite convention, ANYWHERE in the tree — not just this directory.
  #
  # This used to be `find "${HERE}" -maxdepth 1`, which made the suite convention a
  # platform/monitoring-only privilege while the header above documented it as a general one. The
  # consequence was silent: a suite added under any other component (apps/wave-0/tests/, say) was
  # never discovered, never run, and still LOOKED like coverage in review — the same shape of hole
  # as the rules this harness exists to test. Found 2026-09-13 adding apps/wave-*/tests/. Scoped by
  # NAME only; manifests_for() then refuses (loudly) any *.test.yaml that is not inside a tests/
  # directory, so a misplaced file fails rather than disappearing.
  find "${ROOT}" \( -path "${ROOT}/.git" -o -path "${ROOT}/.claude" \) -prune -o -name '*.test.yaml' -print 2>/dev/null
}

TESTS=()
if [ "$#" -gt 0 ]; then
  for t in "$@"; do
    # Accept absolute, repo-relative, or cwd-relative paths.
    if [ -f "$t" ]; then TESTS+=("$(cd "$(dirname "$t")" && pwd)/$(basename "$t")")
    elif [ -f "${ROOT}/$t" ]; then TESTS+=("${ROOT}/$t")
    else echo "run.sh: no such test file: $t" >&2; exit 2
    fi
  done
else
  while IFS= read -r abs; do
    [ -n "$abs" ] && TESTS+=("$abs")
  done < <(discover | sort -u)
fi

if [ "${#TESTS[@]}" -eq 0 ]; then
  echo "run.sh: SKIP (exit 3) — no *.promtool-test.yaml or tests/*.test.yaml found in ${ROOT}" >&2
  exit 3
fi

# ---------------------------------------------------------------- manifest resolution
# Which committed PrometheusRule files a given test file tests. The conventions are the
# CONTRACT (see the header); anything else is refused rather than guessed at, because a test
# silently run against the wrong (or an empty) rule set is the vacuous pass this harness exists
# to prevent.
manifests_for() {
  local test_abs="$1" base dir
  base="$(basename "$test_abs")"
  dir="$(dirname "$test_abs")"
  case "$base" in
    *.promtool-test.yaml)
      local sib="${dir}/${base%.promtool-test.yaml}.yaml"
      if [ ! -f "$sib" ]; then
        echo "run.sh: ${test_abs#"${ROOT}/"} has no companion manifest (${sib#"${ROOT}/"})" >&2
        return 1
      fi
      printf '%s\n' "$sib" ;;
    *.test.yaml)
      if [ "$(basename "$dir")" != tests ]; then
        echo "run.sh: ${test_abs#"${ROOT}/"}: a *.test.yaml must live in a tests/ directory so its" >&2
        echo "  manifests are the PrometheusRules of the component above it" >&2
        return 1
      fi
      local comp; comp="$(dirname "$dir")"
      local found=0 f
      while IFS= read -r f; do
        found=1; printf '%s\n' "$f"
      done < <(grep -rl --include='*.yaml' -e '^[[:space:]]*kind:[[:space:]]*PrometheusRule[[:space:]]*$' \
                 "$comp" --exclude-dir=tests 2>/dev/null | sort)
      if [ "$found" = 0 ]; then
        echo "run.sh: ${test_abs#"${ROOT}/"}: no PrometheusRule manifest in ${comp#"${ROOT}/"}" >&2
        return 1
      fi ;;
    *)
      echo "run.sh: ${test_abs#"${ROOT}/"}: unrecognised test-file name; use" >&2
      echo "  <manifest>.promtool-test.yaml (companion) or tests/<name>.test.yaml (suite)" >&2
      return 1 ;;
  esac
}

# The rule-file name(s) a test file expects, from its own `rule_files:` block — so the
# extracted rules land under the name the test already references and no test file needs to
# know this harness exists.
rule_file_names() {
  sed -n '/^rule_files:[[:space:]]*$/,/^[^[:space:]-]/p' "$1" \
    | sed -n 's/^[[:space:]]*-[[:space:]]*//p' | sed 's/[[:space:]]*$//' | grep . || true
}

# ------------------------------------------------------------------------------- --list
if [ "$LIST_ONLY" = 1 ]; then
  lrc=0
  for test_abs in "${TESTS[@]}"; do
    mapfile -t mans < <(manifests_for "$test_abs") || { lrc=1; continue; }
    for m in "${mans[@]}"; do
      printf '%s\t%s\n' "${test_abs#"${ROOT}/"}" "${m#"${ROOT}/"}"
    done
  done
  exit "$lrc"
fi

# ---------------------------------------------------------------------------------- run
rc=0
for test_abs in "${TESTS[@]}"; do
  rel="${test_abs#"${ROOT}/"}"
  mapfile -t mans < <(manifests_for "$test_abs") || { rc=1; continue; }
  mapfile -t rule_names < <(rule_file_names "$test_abs")
  if [ "${#rule_names[@]}" -eq 0 ]; then
    echo "run.sh: ${rel}: no \`rule_files:\` entries — promtool would test nothing" >&2
    rc=1; continue
  fi

  work="$(mktemp -d)"
  # The test file references its rule file by bare name, so everything sits in one directory.
  gen="${work}/${rule_names[0]}"
  ex_rc=0
  python3 "${HERE}/extract_rules.py" --out "$gen" "${mans[@]}" || ex_rc=$?
  if [ "$ex_rc" -eq 4 ]; then
    echo "run.sh: SKIP (exit 3) — PyYAML unavailable, cannot flatten the PrometheusRule manifests" >&2
    rm -rf "$work"; exit 3
  elif [ "$ex_rc" -ne 0 ]; then
    rm -rf "$work"; rc=1; continue
  fi
  # A test file may legally name the same rule file more than once, or several names.
  for n in "${rule_names[@]:1}"; do cp "$gen" "${work}/${n}"; done
  cp "$test_abs" "${work}/"

  # Syntax gate first: rules that do not parse would make every unit test pass vacuously.
  echo "run.sh: promtool check rules  ${rel}  (manifests: ${mans[*]#"${ROOT}/"})" >&2
  ( cd "$work" && "$PROMTOOL_BIN" check rules "${rule_names[0]}" ) || rc=1
  echo "run.sh: promtool test rules   ${rel}" >&2
  ( cd "$work" && "$PROMTOOL_BIN" test rules "$(basename "$test_abs")" ) || rc=1
  rm -rf "$work"
done

if [ "$rc" -eq 0 ]; then
  echo "run.sh: OK — ${#TESTS[@]} rule-unit test file(s) passed under $("$PROMTOOL_BIN" --version 2>&1 | head -1)" >&2
fi
exit "$rc"