diff --git a/.gitea/workflows/scripts/install-tools.sh b/.gitea/workflows/scripts/install-tools.sh index 64c930e..c428195 100755 --- a/.gitea/workflows/scripts/install-tools.sh +++ b/.gitea/workflows/scripts/install-tools.sh @@ -39,6 +39,21 @@ install_chainsaw() { curl -fsSL "$url" | tar -xz -C "$bindir" chainsaw } +# promtool — the alert-rule UNIT-TEST engine the L0 check +# conformance/tests/l0-static/alert-rules-unit-tested runs. Installed for the l0 stage because +# without it that check exits 3 (SKIPPED, never green) on the runner and the alert unit tests +# would be enforced nowhere — the exact "coverage that looks present" failure it exists to end. +# Pinned from ci/versions.env to the Prometheus version prod runs (promtool IS the rule engine); +# the check asserts `promtool --version` equals that pin, so a drifted install FAILS, not skips. +install_promtool() { + have promtool && return 0 + log "promtool ${PROMTOOL_VERSION} (from the prometheus release tarball)" + local tarball="prometheus-${PROMTOOL_VERSION}.${os}-${arch}" + curl -fsSL "https://github.com/prometheus/prometheus/releases/download/v${PROMTOOL_VERSION}/${tarball}.tar.gz" \ + | tar -xz -C "$bindir" --strip-components=1 "${tarball}/promtool" + chmod +x "${bindir}/promtool" +} + install_kubectl() { have kubectl && return 0 log "kubectl ${KUBECTL_VERSION}" @@ -50,10 +65,10 @@ have() { command -v "$1" >/dev/null 2>&1; } case "$stage" in lint) install_yamllint ;; - l0) install_yamllint; install_check_jsonschema; install_kyverno ;; # schema (check-jsonschema) + Kyverno L0 + l0) install_yamllint; install_check_jsonschema; install_kyverno; install_promtool ;; # schema (check-jsonschema) + Kyverno L0 + alert unit tests l1) install_chainsaw ;; # L1 render via conformance runner (Chainsaw) conformance) install_chainsaw; install_kubectl ;; # trigger the in-cluster Job - all) install_yamllint; install_check_jsonschema; install_kyverno; install_chainsaw; install_kubectl ;; + all) install_yamllint; install_check_jsonschema; install_kyverno; install_promtool; install_chainsaw; install_kubectl ;; *) echo "unknown stage: $stage" >&2; exit 2 ;; esac log "stage '${stage}' tools ready" diff --git a/ci/Makefile b/ci/Makefile index c51a1ee..3d426da 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -12,7 +12,7 @@ CI_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) .DEFAULT_GOAL := help -.PHONY: help spec-immutability lint schema l0 render-golden l1 prom-rules conformance gate all touched versions tools test +.PHONY: help spec-immutability lint schema l0 render-golden l1 conformance gate all touched versions tools test help: ## Show this help @awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " \033[36m%-14s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST) @@ -32,9 +32,6 @@ render-golden: ## render==golden: canonical render reproduces committed apps/ (n l1: ## L1 rendering checks (no cluster; live now) @"$(CI_DIR)l1.sh" -prom-rules: ## promtool rule-unit tests for platform/monitoring alerts (no cluster; live now) - @"$(CI_DIR)prom-rules.sh" - conformance: ## L2/L3 in-cluster Chainsaw Job (SKIPS w/o CLUSTER_AVAILABLE=true) @"$(CI_DIR)conformance.sh" diff --git a/ci/README.md b/ci/README.md index e97cc3f..752e049 100644 --- a/ci/README.md +++ b/ci/README.md @@ -22,7 +22,6 @@ Individual stages: make -C ci lint # yaml/lint (no cluster; live now) make -C ci l0 # schema + Kyverno + L0 static conformance (no cluster; live now) make -C ci l1 # L1 rendering checks (no cluster; live now) -make -C ci prom-rules # promtool rule-unit tests on alerts (no cluster; live now) make -C ci conformance # L2/L3 in-cluster Chainsaw Job (SKIPS unless CLUSTER_AVAILABLE=true) make -C ci test # unit tests for these scripts themselves (ci/tests/*.test.sh) ``` @@ -103,14 +102,13 @@ in that case — nothing goes untested — and says plainly that the *scoped* ch ``` ci/ - Makefile targets: lint · l0 · l1 · prom-rules · conformance · gate · touched · test · tools · versions + Makefile targets: lint · l0 · l1 · conformance · gate · touched · test · tools · versions versions.env PINNED tool versions (R-TEST-02) — single source of truth lib.sh shared helpers: logging, tool checks, cluster detection, artifact paths yamllint.yaml lint config for the yaml/lint stage lint.sh Stage: yaml/lint l0.sh Stage: schema validation + Kyverno L0 gate + L0 static conformance l1.sh Stage: L1 rendering checks (Chainsaw L1 via the conformance runner) - prom-rules.sh Stage: `promtool test rules` over platform/monitoring's PrometheusRules (harness: platform/monitoring/tests/). Catches the two alerting bugs no other stage can see — permanently-firing and structurally INERT rules. conformance.sh Stage: L2/L3 in-cluster Chainsaw Job (trigger + collect) diff --git a/ci/gate.sh b/ci/gate.sh index d798cc7..8df27b4 100755 --- a/ci/gate.sh +++ b/ci/gate.sh @@ -42,7 +42,6 @@ run_stage lint "${CI_DIR}/lint.sh" run_stage l0 "${CI_DIR}/l0.sh" run_stage render-golden "${CI_DIR}/render-golden.sh" run_stage l1 "${CI_DIR}/l1.sh" -run_stage prom-rules "${CI_DIR}/prom-rules.sh" run_stage conformance "${CI_DIR}/conformance.sh" echo >&2 @@ -87,7 +86,7 @@ esac # Phase 0 that is tolerated (placeholders), but surfaced. render-golden is a # no-cluster reproducibility gate (canonical render == committed apps/ goldens): # a FAIL means a golden drifted from its ServiceDefinition source — always blocks. -for s in lint l0 render-golden l1 prom-rules; do +for s in lint l0 render-golden l1; do case "${OUTCOME[$s]}" in fail) err "${s}: FAILED -> blocks merge"; BLOCK=1 ;; skip) warn "${s}: skipped (unwired seam / missing tool) — not green" ;; diff --git a/ci/prom-rules.sh b/ci/prom-rules.sh deleted file mode 100755 index a849645..0000000 --- a/ci/prom-rules.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# ci/prom-rules.sh — Stage: Prometheus RULE-UNIT tests (no cluster; live now). -# -# Thin wrapper over platform/monitoring/tests/run.sh, which runs `promtool test rules` against the -# COMMITTED PrometheusRule manifests. It is a gate stage because the two alerting defects this repo -# has actually shipped — a permanently-firing rule and a structurally INERT one — are both invisible -# to every other stage: yamllint parses them, Kyverno admits them, and a healthy cluster cannot tell -# a correct alert from one that can never fire. Only the bad state, constructed on purpose, can. -# -# Exit codes per ci/lib.sh: 0 pass / 1 fail / 3 skipped (never green). -set -euo pipefail -CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck disable=SC1091 -source "${CI_DIR}/lib.sh" - -log "Stage: Prometheus rule-unit tests (promtool test rules; no cluster)" - -RUNNER="${REPO_ROOT}/platform/monitoring/tests/run.sh" -if [ ! -x "$RUNNER" ]; then - skip "rule-unit harness not present at ${RUNNER} — UNWIRED, not green" - summary "- prom-rules: **SKIPPED** (harness missing)" - exit 3 -fi - -set +e -"$RUNNER" -rc=$? -set -e - -case "$rc" in - 0) - ok "Prometheus rule-unit tests passed" - summary "- prom-rules: **passed**" - ;; - 3) - skip "promtool unavailable (no promtool on PATH, no podman/docker) — NOT green" - summary "- prom-rules: **SKIPPED** (promtool toolchain unavailable)" - ;; - *) - err "Prometheus rule-unit tests FAILED" - summary "- prom-rules: **FAILED**" - ;; -esac -exit "$rc" diff --git a/ci/versions.env b/ci/versions.env index a739f85..e50d652 100644 --- a/ci/versions.env +++ b/ci/versions.env @@ -21,6 +21,15 @@ YAMLLINT_VERSION="1.35.1" KUBECTL_VERSION="1.36.4" YQ_VERSION="4.44.6" +# Alert-rule unit tests (promtool test rules) — the L0 check +# conformance/tests/l0-static/alert-rules-unit-tested runs every +# platform/**/*.promtool-test.yaml through it. MUST stay in lockstep with +# infra/devcontainer/versions.env (which bakes it into the image) AND with the Prometheus +# version running on prod, because promtool IS the rule engine: verified 2026-09-12 as +# v3.14.0-distroless via `kubectl get prometheus -n monitoring -o jsonpath='{.items[*].spec.version}'`. +# Shipped inside the prometheus release tarball, not as a separate release. +PROMTOOL_VERSION="3.14.0" + # ServiceDefinition JSON-Schema validation (schemas/validate.sh). Pip-installed; # pinned here so CI and a laptop validate against the exact same validator. CHECK_JSONSCHEMA_VERSION="0.29.4" diff --git a/conformance/tests/l0-static/alert-rules-unit-tested/chainsaw-test.yaml b/conformance/tests/l0-static/alert-rules-unit-tested/chainsaw-test.yaml new file mode 100644 index 0000000..c00a374 --- /dev/null +++ b/conformance/tests/l0-static/alert-rules-unit-tested/chainsaw-test.yaml @@ -0,0 +1,95 @@ +# ============================================================================= +# L0 STATIC — the alert rules are UNIT-TESTED, by a pinned engine, and the gaps are declared +# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster. +# +# R-OBS-04 ("the platform alerts on its own health") is implemented by PrometheusRule CRs, and +# this repo has shipped both ways they fail SILENTLY: +# * PERMANENTLY FIRING — the CNPG primaries alert (fixed 2026-09-12), VPNTunnelDown before it; +# * STRUCTURALLY INERT — VeleroNoRecentBackup, whose `time() - ` yields no +# series, so a schedule that had NEVER backed up could not raise it. Prod ran with zero +# volume backups and nothing alerted, masked by a resources-only schedule completing +# cleanly every night (2026-09-12). +# An inert rule and a correct rule are indistinguishable while the platform is healthy. Only +# `promtool test rules` separates them, by constructing the bad state on purpose, offline. +# +# The unit tests were then written — and RUN BY NOTHING. promtool was not in the pinned +# toolchain, no gate stage evaluated the files, and the two harnesses that grew up independently +# each fetched an engine off the network mid-run if promtool was missing (a GitHub release +# download in one, a prometheus container image in the other). Coverage that looks present and +# is enforced nowhere is the failure mode this test ends. +# +# GIVEN the repository as authored, +# WHEN the ONE rule-unit harness (platform/monitoring/tests/run.sh, which flattens the +# COMMITTED PrometheusRule CRs into the rules file promtool wants — never a copy) is run +# over every test file in the repo with the PINNED promtool, +# THEN promtool is pinned in BOTH versions.env files in lockstep at the Prometheus version prod +# actually runs, baked into the image and installed on the CI runner; the promtool in use +# reports exactly that version; no harness acquires an engine at run time; there is +# exactly ONE harness; every rule-unit test file PASSES; the harness is proven able to +# FAIL by a mutated synthetic test on every run; and every alert rule with NO unit test is +# ENUMERATED in a shrink-only register, so the gap is reported rather than hidden. +# +# MISSING TESTS: RATCHET, NOT SKIP, NOT BLANKET-FAIL. 48 of 55 alert rules have no unit test +# today. Failing on all of them would make the gate red until ~48 tests exist (so it would be +# disabled, and a disabled gate protects nothing); merely reporting is precisely how the CNPG +# tests came to be enforced by nothing. So the gap is DECLARED in check.sh, printed with counts +# every run, and CANNOT GROW: a new PrometheusRule, or a new alert in an already-tested file, +# FAILS without a test, and a stale register entry FAILS too. A missing test is never an exit-3 +# skip — "SKIPPED is not green" is preserved for the one thing that is genuinely an environment +# gap: promtool being absent. +# +# STATUS: EXECUTABLE. `check.sh` is an offline assertion script (standard library + the harness +# it delegates to); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this +# Chainsaw Test wraps the SAME script. +# +# NOT proven (named in check.sh's header): that an alert reaches a human (Alertmanager routing +# and SMTP are L3 / R-OBS-02), that a threshold is the right one, that a rule file is actually +# reconciled into a cluster, or that an author's assertions are meaningful — coverage of one +# alert is detected by its name appearing in a covering test file, and only the mutation +# self-test proves the harness CAN fail. R-OBS-04 is tagged for the static slice only: the rules +# the repo ships behave, under the engine prod runs, as their own unit tests say. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: alert-rules-unit-tested + labels: + conformance.home.k3s/R-OBS-04: "true" + conformance.home.k3s/R-TEST-02: "true" + annotations: + conformance.home.k3s/requirements: "R-OBS-04, R-TEST-02" + conformance.home.k3s/layer: "L0" + # WHOLE-REPO SCOPE — this invariant quantifies over every PrometheusRule in the tree + # (apps/wave-0..2, platform/cnpg, platform/kubevirt, platform/monitoring), and the register + # it maintains is a property of the WHOLE repo, not of any one path. + # + # WHY IT MUST NOT BE REQUIREMENT-SCOPED: ci/touched-requirements.sh's coarse path map keys + # apps/* to NOTHING and platform/monitoring/* to R-OBS-01/02 — not R-OBS-04. So adding a new + # untested PrometheusRule under apps/, or editing one under platform/kubevirt/, attributes + # no requirement this test carries, and a requirement-scoped selection would filter the + # check out of exactly the change that adds the untested alert. The same blind spot that let + # `prod-baremetal-not-dev-surface` be filtered out of the run that gated its own breakage + # (DECISIONS-LOG 2026-09-12). + # + # Declaring the scope also makes an honest SKIP loud: lib/run_l1_offline.py turns a rc-3 + # skip of a scope-declaring check into the runner's rc 3, and ci/l0.sh into the gate's rc 3. + # So a gate run on a toolchain WITHOUT promtool reports UNWIRED instead of green — which is + # the whole reason this check exists rather than another hand-written ci/l0.sh stage. + conformance.home.k3s/scope: "whole-repo" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Every committed alert rule that has unit tests passes them under the pinned promtool that + matches prod's Prometheus, exactly one harness runs them, nothing fetches a rule engine at + run time, the harness is proven able to fail, and every alert rule without a unit test is + enumerated in a register that can only shrink. +spec: + steps: + - name: run-every-alert-rule-unit-test-and-report-the-gaps + description: >- + Assert the promtool pin is in lockstep and baked, run the single rule-unit harness over + every test file in the repo, prove it can fail via a mutated synthetic test, and + enumerate every PrometheusRule with no unit tests against the shrink-only register. + try: + - script: + content: bash ./check.sh diff --git a/conformance/tests/l0-static/alert-rules-unit-tested/check.sh b/conformance/tests/l0-static/alert-rules-unit-tested/check.sh new file mode 100755 index 0000000..f0a3fe8 --- /dev/null +++ b/conformance/tests/l0-static/alert-rules-unit-tested/check.sh @@ -0,0 +1,550 @@ +#!/usr/bin/env bash +# ============================================================================= +# L0 static check (offline, no cluster) — alert-rules-unit-tested +# Serves: R-OBS-04 ("Platform alerts on its own health (nodes, storage, GitOps, certs, +# VPN)." — spec docs/10-observability-and-alerting.md) +# R-TEST-02 ("Tests are reproducible: deterministic, hermetic, single-command." +# — spec docs/14-testing-and-conformance.md) +# +# WHY THIS EXISTS — AN UNTESTED ALERT IS NOT A MONITORING CONTROL +# -------------------------------------------------------------- +# This repo has shipped BOTH halves of the alerting failure space to prod: +# * a PERMANENTLY-FIRING `critical` (the CNPG alert fixed 2026-09-12; VPNTunnelDown before +# it), and +# * a STRUCTURALLY INERT rule — VeleroNoRecentBackup, whose `time() - ` +# yields no series, so a schedule that had NEVER backed up could not raise it. Prod ran +# with no volume backups at all and nothing alerted (2026-09-12). +# Neither is visible from the expression, from `kubectl apply`, or from a healthy cluster: an +# inert rule and a correct rule are indistinguishable while everything is fine. `promtool test +# rules` is the only tool that separates them, because the BAD state can be constructed on +# purpose. Unit tests for those rules were then WRITTEN — and evaluated by nothing: promtool +# was not in the pinned toolchain, no gate stage ran the files, and each of the two harnesses +# that existed acquired an engine off the network mid-run if one was missing. Coverage that +# looks present and is never enforced is the failure mode this check ends. +# +# GIVEN the repository as authored, +# WHEN the ONE rule-unit harness (platform/monitoring/tests/run.sh, which flattens the +# COMMITTED PrometheusRule CRs — never a copy) is asked which manifests are under test +# and then run over every test file in the repo, +# THEN (1) promtool is a PINNED member of the toolchain: PROMTOOL_VERSION is declared in BOTH +# infra/devcontainer/versions.env and ci/versions.env and the two AGREE (the repo's +# lockstep rule), the Containerfile takes it as a build ARG, installs it from the +# pinned prometheus release and records it for provenance, dev.sh passes it, and the +# CI runner installs it for the l0 stage; +# (2) the promtool ACTUALLY IN USE reports exactly that pinned version — a unit test is +# evidence only if the engine matches the one prod evaluates rules with; +# (3) NO harness acquires a rule engine at RUN TIME — neither `curl | tar -xz` of a +# GitHub release nor a `podman/docker run` of a prometheus image. Both existed on +# 2026-09-12 and both are removed; this is the regression guard; +# (4) there is exactly ONE harness. Two were written independently within an hour of +# each other; the one nothing runs is the one that rots; +# (5) every rule-unit test file in the repo PASSES, run through that harness (which +# `promtool check rules` first, so a non-parsing rules file cannot make unit tests +# pass vacuously by finding no rules at all); +# (6) the harness is NOT VACUOUS: a synthetic PrometheusRule + companion test is run +# through the SAME harness twice — intact (must PASS) and with one expectation +# mutated (must FAIL) — so a harness that has silently stopped evaluating anything +# is caught on every run, not only when someone thinks to try it; +# AND (7) the alert rules with NO unit tests are ENUMERATED and RATCHETED: each is declared +# below, printed with counts on every run, and the register can only SHRINK. +# +# WHY A MISSING TEST IS A RATCHET AND NOT AN OUTRIGHT FAIL (the decision, and its reasons) +# --------------------------------------------------------------------------------------- +# 48 of the repo's 55 alert rules have no unit test today. Three options were on the table: +# * FAIL on any missing test — correct in principle, but it makes the merge gate red for +# every change until ~48 unit tests exist, so in practice the gate gets disabled, and a +# disabled gate protects nothing. +# * merely REPORT — the gap prints and nothing holds it. That is exactly how the CNPG unit +# tests came to exist and be enforced by nothing; "it is in the output somewhere" is not +# enforcement. +# * RATCHET (chosen) — the gap is DECLARED IN THE REPO as a reviewable register, printed with +# counts on every gate run, and it CANNOT GROW: a new PrometheusRule, or a new alert in a +# file that already has tests, FAILS unless it ships a unit test. Removing an entry requires +# writing the test, and a STALE entry (file gone, or now covered) also FAILS, so the +# register cannot rot into fiction. +# "SKIPPED is not green" is honoured: a missing test is NEVER an exit-3 skip here. This check +# exits 0 only with the gap fully enumerated, and exits 1 the moment the debt grows, the +# register drifts, or a test that exists stops passing. +# The one exit-3 path is promtool itself being ABSENT — an environment gap, not a code property +# (the same convention as ci/l0.sh's kyverno handling), and this test DECLARES +# `conformance.home.k3s/scope: whole-repo`, which makes lib/run_l1_offline.py turn that skip +# into the runner's rc 3 and ci/l0.sh into the gate's rc 3: UNWIRED, never green. promtool +# PRESENT AT THE WRONG VERSION is a FAILURE, not a skip: that case still prints SUCCESS while +# evaluating the rules with an engine prod does not run. +# +# WHAT THIS DOES *NOT* PROVE (honest gaps): +# - It does not prove an alert reaches a human. promtool evaluates rules against synthetic +# series in-process; Alertmanager routing, SMTP delivery and inhibition are L3 +# (tests/l3-fault-injection/alerting) and R-OBS-02, and are not claimed here. +# - It does not prove the THRESHOLDS are right, only that a rule behaves as its own unit +# test says. +# - It does not prove a rule file is RECONCILED into a cluster. An alert file no +# kustomization includes is a separate "looks applied and isn't" failure, not audited here. +# - Coverage of an individual alert is detected by its NAME appearing in a test file that +# covers its manifest. A test file that merely mentions a name without asserting anything +# counts as covered; only the mutation self-test (6) proves the harness CAN fail, not that +# every author wrote a meaningful assertion. Review still owns test quality. +# - It does not read the live cluster, so "the pin matches prod's Prometheus" is asserted +# against the repo's pin, which was established by a live read recorded in +# infra/devcontainer/versions.env (2026-09-12: v3.14.0-distroless). +# ============================================================================= +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root +PYTHON="${PYTHON:-python3}" +HARNESS="${ALERT_RULE_HARNESS:-${ROOT}/platform/monitoring/tests/run.sh}" + +# Honest SKIP (never green) if the thing under test is not present at all. Because this test +# declares scope: whole-repo, the runner turns this into rc 3 for the whole L0 stage — a +# PrometheusRule-less repo is a vanished tree, not a pass. +if ! grep -rqls --include='*.yaml' -e '^[[:space:]]*kind:[[:space:]]*PrometheusRule[[:space:]]*$' \ + "${ROOT}/platform" "${ROOT}/apps" 2>/dev/null; then + echo "check SKIPPED: no PrometheusRule under platform/ or apps/ — nothing to test" >&2 + exit 3 +fi + +# promtool resolution. PROMTOOL= overrides (the same override the harness honours); otherwise +# PATH. Absent -> every OFFLINE assertion still runs and the check exits 3, so a toolchain gap +# is reported as a gap and never as a pass. +PROMTOOL_BIN="${PROMTOOL:-$(command -v promtool || true)}" +if [ -n "${PROMTOOL_BIN}" ] && [ -x "${PROMTOOL_BIN}" ]; then + PROMTOOL_BIN="$(cd "$(dirname "${PROMTOOL_BIN}")" && pwd)/$(basename "${PROMTOOL_BIN}")" +else + PROMTOOL_BIN="" +fi + +cd "${ROOT}" +"${PYTHON}" - "${ROOT}" "${HARNESS}" "${PROMTOOL_BIN}" <<'PY' +"""Assert the alert-rule unit tests are pinned, enforced, non-vacuous, and that the alert +rules with no tests are enumerated and ratcheted. Standard library + the harness itself.""" +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import tempfile +from collections import defaultdict +from pathlib import Path + +ROOT = Path(sys.argv[1]) +HARNESS = Path(sys.argv[2]) +PROMTOOL = sys.argv[3] +sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree + +FAILURES: list[str] = [] +CHECKS = 0 + + +def check(cond, msg): + global CHECKS + CHECKS += 1 + if not cond: + FAILURES.append(msg) + + +# ============================================================================= +# THE DECLARED REGISTER (assertion 7). Shrink-only; see the header for why this is a ratchet. +# +# UNTESTED_FILES — PrometheusRule files no test file covers AT ALL. The twelve apps/* entries +# are single `-down` liveness rules generated from one ServiceDefinition template; +# platform/kubevirt/alerts.yaml (4 rules) is the substantial one and is where the next tests +# belong. A file listed here that later gains coverage, or stops carrying a PrometheusRule, +# FAILS as stale. +# ============================================================================= +UNTESTED_FILES = { + "apps/wave-0/gitea.yaml", + "apps/wave-0/jackett.yaml", + "apps/wave-0/pihole.yaml", + "apps/wave-0/plex.yaml", + "apps/wave-0/qbittorrent.yaml", + "apps/wave-0/vm-demo.yaml", + "apps/wave-1/keycloak.yaml", + "apps/wave-1/nextcloud.yaml", + "apps/wave-1/radarr.yaml", + "apps/wave-1/semaphore.yaml", + "apps/wave-1/sonarr.yaml", + "apps/wave-2/seerr.yaml", + "platform/kubevirt/alerts.yaml", +} + +# UNTESTED_ALERTS — individual alerts inside a file that IS covered by a test file, but whose +# own rule nothing asserts on. This is the half that stops a new alert being slipped into an +# already-"tested" file untested: anything not listed here must appear in a covering test. +# The monitoring block is large because backups.test.yaml (the repo's first suite) deliberately +# covers the Velero/backup rules only; pvc-capacity-alerts.yaml is flattened into the same +# rules file by the suite convention but has no assertions of its own yet. +UNTESTED_ALERTS = { + ("platform/cnpg/alerts.yaml", "CNPGWALArchiveBacklog"), + ("platform/cnpg/alerts.yaml", "CNPGNoRecoverabilityPoint"), + ("platform/cnpg/alerts.yaml", "CNPGNoRecentBaseBackup"), + ("platform/cnpg/alerts.yaml", "CNPGWALVolumeNotRecycling"), + ("platform/cnpg/alerts.yaml", "CNPGMetricsCollectorErroring"), + ("platform/monitoring/alerts.yaml", "NodeExporterDown"), + ("platform/monitoring/alerts.yaml", "NodeMemoryPressureHigh"), + ("platform/monitoring/alerts.yaml", "NodeDiskFillingUp"), + ("platform/monitoring/alerts.yaml", "NodeCPUPressureHigh"), + ("platform/monitoring/alerts.yaml", "NodeCPULoadHigh"), + ("platform/monitoring/alerts.yaml", "NodeIOPressureHigh"), + ("platform/monitoring/alerts.yaml", "NodeIOPressureCritical"), + ("platform/monitoring/alerts.yaml", "LonghornVolumeDegraded"), + ("platform/monitoring/alerts.yaml", "LonghornVolumeFaulted"), + ("platform/monitoring/alerts.yaml", "LonghornVolumeCapacityHigh"), + ("platform/monitoring/alerts.yaml", "LonghornNodeStorageHigh"), + ("platform/monitoring/alerts.yaml", "FluxReconciliationFailure"), + ("platform/monitoring/alerts.yaml", "FluxResourceSuspended"), + ("platform/monitoring/alerts.yaml", "CertificateExpiringSoon"), + ("platform/monitoring/alerts.yaml", "CertificateExpiryImminent"), + ("platform/monitoring/alerts.yaml", "CertificateNotReady"), + ("platform/monitoring/alerts.yaml", "VPNTunnelDown"), + ("platform/monitoring/alerts.yaml", "HubbleDeniedFlowSpike"), + ("platform/monitoring/alerts.yaml", "KeycloakDown"), + ("platform/monitoring/alerts.yaml", "KeycloakDatabaseDown"), + ("platform/monitoring/alerts.yaml", "KeycloakClientSecretReconcilerFailing"), + ("platform/monitoring/alerts.yaml", "KeycloakClientSecretReconcilerStale"), + ("platform/monitoring/alerts.yaml", "MonitoringVolumeFillingUp"), + ("platform/monitoring/alerts.yaml", "PrometheusTargetsMissing"), + ("platform/monitoring/alerts.yaml", "ImageUpdateAutomationFailing"), + ("platform/monitoring/alerts.yaml", "ImagePolicyStalled"), + ("platform/monitoring/alerts.yaml", "UpdateHealthGateFailed"), + ("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeFillingUp"), + ("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeAlmostFull"), + ("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeFillingUpPredicted"), +} + +# --------------------------------------------------------------- (1) the pins, in lockstep +def env_pin(rel: str, key: str) -> str | None: + f = ROOT / rel + if not f.is_file(): + return None + m = re.search(rf'^{re.escape(key)}=["\']?([^"\'\s#]+)', f.read_text(), re.M) + return m.group(1) if m else None + + +DEV_PIN = env_pin("infra/devcontainer/versions.env", "PROMTOOL_VERSION") +CI_PIN = env_pin("ci/versions.env", "PROMTOOL_VERSION") +check(DEV_PIN is not None, + "infra/devcontainer/versions.env declares no PROMTOOL_VERSION — promtool is not part of the " + "baked toolchain, so the alert unit tests are evaluated by whatever the host happens to " + "have, or by nothing at all (which is where this repo was on 2026-09-12)") +check(CI_PIN is not None, + "ci/versions.env declares no PROMTOOL_VERSION — the gate's own version file does not pin the " + "rule engine, so CI and the devcontainer can silently diverge") +check(DEV_PIN is None or DEV_PIN == CI_PIN, + f"PROMTOOL_VERSION is NOT in lockstep: infra/devcontainer/versions.env={DEV_PIN!r} vs " + f"ci/versions.env={CI_PIN!r}. The repo's rule is that the conformance-tool versions in the " + f"two files stay identical, so a laptop and CI run the same engine") + +CF = ROOT / "infra/devcontainer/Containerfile" +cf_text = CF.read_text() if CF.is_file() else "" +check(re.search(r"^ARG PROMTOOL_VERSION\s*$", cf_text, re.M), + "infra/devcontainer/Containerfile has no `ARG PROMTOOL_VERSION` — the pin is declared but " + "the image build does not consume it") +check("prometheus/prometheus/releases/download" in cf_text and "promtool" in cf_text, + "infra/devcontainer/Containerfile does not install promtool from the pinned prometheus " + "release tarball (promtool ships inside it)") +check('echo "promtool=${PROMTOOL_VERSION}"' in cf_text, + "infra/devcontainer/Containerfile does not record promtool in /etc/dev-toolchain-versions — " + "that provenance line is how an operator tells a stale image from a current one") + +DEVSH = ROOT / "infra/devcontainer/dev.sh" +check(DEVSH.is_file() and "--build-arg PROMTOOL_VERSION=" in DEVSH.read_text(), + "infra/devcontainer/dev.sh does not pass --build-arg PROMTOOL_VERSION — the ARG would be " + "empty and the build would fetch a bogus URL") + +INSTALL = ROOT / ".gitea/workflows/scripts/install-tools.sh" +if INSTALL.is_file(): + itext = INSTALL.read_text() + check("PROMTOOL_VERSION" in itext + and re.search(r"^\s*l0\)[^\n]*install_promtool", itext, re.M), + ".gitea/workflows/scripts/install-tools.sh does not install promtool for the l0 stage — " + "on the CI runner this check would exit 3 (UNWIRED) forever and the alert unit tests " + "would be enforced nowhere, which is the exact gap it exists to close") + +# --------------------------------------------- (3)(4) one harness, and it fetches no engine +check(HARNESS.is_file() and os.access(HARNESS, os.X_OK), + f"the rule-unit harness is missing or not executable at {HARNESS.relative_to(ROOT)} — " + f"nothing would run the alert unit tests") + +# Any script that invokes `promtool test rules` is a rule-unit harness. There must be exactly +# one: two harnesses were written independently on 2026-09-12 (a cnpg-only script and this one), +# and the one nothing runs is the one that rots out of sync with the rules. +harnesses: list[str] = [] +fetchers: list[str] = [] +FETCH_RE = re.compile( + r"(curl|wget)[^\n]*prometheus/prometheus/releases" # GitHub release download + r"|(podman|docker)\s+run[^\n]*prometheus", # container-image pull + re.I) +# Two exclusions, both the repo's existing "reference, not declaration" rule (see +# ci/touched-requirements.sh changed_diff() rules 2-3): +# * THIS CHECK'S OWN DIRECTORY. It names every forbidden pattern in order to forbid it, so +# scanning itself would make both guards permanently red — the self-referential trap. +# * THE PINNED TOOLCHAIN INSTALLERS. Downloading the pinned promtool is their JOB, at a +# reviewed seam (an image build, or the runner's tool-install step), from a version in +# versions.env. What is forbidden is a RULE-UNIT RUN acquiring its own engine: then the +# version, the network and the machine are decided silently at test time, and a missing +# toolchain becomes a green run instead of an exit 3. +SELF_DIR = "conformance/tests/l0-static/alert-rules-unit-tested/" +INSTALLERS = (".gitea/workflows/scripts/install-tools.sh", "infra/devcontainer/") +for p in sorted(ROOT.rglob("*.sh")): + rel = str(p.relative_to(ROOT)) + if rel.startswith((".git/", "conformance/runs/", SELF_DIR)): + continue + try: + text = p.read_text() + except (OSError, UnicodeDecodeError): + continue + body = "\n".join(l for l in text.splitlines() if not l.lstrip().startswith("#")) + if "promtool" in body and re.search(r"test\s+rules", body): + harnesses.append(rel) + if FETCH_RE.search(body) and not rel.startswith(INSTALLERS): + fetchers.append(rel) + +check(harnesses == [str(HARNESS.relative_to(ROOT))], + f"expected exactly ONE rule-unit harness ({HARNESS.relative_to(ROOT)}) but found " + f"{harnesses}. A second harness means two behaviours and two things to keep in step; the " + f"one the gate does not run is the one that rots. Fold it in instead") +check(not fetchers, + f"{fetchers} acquire a rule ENGINE at run time (a prometheus release download or a " + f"prometheus container image). promtool must come from the pinned toolchain " + f"(PROMTOOL_VERSION): a run-time fetch decides the version, the network and the machine " + f"silently at test time, makes the tests non-hermetic (R-TEST-02), and quietly turns 'the " + f"toolchain has no promtool' into a green run — which is how it stayed missing") + +# ------------------------------------------------------------------ discover the alert files +# Alert files: tracked YAML carrying a PrometheusRule. conformance/ is excluded deliberately — +# the PrometheusRules under tests/l1-rendering/*/golden/ are EXPECTED-RENDER fixtures for the +# render checks, not rules any cluster evaluates. Everything a cluster actually gets lives under +# apps/, platform/, platform-config/ or clusters/. +def tracked_yaml() -> list[Path]: + out = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z", "*.yaml", "*.yml"], + capture_output=True, text=True) + if out.returncode == 0 and out.stdout: + return [ROOT / p for p in out.stdout.split("\0") if p] + # No usable index (the gate container mounts a worktree whose .git points outside the + # mount). Fall back to a filesystem walk rather than auditing nothing. + return sorted(p for p in ROOT.rglob("*.y*ml") if ".git/" not in str(p)) + + +KIND_RE = re.compile(r"^\s*kind:\s*PrometheusRule\s*$", re.M) +ALERT_RE = re.compile(r"^\s*-\s*alert:\s*(\S+)", re.M) + +alert_files: dict[str, list[str]] = {} +for p in tracked_yaml(): + rel = str(p.relative_to(ROOT)) + if rel.startswith("conformance/") or not p.is_file(): + continue + try: + text = p.read_text() + except (OSError, UnicodeDecodeError): + continue + if KIND_RE.search(text): + alert_files[rel] = ALERT_RE.findall(text) + +check(len(alert_files) > 0, + "no PrometheusRule found outside conformance/ — discovery or the repo layout changed, and " + "this check would silently audit nothing") + +# ------------------- ask the HARNESS which manifests are under test (never re-derive it here) +# Two implementations of "which rules does this test cover" would eventually disagree, and the +# disagreement shows up as a file the gate believes is tested and the harness never touches. +listing = subprocess.run(["bash", str(HARNESS), "--list"], capture_output=True, text=True, + cwd=str(ROOT)) +check(listing.returncode == 0, + f"`{HARNESS.relative_to(ROOT)} --list` failed (rc={listing.returncode}): " + f"{(listing.stderr or listing.stdout).strip()[-600:]}") +covering: dict[str, list[str]] = defaultdict(list) # manifest -> [test files] +for line in listing.stdout.splitlines(): + if "\t" in line: + tfile, manifest = line.split("\t", 1) + covering[manifest.strip()].append(tfile.strip()) + +for manifest in sorted(covering): + check(manifest in alert_files, + f"the harness reports {manifest} as under test, but it is not one of the " + f"PrometheusRule files found in the repo — the mapping and the tree disagree") + +# --------------------------------------------------------- (2) the engine is the pinned one +version_line = "" +if PROMTOOL: + vp = subprocess.run([PROMTOOL, "--version"], capture_output=True, text=True) + out = (vp.stdout + vp.stderr).strip() + version_line = out.splitlines()[0] if out else "" + m = re.search(r"version\s+(\d+\.\d+\.\d+)", version_line) + found = m.group(1) if m else None + check(found is not None, + f"could not parse a version from `promtool --version`: {version_line!r}") + check(found is None or DEV_PIN is None or found == DEV_PIN, + f"promtool reports {found}, but the toolchain pins PROMTOOL_VERSION={DEV_PIN} (the " + f"Prometheus version prod runs). promtool IS the rule engine: a unit test green under " + f"one version and red under another is worse than no test. Run the gate in the pinned " + f"devcontainer (rebuild it: ./infra/devcontainer/dev.sh build), or set PROMTOOL= to a " + f"v{DEV_PIN} binary. A FAILURE, not a skip, because the wrong engine still prints " + f"SUCCESS") + +# ------------------------------------------------- (5) every rule-unit test file must pass +if PROMTOOL: + env = dict(os.environ, PROMTOOL=PROMTOOL) + run = subprocess.run(["bash", str(HARNESS)], capture_output=True, text=True, + cwd=str(ROOT), env=env) + tailed = "\n ".join((run.stdout + run.stderr).strip().splitlines()[-40:]) + check(run.returncode == 0, + f"`{HARNESS.relative_to(ROOT)}` FAILED (rc={run.returncode}) — the committed alert " + f"rules do not behave as their own unit tests say:\n {tailed}") + + # ------------------------------------------- (6) the harness is proven non-vacuous + # A synthetic PrometheusRule + companion test through the SAME harness, outside the repo so + # discovery never picks it up. Intact must PASS; a mutated expectation must FAIL. Without + # this, a harness that stopped evaluating anything (an empty extraction, a rule file written + # under the wrong name) would report green forever — which is the disease being treated. + work = tempfile.mkdtemp(prefix="alert-rules-selftest.") + try: + (Path(work) / "synth.yaml").write_text("""apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: selftest +spec: + groups: + - name: selftest + rules: + - alert: SelfTestCanary + expr: selftest_metric > 0 + for: 5m + annotations: + runbook_url: https://${BASE_DOMAIN}/runbooks/selftest +""") + test_yaml = """rule_files: + - generated.rules.yaml +evaluation_interval: 1m +tests: + - interval: 1m + name: canary fires only after `for` elapses + input_series: + - series: 'selftest_metric' + values: '1x10' + promql_expr_test: + - expr: 'ALERTS{alertname="SelfTestCanary",alertstate="firing"}' + eval_time: EVAL_TIME + exp_samples: + - labels: 'ALERTS{alertname="SelfTestCanary",alertstate="firing"}' + value: 1 +""" + synth_test = Path(work) / "synth.promtool-test.yaml" + synth_test.write_text(test_yaml.replace("EVAL_TIME", "6m")) + good = subprocess.run(["bash", str(HARNESS), str(synth_test)], + capture_output=True, text=True, cwd=str(ROOT), env=env) + check(good.returncode == 0, + "self-test: the INTACT synthetic unit test did not pass, so the harness itself is " + "broken and every green result above is meaningless:\n " + + "\n ".join((good.stdout + good.stderr).strip().splitlines()[-20:])) + # Mutation: assert firing at 2m, which `for: 5m` forbids. + synth_test.write_text(test_yaml.replace("EVAL_TIME", "2m")) + bad = subprocess.run(["bash", str(HARNESS), str(synth_test)], + capture_output=True, text=True, cwd=str(ROOT), env=env) + check(bad.returncode != 0, + "self-test: a MUTATED expectation (firing before `for:` elapsed) still PASSED — " + "the harness is not actually evaluating anything, so every green result above is " + "vacuous") + finally: + shutil.rmtree(work, ignore_errors=True) + +# --------------------------------------------------------- (7) the gap inventory + ratchet +INVENTORY: list[str] = [] +GAP_FILES: list[str] = [] +alerts_total = covered_total = 0 + +for rel in sorted(alert_files): + names = alert_files[rel] + alerts_total += len(names) + check(len(names) > 0, + f"{rel} declares a PrometheusRule but no `alert:` rule was parsed out of it — the " + f"parser or the file shape changed, and any unit test over it would pass vacuously") + tests = covering.get(rel, []) + if not tests: + GAP_FILES.append(rel) + INVENTORY.append(f" {rel:<46} alerts={len(names):<3} tested=0/{len(names):<3} NO TEST FILE") + check(rel in UNTESTED_FILES, + f"{rel} carries {len(names)} alert rule(s) and NO test file covers it. A new " + f"PrometheusRule must ship promtool unit tests: this repo has shipped both a " + f"permanently-firing critical and a structurally inert alert, and neither was " + f"visible without them. Add `{rel[:-5]}.promtool-test.yaml` (companion convention) " + f"or a /tests/*.test.yaml suite. If this is deliberate debt it must be " + f"DECLARED in UNTESTED_FILES in this check — the register is what keeps the gap " + f"visible instead of silent") + continue + + check(rel not in UNTESTED_FILES, + f"{rel} is listed in UNTESTED_FILES but is now covered by {tests} — the register is " + f"shrink-only: delete the line so the next missing test is caught") + text = "".join((ROOT / t).read_text() for t in tests if (ROOT / t).is_file()) + covered = [n for n in names if re.search(rf"\b{re.escape(n)}\b", text)] + covered_total += len(covered) + for n in names: + if n in covered: + check((rel, n) not in UNTESTED_ALERTS, + f"{rel}: alert {n} is listed in UNTESTED_ALERTS but IS asserted on now — " + f"remove the stale entry (the register is shrink-only)") + else: + check((rel, n) in UNTESTED_ALERTS, + f"{rel}: alert {n} has NO unit test in {tests}. This file is already covered, " + f"so a new alert in it must be tested too — otherwise a 'tested' alert file " + f"quietly grows untested rules") + INVENTORY.append(f" {rel:<46} alerts={len(names):<3} tested={len(covered)}/{len(names):<3} " + f"{','.join(tests)}") + +# stale register entries +for rel in sorted(UNTESTED_FILES - set(alert_files)): + check(False, f"UNTESTED_FILES lists {rel}, which carries no PrometheusRule in this repo any " + f"more — a stale exemption hides the next real gap; delete the line") +for rel in sorted({f for f, _ in UNTESTED_ALERTS} - set(alert_files)): + check(False, f"UNTESTED_ALERTS lists alerts in {rel}, which carries no PrometheusRule in this " + f"repo any more — delete those lines") + +# --------------------------------------------------------------------------------- report +print("Alert-rule unit-test inventory (PrometheusRule -> covering test file(s)):") +for line in INVENTORY: + print(line) +print(f"\nharness: {HARNESS.relative_to(ROOT)} promtool: {version_line or 'NOT FOUND'}") +print(f"pinned: PROMTOOL_VERSION={DEV_PIN} (devcontainer) / {CI_PIN} (ci) — prod runs Prometheus " + f"v{DEV_PIN}") +print(f"alert files: {len(alert_files)} covered by a test file: " + f"{len(alert_files) - len(GAP_FILES)} with NO test file: {len(GAP_FILES)}") +print(f"alert rules: {alerts_total} unit-tested: {covered_total} " + f"NOT unit-tested: {alerts_total - covered_total}") + +if GAP_FILES: + print("\nDECLARED GAPS — alert files nothing tests at all (shrink-only register; each is a") + print("rule nothing has ever evaluated, NOT a rule known to work):") + for g in GAP_FILES: + print(f" - {g} ({len(alert_files[g])} alert(s))") +if UNTESTED_ALERTS: + print("\nDECLARED GAPS — individual alerts inside a covered file that nothing asserts on:") + for f, n in sorted(UNTESTED_ALERTS): + print(f" - {f} {n}") + +print() +if not PROMTOOL and not FAILURES: + print("check SKIPPED (exit 3, NOT a pass): promtool is not on PATH, so NO alert rule was " + "parsed and NO unit test was evaluated.", file=sys.stderr) + print(f" promtool is pinned at v{DEV_PIN} in infra/devcontainer/versions.env + " + f"ci/versions.env.", file=sys.stderr) + print(" Run the gate in the pinned toolbox (./infra/devcontainer/dev.sh build, then " + "make -C ci gate), or set PROMTOOL=/path/to/promtool.", file=sys.stderr) + print(f" The {CHECKS} offline assertions above (pins, lockstep, one harness, no run-time " + f"engine fetch, the gap register) DID run and passed.", file=sys.stderr) + print(" This test declares scope: whole-repo, so the runner reports rc 3 — UNWIRED, never " + "green.", file=sys.stderr) + raise SystemExit(3) + +if not FAILURES: + print(f"check OK: alert-rules-unit-tested — {CHECKS} assertions " + f"({covered_total}/{alerts_total} alert rules unit-tested, " + f"{len(GAP_FILES)} files with no test file, all declared) (R-OBS-04, R-TEST-02)") + raise SystemExit(0) +print(f"CHECK FAIL (alert-rules-unit-tested): {len(FAILURES)} failed, " + f"{CHECKS - len(FAILURES)} passed", file=sys.stderr) +for f in FAILURES: + print(f" FAIL {f}", file=sys.stderr) +raise SystemExit(1) +PY diff --git a/infra/devcontainer/Containerfile b/infra/devcontainer/Containerfile index c697215..e082b1a 100644 --- a/infra/devcontainer/Containerfile +++ b/infra/devcontainer/Containerfile @@ -1,7 +1,7 @@ # Reproducible dev toolbox for the home-network-k3s platform (Phase 1+). # Bakes the NON-Python prereqs (OpenTofu, age, sops, flux) plus ansible + kube tooling # AND the pinned conformance-harness toolchain (kyverno CLI, kyverno-json, chainsaw, -# yamllint, check-jsonschema, yq) into a single pinned baseline image, so `make -C ci gate` +# yamllint, check-jsonschema, yq, promtool) into a single pinned baseline image, so `make -C ci gate` # runs identically here and in CI. This is the Proxmox (bpg/proxmox) toolbox + conformance # harness: OpenTofu drives the Proxmox-hosted k3s cluster via the bpg provider (dev.sh opens # an SSH tunnel to the kube-API), NOT a local libvirt/KVM host. The libvirt-client/qemu-img @@ -23,6 +23,10 @@ ARG CHAINSAW_VERSION ARG YAMLLINT_VERSION ARG YQ_VERSION ARG CHECK_JSONSCHEMA_VERSION +# promtool: the engine that evaluates the alert-rule UNIT TESTS (platform/**/*.promtool-test.yaml). +# Pinned to the Prometheus version prod actually runs — see versions.env for why that lockstep is +# the point, and for the live-cluster command that establishes it. +ARG PROMTOOL_VERSION ARG TARGETARCH=amd64 # Distro packages: libvirt client libs (harmless disk-image utilities), ansible, ssh, @@ -59,7 +63,11 @@ RUN set -eux; \ KYVERNO_CLI_ARCH="$([ "${TARGETARCH}" = amd64 ] && echo x86_64 || echo "${TARGETARCH}")"; \ curl -fsSL "https://github.com/kyverno/kyverno/releases/download/v${KYVERNO_VERSION}/kyverno-cli_v${KYVERNO_VERSION}_linux_${KYVERNO_CLI_ARCH}.tar.gz" | tar -xz -C /usr/local/bin kyverno; \ curl -fsSL "https://github.com/kyverno/kyverno-json/releases/download/v${KYVERNO_JSON_VERSION}/kyverno-json_linux_${TARGETARCH}.tar.gz" | tar -xz -C /usr/local/bin kyverno-json; \ - chmod +x /usr/local/bin/yq /usr/local/bin/chainsaw /usr/local/bin/kyverno /usr/local/bin/kyverno-json; \ + PROM_TARBALL="prometheus-${PROMTOOL_VERSION}.linux-${TARGETARCH}"; \ + curl -fsSL "https://github.com/prometheus/prometheus/releases/download/v${PROMTOOL_VERSION}/${PROM_TARBALL}.tar.gz" \ + | tar -xz -C /usr/local/bin --strip-components=1 "${PROM_TARBALL}/promtool"; \ + chmod +x /usr/local/bin/yq /usr/local/bin/chainsaw /usr/local/bin/kyverno /usr/local/bin/kyverno-json \ + /usr/local/bin/promtool; \ rm -rf /tmp/* # Record the baked toolchain for provenance. @@ -68,6 +76,7 @@ RUN { echo "opentofu=${OPENTOFU_VERSION}"; echo "age=${AGE_VERSION}"; echo "sops echo "kyverno=${KYVERNO_VERSION}"; echo "kyverno-json=${KYVERNO_JSON_VERSION}"; \ echo "chainsaw=${CHAINSAW_VERSION}"; echo "yamllint=${YAMLLINT_VERSION}"; \ echo "yq=${YQ_VERSION}"; echo "check-jsonschema=${CHECK_JSONSCHEMA_VERSION}"; \ + echo "promtool=${PROMTOOL_VERSION}"; \ } > /etc/dev-toolchain-versions WORKDIR /work diff --git a/infra/devcontainer/dev.sh b/infra/devcontainer/dev.sh index 88f4d7e..d86561c 100755 --- a/infra/devcontainer/dev.sh +++ b/infra/devcontainer/dev.sh @@ -86,6 +86,7 @@ build() { --build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \ --build-arg YQ_VERSION="$YQ_VERSION" \ --build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \ + --build-arg PROMTOOL_VERSION="$PROMTOOL_VERSION" \ -t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}" } diff --git a/infra/devcontainer/versions.env b/infra/devcontainer/versions.env index 9540fc6..2c8acea 100644 --- a/infra/devcontainer/versions.env +++ b/infra/devcontainer/versions.env @@ -2,7 +2,7 @@ # The non-Python prereqs (OpenTofu, age, sops, flux) are Go binaries, so — per the # operator's rule — they are baked into a container image instead of a venv. The # conformance test toolchain (kyverno CLI, kyverno-json, chainsaw, yamllint, -# check-jsonschema, yq) is likewise baked in so `make -C ci gate` runs here == in CI. +# check-jsonschema, yq, promtool) is likewise baked in so `make -C ci gate` runs here == in CI. # # NOTE: these are best-effort pins; bump to current stable if a download 404s at build. # The conformance-tool versions below MUST stay in lockstep with ci/versions.env. @@ -28,4 +28,20 @@ CHAINSAW_VERSION=0.2.12 YAMLLINT_VERSION=1.35.1 YQ_VERSION=4.44.6 CHECK_JSONSCHEMA_VERSION=0.29.4 +# promtool — the Prometheus rule linter + UNIT-TEST runner (`promtool test rules`), which is +# what evaluates platform/**/*.promtool-test.yaml in the L0 gate +# (conformance/tests/l0-static/alert-rules-unit-tested). +# +# THIS PIN IS NOT FREE-FLOATING: it MUST equal the Prometheus version actually running on prod, +# because promtool is the rule ENGINE and a different major/minor can parse or evaluate PromQL +# differently — a unit test green under one engine and red under the other is worse than no test. +# Verified against the live cluster on 2026-09-12: +# kubectl --context k3s-prod get prometheus -n monitoring -o jsonpath='{.items[*].spec.version}' +# -> v3.14.0-distroless (image quay.io/prometheus/prometheus:v3.14.0-distroless) +# When kube-prometheus-stack ships a new Prometheus to prod, bump this (and ci/versions.env) with +# it; the L0 check asserts the promtool on PATH reports exactly this version, so a drifted +# toolchain FAILS the gate instead of quietly testing with the wrong engine. +# promtool ships INSIDE the prometheus release tarball (prometheus-.linux-.tar.gz), so +# the version string here is a Prometheus release, not a separate promtool release. +PROMTOOL_VERSION=3.14.0 # Proxmox provider (bpg/proxmox) is pinned in the tofu module (infra/tofu/versions.tf), not here. diff --git a/platform/cnpg/alerts.promtool-test.yaml b/platform/cnpg/alerts.promtool-test.yaml index 2d18971..087d850 100644 --- a/platform/cnpg/alerts.promtool-test.yaml +++ b/platform/cnpg/alerts.promtool-test.yaml @@ -9,11 +9,25 @@ # not evidence that it can fire, and it is certainly not evidence that it can stop. These tests pin # BOTH directions plus the two ways the rule is meant to stay quiet. # -# HOW TO RUN. `./run-alert-tests.sh` in this directory — it extracts `.spec` from the PrometheusRule -# (promtool wants a bare Prometheus rules file, not a Kubernetes CR), substitutes ${BASE_DOMAIN}, -# and runs promtool against both files in a temp dir. promtool is NOT in the baked devcontainer -# toolchain (infra/devcontainer/versions.env); the script downloads the version that matches the -# Prometheus actually running on prod (v3.14.0) if it is not already on PATH. +# HOW TO RUN. `platform/monitoring/tests/run.sh` — the ONE repo-wide rule-unit harness. It finds +# this file by the companion convention (`.promtool-test.yaml` sits beside +# `.yaml`), flattens the PrometheusRule CR's `.spec.groups` into the bare Prometheus +# rules file promtool wants (a CR is not a rules file), writes it under the name `rule_files:` +# below declares, `promtool check rules` it first so a non-parsing file cannot make these tests +# pass vacuously, and runs them — all in a temp dir, because a generated file sitting next to its +# source is how a stale copy becomes the thing under test. Run just this file with +# `platform/monitoring/tests/run.sh platform/cnpg/alerts.promtool-test.yaml`. +# (There used to be a cnpg-only `run-alert-tests.sh` here; it was folded into that harness on +# 2026-09-12 — two harnesses means two behaviours, and the one nothing runs rots.) +# +# promtool IS in the baked devcontainer toolchain (PROMTOOL_VERSION in +# infra/devcontainer/versions.env, mirrored in ci/versions.env), pinned to the Prometheus version +# prod actually runs (v3.14.0) because promtool is the rule ENGINE. A missing promtool is exit 3 +# (SKIPPED, never green), not a silent download. +# +# WHO RUNS IT IN CI. conformance/tests/l0-static/alert-rules-unit-tested — the L0 static check +# that runs the harness over every *.promtool-test.yaml in the repo and reports which alert files +# still have none. Before that check existed these tests were never evaluated by anything. # # WHY THE ASSERTIONS USE `ALERTS` AND NOT `exp_alerts`. promtool's alert_rule_test compares # annotations VERBATIM, which would mean pasting this rule's ~1200-character runbook description in diff --git a/platform/cnpg/alerts.yaml b/platform/cnpg/alerts.yaml index bdbb13a..74df07c 100644 --- a/platform/cnpg/alerts.yaml +++ b/platform/cnpg/alerts.yaml @@ -349,7 +349,9 @@ spec: # on k3s-prod-1, keycloak-db-1 on k3s-prod-2, nextcloud-db-1 on k3s-prod-3. Quiet because # the condition is false, NOT because the query went blind. # * UNIT-TESTED, AND THE TESTS CAN FAIL. alerts.promtool-test.yaml (run it with - # ./run-alert-tests.sh) covers four cases under promtool 3.14.0, the exact Prometheus + # platform/monitoring/tests/run.sh — the one repo-wide harness; the merge gate runs the + # same path via conformance/tests/l0-static/alert-rules-unit-tested) covers four cases + # under promtool 3.14.0, the exact Prometheus # version running here: co-located fires after 30m, one-per-node is silent, a NotReady # node suppresses it and then re-arms when the node returns, and a non-CNPG `*-rw` # Service is ignored. Mutation-checked — deleting the cnpg_collector_up join, the diff --git a/platform/cnpg/run-alert-tests.sh b/platform/cnpg/run-alert-tests.sh deleted file mode 100755 index c190c5d..0000000 --- a/platform/cnpg/run-alert-tests.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -############################################################################### -# platform/cnpg/run-alert-tests.sh — run the promtool unit tests for alerts.yaml. -# -# WHY A SCRIPT AND NOT JUST `promtool test rules`. Two impedance mismatches: -# 1. alerts.yaml is a Kubernetes PrometheusRule CR; promtool wants a bare Prometheus -# rules file (`groups:` at the top level). So `.spec` is extracted first. -# 2. runbook_url carries `${BASE_DOMAIN}`, a Flux postBuild substitution that is not -# resolved in git. promtool does not care what the string says, but leaving a raw -# `${...}` in a rendered rules file is exactly the kind of thing that gets copied -# somewhere it DOES matter, so it is substituted with a placeholder here. -# Both derived files go to a temp dir — nothing generated is left in the worktree, because -# a generated file sitting next to its source is how a stale copy becomes the thing under -# test. -# -# promtool is NOT in the baked devcontainer toolchain (infra/devcontainer/versions.env). -# If it is not on PATH this fetches the version that matches the Prometheus actually -# running on prod, so the unit tests are evaluated by the same engine that will evaluate -# the rules for real. Override with PROMTOOL=/path/to/promtool. -# -# ./run-alert-tests.sh -############################################################################### -set -euo pipefail - -# Match the Prometheus running on prod: quay.io/prometheus/prometheus:v3.14.0-distroless -# (`kubectl get prometheus -n monitoring -o jsonpath='{.items[*].spec.version}'`). -PROMETHEUS_VERSION="${PROMETHEUS_VERSION:-3.14.0}" - -here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT - -promtool="${PROMTOOL:-$(command -v promtool || true)}" -if [[ -z "$promtool" ]]; then - echo "==> promtool not on PATH; fetching v${PROMETHEUS_VERSION} (matches prod Prometheus)" - tarball="prometheus-${PROMETHEUS_VERSION}.linux-amd64" - curl -fsSL "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${tarball}.tar.gz" \ - | tar -xz -C "$tmp" --strip-components=1 "${tarball}/promtool" - promtool="$tmp/promtool" -fi -# Absolutised because step 3 runs promtool from inside $tmp — a relative PROMTOOL= would -# break there, and it did during development. -promtool="$(cd "$(dirname "$promtool")" && pwd)/$(basename "$promtool")" -echo "==> $("$promtool" --version 2>&1 | head -1)" - -# 1. PrometheusRule CR -> bare Prometheus rules file, with ${BASE_DOMAIN} resolved. -sed 's/\${BASE_DOMAIN}/base-domain.invalid/g' "$here/alerts.yaml" \ - | python3 -c 'import sys,yaml; yaml.safe_dump(yaml.safe_load(sys.stdin)["spec"], sys.stdout, sort_keys=False, width=10**6)' \ - > "$tmp/alerts.rules.yaml" -cp "$here/alerts.promtool-test.yaml" "$tmp/" - -# 2. Syntax gate first — a rules file that does not parse would make every unit test -# "pass" vacuously by finding no rules at all. -echo "==> promtool check rules" -"$promtool" check rules "$tmp/alerts.rules.yaml" - -# 3. The unit tests. -echo "==> promtool test rules" -cd "$tmp" -"$promtool" test rules alerts.promtool-test.yaml diff --git a/platform/monitoring/README.md b/platform/monitoring/README.md index 7c98a73..63569f0 100644 --- a/platform/monitoring/README.md +++ b/platform/monitoring/README.md @@ -18,7 +18,7 @@ | `loki.yaml` | `HelmRelease` — Loki single-binary, filesystem, compactor-enforced retention | | `alloy.yaml` | `HelmRelease` — Grafana Alloy DaemonSet, tails pod logs → Loki | | `alerts.yaml` | Platform-health `PrometheusRule`s (R-OBS-04) | -| `tests/` | `promtool test rules` unit tests over the committed `PrometheusRule`s — each rule pinned quiet on the good state and firing on the bad one (`tests/run.sh`, or `make -C ci prom-rules`) | +| `tests/` | `promtool test rules` unit tests over the committed `PrometheusRule`s — each rule pinned quiet on the good state and firing on the bad one (`tests/run.sh` — the repo-wide harness, it also runs other components' companion `*.promtool-test.yaml`; enforced by the L0 check `alert-rules-unit-tested`) | | `kustomization.yaml` | builds all of the above | ## Requirement mapping diff --git a/platform/monitoring/tests/backups.test.yaml b/platform/monitoring/tests/backups.test.yaml index 7fbf8f2..7a40624 100644 --- a/platform/monitoring/tests/backups.test.yaml +++ b/platform/monitoring/tests/backups.test.yaml @@ -1,5 +1,6 @@ # promtool rule-unit tests for the platform-health.backups group (platform/monitoring/alerts.yaml). -# Run with: platform/monitoring/tests/run.sh (or `make -C ci prom-rules`) +# Run with: platform/monitoring/tests/run.sh (the merge gate runs the same harness via +# conformance/tests/l0-static/alert-rules-unit-tested) # # Every rule is pinned in BOTH directions, because this repo has shipped both failure modes to # prod: a permanently-firing alert (CNPG, 2026-09-12) and a structurally inert one diff --git a/platform/monitoring/tests/extract_rules.py b/platform/monitoring/tests/extract_rules.py index c527a18..9c596c4 100755 --- a/platform/monitoring/tests/extract_rules.py +++ b/platform/monitoring/tests/extract_rules.py @@ -12,10 +12,19 @@ would let the fixture stay green while the deployed rule rots -- which is the cl this harness exists to catch (a permanently-firing CNPG alert and a structurally inert Velero alert both reached prod in 2026-09). -Flux `postBuild` placeholders (`${BASE_DOMAIN}`) are left verbatim: no expression in -platform/monitoring uses one, and leaving them alone keeps annotation text identical to -the file under test. If a placeholder ever appears inside an `expr`, that is refused -loudly rather than silently producing an untestable rule file. +Flux `postBuild` placeholders (`${BASE_DOMAIN}`) are left verbatim: no expression in any +tested manifest uses one, and leaving them alone keeps annotation text identical to the +file under test. If a placeholder ever appears inside an `expr`, that is refused loudly +rather than silently producing an untestable rule file. + +The cnpg-only script this replaced (platform/cnpg/run-alert-tests.sh, folded in 2026-09-12) +SUBSTITUTED those placeholders instead, on the reasoning that "leaving a raw ${...} in a +rendered rules file is exactly the kind of thing that gets copied somewhere it DOES +matter". That hazard is real but it is addressed by WHERE the file goes, not by rewriting +it: run.sh writes every generated rule file into a temp dir it deletes, so no rendered copy +ever exists next to its source to be picked up. Verbatim is then the better default — +one transformation fewer between the committed rule and the rule under test, and an +`exp_alerts` assertion on annotation text still matches the file as shipped. Group names are made unique per source file so two files may legally reuse a name. diff --git a/platform/monitoring/tests/run.sh b/platform/monitoring/tests/run.sh index a024b9d..56cf6b3 100755 --- a/platform/monitoring/tests/run.sh +++ b/platform/monitoring/tests/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# platform/monitoring/tests/run.sh — Prometheus RULE-UNIT-TEST harness (no cluster). +# platform/monitoring/tests/run.sh — the repo's Prometheus RULE-UNIT-TEST harness (no cluster). # # WHY THIS EXISTS # This repo has now shipped both halves of the alerting failure space to prod: @@ -17,76 +17,228 @@ # PrometheusRule CRs into a rule file) — never a copy, so a fixture cannot drift green while # the deployed rule rots. # -# Usage: ./run.sh # run every *.test.yaml here -# ./run.sh foo.test.yaml +# ONE HARNESS, REPO-WIDE (consolidated 2026-09-12). This started as the monitoring-only +# runner and there was a second, cnpg-only script (`platform/cnpg/run-alert-tests.sh`) doing +# the same three steps by hand. Two harnesses means two behaviours, and the one nothing ran +# rots — so the cnpg script is gone and this one covers EVERY PrometheusRule in the repo that +# has unit tests. It still lives under platform/monitoring/ because observability owns alert +# testing; the tests themselves live next to the rules they test. +# +# TWO TEST-FILE CONVENTIONS, both discovered automatically: +# 1. COMPANION — `.promtool-test.yaml` beside `.yaml`. The manifest under +# test is that sibling. This is how a platform component ships tests with its own alerts +# (platform/cnpg/alerts.promtool-test.yaml). +# 2. SUITE — `/tests/*.test.yaml`. The manifests under test are every PrometheusRule +# file in `` (platform/monitoring/tests/backups.test.yaml covers alerts.yaml + +# pvc-capacity-alerts.yaml, so one suite can assert across rule groups). +# +# WHAT THE ADAPTER DOES, AND WHY (inherited from the cnpg script's header, which was right): +# * A PrometheusRule is a Kubernetes CR; promtool wants a bare rules file with `groups:` at +# the top level. So `.spec.groups` is extracted (extract_rules.py), never re-typed. +# * The extracted file is written under whatever name the test file's own `rule_files:` +# declares, so a test file needs no knowledge of this harness. +# * `promtool check rules` runs FIRST. A rules file that does not parse would make every +# unit test "pass" vacuously by finding no rules at all. +# * Everything generated goes to a temp dir. A generated file sitting next to its source is +# how a stale copy becomes the thing under test — and it is also why leaving Flux's +# `${BASE_DOMAIN}` placeholders verbatim (see extract_rules.py) is safe here. +# +# Usage: ./run.sh # every test file in the repo (both conventions) +# ./run.sh platform/cnpg/alerts.promtool-test.yaml # one file +# ./run.sh --list # print `\t` pairs, run nothing +# +# `--list` exists so the L0 check can ask THIS script which manifests are under test instead of +# re-deriving the conventions itself. Two implementations of "which rules does this test cover" +# would eventually disagree, and the disagreement would show up as a file the gate believes is +# tested and the harness never touches. It needs no promtool, so it works on any host. # # Exit codes follow ci/lib.sh: 0 pass, 1 fail, 3 SKIPPED (toolchain unavailable — never green). -# promtool is taken from $PATH if present, otherwise run from the pinned Prometheus image via -# podman/docker. The image tag matches the version prod runs, so "passes here" means "passes -# with the engine that will evaluate it". +# +# PROMTOOL COMES FROM THE PINNED TOOLCHAIN, AND NOWHERE ELSE. `PROMTOOL_VERSION` in +# infra/devcontainer/versions.env (baked into the image) and ci/versions.env (installed on the +# CI runner), pinned to the Prometheus version prod actually runs — promtool IS the rule +# engine, so a test green under one version and red under another is worse than no test. +# The two run-time acquisition paths this harness and the cnpg script used to have — a +# `curl | tar -xz` of a GitHub release, and a `podman run` of quay.io/prometheus/prometheus — +# are both REMOVED. Not because a pinned download is unusual here (the Containerfile downloads +# every tool it bakes) but because those decided the version, the network and the machine +# SILENTLY, at test time, on whatever host ran the tests instead of once at reviewed +# image-build time; and each one quietly turned "the toolchain has no promtool" into a green +# run, which is how the toolchain stayed without one. Missing promtool is now exit 3, never a +# pass. Override with PROMTOOL=/path/to/promtool. +# +# ENFORCEMENT. This script is the developer entrypoint. The merge gate runs it through +# conformance/tests/l0-static/alert-rules-unit-tested, which additionally reports every +# PrometheusRule in the repo that has NO unit tests at all. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MONITORING_DIR="$(cd "${HERE}/.." && pwd)" -PROM_IMAGE="${PROM_IMAGE:-quay.io/prometheus/prometheus:v3.14.0}" +ROOT="$(cd "${HERE}/../../.." && pwd)" -MANIFESTS=( - "${MONITORING_DIR}/alerts.yaml" - "${MONITORING_DIR}/pvc-capacity-alerts.yaml" -) - -if [ "$#" -gt 0 ]; then - TESTS=("$@") -else - mapfile -t TESTS < <(find "${HERE}" -maxdepth 1 -name '*.test.yaml' | sort) -fi -if [ "${#TESTS[@]}" -eq 0 ]; then - echo "run.sh: no *.test.yaml found in ${HERE}" >&2 - exit 3 -fi +LIST_ONLY=0 +if [ "${1:-}" = "--list" ]; then LIST_ONLY=1; shift; fi # ---------------------------------------------------------------- toolchain resolution -RUNNER="" -if command -v promtool >/dev/null 2>&1; then - RUNNER="native" -elif command -v podman >/dev/null 2>&1; then - RUNNER="podman" -elif command -v docker >/dev/null 2>&1; then - RUNNER="docker" +# Skipped for --list: that mode runs no rules, and the L0 check must be able to resolve the +# test->manifest mapping (and report the gaps) even on a host with no promtool. +PROMTOOL_BIN="" +if [ "$LIST_ONLY" = 0 ]; then +PROMTOOL_BIN="${PROMTOOL:-$(command -v promtool || true)}" +if [ -z "${PROMTOOL_BIN}" ]; then + pinned="$(sed -n 's/^PROMTOOL_VERSION="\{0,1\}\([^"]*\)"\{0,1\}.*/\1/p' \ + "${ROOT}/ci/versions.env" 2>/dev/null | head -1)" + { + echo "run.sh: SKIP (exit 3, NOT a pass) — promtool is not on PATH, and this harness no" + echo " longer fetches one (no GitHub release download, no container-image pull)." + echo " promtool is pinned at v${pinned:-} — the Prometheus version prod runs:" + echo " * infra/devcontainer/versions.env PROMTOOL_VERSION (baked into the image)" + echo " * ci/versions.env PROMTOOL_VERSION (installed on the CI runner)" + echo " Run it in the pinned toolbox:" + echo " ./infra/devcontainer/dev.sh build # if the image predates the promtool pin" + echo " ./infra/devcontainer/dev.sh run bash -c 'platform/monitoring/tests/run.sh'" + echo " Or point at a local binary of the same version: PROMTOOL=/path/to/promtool ./run.sh" + } >&2 + exit 3 +fi +# A PROMTOOL= that does not resolve is an OPERATOR MISTAKE, not a missing toolchain: exit 2, +# so a typo'd override cannot masquerade as the honest "no promtool here" skip above. +if [ ! -x "${PROMTOOL_BIN}" ]; then + echo "run.sh: PROMTOOL='${PROMTOOL_BIN}' is not an executable file — fix the override" >&2 + exit 2 +fi +# Absolutised: each test runs with cwd inside a temp dir, where a relative PROMTOOL= breaks +# (it did, during development). +PROMTOOL_BIN="$(cd "$(dirname "${PROMTOOL_BIN}")" && pwd)/$(basename "${PROMTOOL_BIN}")" +fi + +# ---------------------------------------------------------------- which test files to run +# NOT `git ls-files`: the gate runs inside a container that mounts only the worktree, whose +# `.git` file points outside the mount, so git cannot read the index there — and with +# `pipefail` that failure silently emptied the discovery, i.e. a green run over ZERO tests +# (observed 2026-09-12). `find` needs no repo metadata, so discovery cannot be disabled by +# how the tree happens to be mounted. .git/ is pruned so a stale object copy is never picked up. +discover() { + # Companion convention, anywhere in the tree. + find "${ROOT}" -path "${ROOT}/.git" -prune -o -name '*.promtool-test.yaml' -print 2>/dev/null + # Suite convention: this directory's own *.test.yaml files. + find "${HERE}" -maxdepth 1 -name '*.test.yaml' -print 2>/dev/null +} + +TESTS=() +if [ "$#" -gt 0 ]; then + for t in "$@"; do + # Accept absolute, repo-relative, or cwd-relative paths. + if [ -f "$t" ]; then TESTS+=("$(cd "$(dirname "$t")" && pwd)/$(basename "$t")") + elif [ -f "${ROOT}/$t" ]; then TESTS+=("${ROOT}/$t") + else echo "run.sh: no such test file: $t" >&2; exit 2 + fi + done else - echo "run.sh: SKIP — no promtool on PATH and no podman/docker to run ${PROM_IMAGE}" >&2 + while IFS= read -r abs; do + [ -n "$abs" ] && TESTS+=("$abs") + done < <(discover | sort -u) +fi + +if [ "${#TESTS[@]}" -eq 0 ]; then + echo "run.sh: SKIP (exit 3) — no *.promtool-test.yaml or tests/*.test.yaml found in ${ROOT}" >&2 exit 3 fi -WORK="$(mktemp -d)" -trap 'rm -rf "${WORK}"' EXIT +# ---------------------------------------------------------------- manifest resolution +# Which committed PrometheusRule files a given test file tests. The conventions are the +# CONTRACT (see the header); anything else is refused rather than guessed at, because a test +# silently run against the wrong (or an empty) rule set is the vacuous pass this harness exists +# to prevent. +manifests_for() { + local test_abs="$1" base dir + base="$(basename "$test_abs")" + dir="$(dirname "$test_abs")" + case "$base" in + *.promtool-test.yaml) + local sib="${dir}/${base%.promtool-test.yaml}.yaml" + if [ ! -f "$sib" ]; then + echo "run.sh: ${test_abs#"${ROOT}/"} has no companion manifest (${sib#"${ROOT}/"})" >&2 + return 1 + fi + printf '%s\n' "$sib" ;; + *.test.yaml) + if [ "$(basename "$dir")" != tests ]; then + echo "run.sh: ${test_abs#"${ROOT}/"}: a *.test.yaml must live in a tests/ directory so its" >&2 + echo " manifests are the PrometheusRules of the component above it" >&2 + return 1 + fi + local comp; comp="$(dirname "$dir")" + local found=0 f + while IFS= read -r f; do + found=1; printf '%s\n' "$f" + done < <(grep -rl --include='*.yaml' -e '^[[:space:]]*kind:[[:space:]]*PrometheusRule[[:space:]]*$' \ + "$comp" --exclude-dir=tests 2>/dev/null | sort) + if [ "$found" = 0 ]; then + echo "run.sh: ${test_abs#"${ROOT}/"}: no PrometheusRule manifest in ${comp#"${ROOT}/"}" >&2 + return 1 + fi ;; + *) + echo "run.sh: ${test_abs#"${ROOT}/"}: unrecognised test-file name; use" >&2 + echo " .promtool-test.yaml (companion) or tests/.test.yaml (suite)" >&2 + return 1 ;; + esac +} -# The test files reference the generated rule file by bare name, so everything must sit in one -# directory (that is also the only directory the container needs mounted). +# The rule-file name(s) a test file expects, from its own `rule_files:` block — so the +# extracted rules land under the name the test already references and no test file needs to +# know this harness exists. +rule_file_names() { + sed -n '/^rule_files:[[:space:]]*$/,/^[^[:space:]-]/p' "$1" \ + | sed -n 's/^[[:space:]]*-[[:space:]]*//p' | sed 's/[[:space:]]*$//' | grep . || true +} + +# ------------------------------------------------------------------------------- --list +if [ "$LIST_ONLY" = 1 ]; then + lrc=0 + for test_abs in "${TESTS[@]}"; do + mapfile -t mans < <(manifests_for "$test_abs") || { lrc=1; continue; } + for m in "${mans[@]}"; do + printf '%s\t%s\n' "${test_abs#"${ROOT}/"}" "${m#"${ROOT}/"}" + done + done + exit "$lrc" +fi + +# ---------------------------------------------------------------------------------- run rc=0 -python3 "${HERE}/extract_rules.py" --out "${WORK}/generated.rules.yaml" "${MANIFESTS[@]}" || rc=$? -if [ "$rc" -eq 4 ]; then - echo "run.sh: SKIP — PyYAML unavailable, cannot flatten the PrometheusRule manifests" >&2 - exit 3 -elif [ "$rc" -ne 0 ]; then - exit "$rc" -fi +for test_abs in "${TESTS[@]}"; do + rel="${test_abs#"${ROOT}/"}" + mapfile -t mans < <(manifests_for "$test_abs") || { rc=1; continue; } + mapfile -t rule_names < <(rule_file_names "$test_abs") + if [ "${#rule_names[@]}" -eq 0 ]; then + echo "run.sh: ${rel}: no \`rule_files:\` entries — promtool would test nothing" >&2 + rc=1; continue + fi -names=() -for t in "${TESTS[@]}"; do - cp "$t" "${WORK}/" - names+=("$(basename "$t")") + work="$(mktemp -d)" + # The test file references its rule file by bare name, so everything sits in one directory. + gen="${work}/${rule_names[0]}" + ex_rc=0 + python3 "${HERE}/extract_rules.py" --out "$gen" "${mans[@]}" || ex_rc=$? + if [ "$ex_rc" -eq 4 ]; then + echo "run.sh: SKIP (exit 3) — PyYAML unavailable, cannot flatten the PrometheusRule manifests" >&2 + rm -rf "$work"; exit 3 + elif [ "$ex_rc" -ne 0 ]; then + rm -rf "$work"; rc=1; continue + fi + # A test file may legally name the same rule file more than once, or several names. + for n in "${rule_names[@]:1}"; do cp "$gen" "${work}/${n}"; done + cp "$test_abs" "${work}/" + + # Syntax gate first: rules that do not parse would make every unit test pass vacuously. + echo "run.sh: promtool check rules ${rel} (manifests: ${mans[*]#"${ROOT}/"})" >&2 + ( cd "$work" && "$PROMTOOL_BIN" check rules "${rule_names[0]}" ) || rc=1 + echo "run.sh: promtool test rules ${rel}" >&2 + ( cd "$work" && "$PROMTOOL_BIN" test rules "$(basename "$test_abs")" ) || rc=1 + rm -rf "$work" done -chmod -R a+rX "${WORK}" -echo "run.sh: promtool test rules (${RUNNER}) ${names[*]}" >&2 -case "$RUNNER" in - native) - ( cd "${WORK}" && promtool test rules "${names[@]}" ) ;; - podman) - podman run --rm --user 0 -v "${WORK}:/rules:z" -w /rules \ - --entrypoint /bin/promtool "${PROM_IMAGE}" test rules "${names[@]}" ;; - docker) - docker run --rm --user 0 -v "${WORK}:/rules" -w /rules \ - --entrypoint /bin/promtool "${PROM_IMAGE}" test rules "${names[@]}" ;; -esac +if [ "$rc" -eq 0 ]; then + echo "run.sh: OK — ${#TESTS[@]} rule-unit test file(s) passed under $("$PROMTOOL_BIN" --version 2>&1 | head -1)" >&2 +fi +exit "$rc"