#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — alert-rules-unit-tested
# Serves: R-OBS-04 ("Platform alerts on its own health (nodes, storage, GitOps, certs,
#                    VPN)." — spec docs/10-observability-and-alerting.md)
#         R-TEST-02 ("Tests are reproducible: deterministic, hermetic, single-command."
#                    — spec docs/14-testing-and-conformance.md)
#
# WHY THIS EXISTS — AN UNTESTED ALERT IS NOT A MONITORING CONTROL
# --------------------------------------------------------------
# This repo has shipped BOTH halves of the alerting failure space to prod:
#   * a PERMANENTLY-FIRING `critical` (the CNPG alert fixed 2026-09-12; VPNTunnelDown before
#     it), and
#   * a STRUCTURALLY INERT rule — VeleroNoRecentBackup, whose `time() - <absent series>`
#     yields no series, so a schedule that had NEVER backed up could not raise it. Prod ran
#     with no volume backups at all and nothing alerted (2026-09-12).
# Neither is visible from the expression, from `kubectl apply`, or from a healthy cluster: an
# inert rule and a correct rule are indistinguishable while everything is fine. `promtool test
# rules` is the only tool that separates them, because the BAD state can be constructed on
# purpose. Unit tests for those rules were then WRITTEN — and evaluated by nothing: promtool
# was not in the pinned toolchain, no gate stage ran the files, and each of the two harnesses
# that existed acquired an engine off the network mid-run if one was missing. Coverage that
# looks present and is never enforced is the failure mode this check ends.
#
# GIVEN the repository as authored,
# WHEN  the ONE rule-unit harness (platform/monitoring/tests/run.sh, which flattens the
#       COMMITTED PrometheusRule CRs — never a copy) is asked which manifests are under test
#       and then run over every test file in the repo,
# THEN  (1) promtool is a PINNED member of the toolchain: PROMTOOL_VERSION is declared in BOTH
#           infra/devcontainer/versions.env and ci/versions.env and the two AGREE (the repo's
#           lockstep rule), the Containerfile takes it as a build ARG, installs it from the
#           pinned prometheus release and records it for provenance, build-image.sh (the one
#           recipe `dev.sh build` and CI both use) passes it, and the
#           CI runner installs it for the l0 stage;
#       (2) the promtool ACTUALLY IN USE reports exactly that pinned version — a unit test is
#           evidence only if the engine matches the one prod evaluates rules with;
#       (3) NO harness acquires a rule engine at RUN TIME — neither `curl | tar -xz` of a
#           GitHub release nor a `podman/docker run` of a prometheus image. Both existed on
#           2026-09-12 and both are removed; this is the regression guard;
#       (4) there is exactly ONE harness. Two were written independently within an hour of
#           each other; the one nothing runs is the one that rots;
#       (5) every rule-unit test file in the repo PASSES, run through that harness (which
#           `promtool check rules` first, so a non-parsing rules file cannot make unit tests
#           pass vacuously by finding no rules at all);
#       (6) the harness is NOT VACUOUS: a synthetic PrometheusRule + companion test is run
#           through the SAME harness twice — intact (must PASS) and with one expectation
#           mutated (must FAIL) — so a harness that has silently stopped evaluating anything
#           is caught on every run, not only when someone thinks to try it;
#   AND (7) the alert rules with NO unit tests are ENUMERATED and RATCHETED: each is declared
#           below, printed with counts on every run, and the register can only SHRINK.
#
# WHY A MISSING TEST IS A RATCHET AND NOT AN OUTRIGHT FAIL (the decision, and its reasons)
# ---------------------------------------------------------------------------------------
# 48 of the repo's 55 alert rules have no unit test today. Three options were on the table:
#   * FAIL on any missing test — correct in principle, but it makes the merge gate red for
#     every change until ~48 unit tests exist, so in practice the gate gets disabled, and a
#     disabled gate protects nothing.
#   * merely REPORT — the gap prints and nothing holds it. That is exactly how the CNPG unit
#     tests came to exist and be enforced by nothing; "it is in the output somewhere" is not
#     enforcement.
#   * RATCHET (chosen) — the gap is DECLARED IN THE REPO as a reviewable register, printed with
#     counts on every gate run, and it CANNOT GROW: a new PrometheusRule, or a new alert in a
#     file that already has tests, FAILS unless it ships a unit test. Removing an entry requires
#     writing the test, and a STALE entry (file gone, or now covered) also FAILS, so the
#     register cannot rot into fiction.
# "SKIPPED is not green" is honoured: a missing test is NEVER an exit-3 skip here. This check
# exits 0 only with the gap fully enumerated, and exits 1 the moment the debt grows, the
# register drifts, or a test that exists stops passing.
# The one exit-3 path is promtool itself being ABSENT — an environment gap, not a code property
# (the same convention as ci/l0.sh's kyverno handling), and this test DECLARES
# `conformance.home.k3s/scope: whole-repo`, which makes lib/run_l1_offline.py turn that skip
# into the runner's rc 3 and ci/l0.sh into the gate's rc 3: UNWIRED, never green. promtool
# PRESENT AT THE WRONG VERSION is a FAILURE, not a skip: that case still prints SUCCESS while
# evaluating the rules with an engine prod does not run.
#
# WHAT THIS DOES *NOT* PROVE (honest gaps):
#   - It does not prove an alert reaches a human. promtool evaluates rules against synthetic
#     series in-process; Alertmanager routing, SMTP delivery and inhibition are L3
#     (tests/l3-fault-injection/alerting) and R-OBS-02, and are not claimed here.
#   - It does not prove the THRESHOLDS are right, only that a rule behaves as its own unit
#     test says.
#   - It does not prove a rule file is RECONCILED into a cluster. An alert file no
#     kustomization includes is a separate "looks applied and isn't" failure, not audited here.
#   - Coverage of an individual alert is detected by its NAME appearing in a test file that
#     covers its manifest. A test file that merely mentions a name without asserting anything
#     counts as covered; only the mutation self-test (6) proves the harness CAN fail, not that
#     every author wrote a meaningful assertion. Review still owns test quality.
#   - It does not read the live cluster, so "the pin matches prod's Prometheus" is asserted
#     against the repo's pin, which was established by a live read recorded in
#     infra/devcontainer/versions.env (2026-09-12: v3.14.0-distroless).
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"
HARNESS="${ALERT_RULE_HARNESS:-${ROOT}/platform/monitoring/tests/run.sh}"

# Honest SKIP (never green) if the thing under test is not present at all. Because this test
# declares scope: whole-repo, the runner turns this into rc 3 for the whole L0 stage — a
# PrometheusRule-less repo is a vanished tree, not a pass.
if ! grep -rqls --include='*.yaml' -e '^[[:space:]]*kind:[[:space:]]*PrometheusRule[[:space:]]*$' \
       "${ROOT}/platform" "${ROOT}/apps" 2>/dev/null; then
  echo "check SKIPPED: no PrometheusRule under platform/ or apps/ — nothing to test" >&2
  exit 3
fi

# promtool resolution. PROMTOOL= overrides (the same override the harness honours); otherwise
# PATH. Absent -> every OFFLINE assertion still runs and the check exits 3, so a toolchain gap
# is reported as a gap and never as a pass.
PROMTOOL_BIN="${PROMTOOL:-$(command -v promtool || true)}"
if [ -n "${PROMTOOL_BIN}" ] && [ -x "${PROMTOOL_BIN}" ]; then
  PROMTOOL_BIN="$(cd "$(dirname "${PROMTOOL_BIN}")" && pwd)/$(basename "${PROMTOOL_BIN}")"
else
  PROMTOOL_BIN=""
fi

cd "${ROOT}"
"${PYTHON}" - "${ROOT}" "${HARNESS}" "${PROMTOOL_BIN}" <<'PY'
"""Assert the alert-rule unit tests are pinned, enforced, non-vacuous, and that the alert
rules with no tests are enumerated and ratcheted. Standard library + the harness itself."""
from __future__ import annotations

import os
import re
import shutil
import subprocess
import sys
import tempfile
from collections import defaultdict
from pathlib import Path

ROOT = Path(sys.argv[1])
HARNESS = Path(sys.argv[2])
PROMTOOL = sys.argv[3]
sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree

FAILURES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(msg)


# =============================================================================
# THE DECLARED REGISTER (assertion 7). Shrink-only; see the header for why this is a ratchet.
#
# UNTESTED_FILES — PrometheusRule files no test file covers AT ALL. A file listed here that later
# gains coverage, or stops carrying a PrometheusRule, FAILS as stale.
#
# THE TWELVE apps/* ENTRIES ARE GONE (2026-09-13). They were the single `<app>-down` liveness rules
# generated from one ServiceDefinition template, and the register's description of them as one
# template was exactly why leaving them untested was so expensive: ONE wrong expression was wrong
# twelve times, and it was — `up{job="<app>"} == 0` against a job no scrape target produced, inert on
# prod for all twelve. They are now covered by apps/wave-0/tests/app-slis.test.yaml,
# apps/wave-1/tests/app-slis.test.yaml and apps/wave-2/tests/app-slis.test.yaml, each rule pinned
# BOTH ways (quiet on the healthy state, firing on no-ready-replica, and firing when the signal
# VANISHES — the failure mode that produced the class). platform/kubevirt/alerts.yaml (4 rules) is
# now the only entry left and is where the next tests belong.
# =============================================================================
UNTESTED_FILES = {
    "platform/kubevirt/alerts.yaml",
}

# UNTESTED_ALERTS — individual alerts inside a file that IS covered by a test file, but whose
# own rule nothing asserts on. This is the half that stops a new alert being slipped into an
# already-"tested" file untested: anything not listed here must appear in a covering test.
# The monitoring block is large because backups.test.yaml (the repo's first suite) deliberately
# covers the Velero/backup rules only. It SHRANK by seven on 2026-09-13, when
# platform/monitoring/tests/longhorn.test.yaml took on the whole platform-health.longhorn group
# plus the three kubelet-based PVC-fill rules it has to be measured against (the per-volume
# Longhorn "capacity" rule was deleted as a permanently-firing duplicate of them, so proving the
# redundancy meant asserting on the rules that survive). Only
# PersistentVolumeFillingUpPredicted is still unasserted in pvc-capacity-alerts.yaml.
# It SHRANK BY TWO MORE on 2026-09-14: platform/monitoring/tests/node-io-cause.test.yaml pins the
# eight new platform-health.node-io-cause rules, and its healthy fixture asserts NodeIOPressureHigh
# and NodeIOPressureCritical quiet on the same measured state — the whole point of that fixture is
# that the two PSI detectors and the eight cause rules must agree about what "healthy" is, so the
# detectors are no longer unasserted.
# It SHRANK BY ONE on 2026-09-19: platform/monitoring/tests/vpn-egress.test.yaml pins the two new
# egress-steering rules, and asserts VPNTunnelDown QUIET in all four fixtures — that silence IS the
# claim those rules exist for (tun0 up and passing traffic while vpn-class egress is dropped), so it
# had to become an assertion rather than a sentence in a comment.
UNTESTED_ALERTS = {
    ("platform/cnpg/alerts.yaml", "CNPGWALArchiveBacklog"),
    ("platform/cnpg/alerts.yaml", "CNPGNoRecoverabilityPoint"),
    ("platform/cnpg/alerts.yaml", "CNPGNoRecentBaseBackup"),
    ("platform/cnpg/alerts.yaml", "CNPGWALVolumeNotRecycling"),
    ("platform/cnpg/alerts.yaml", "CNPGMetricsCollectorErroring"),
    ("platform/monitoring/alerts.yaml", "NodeExporterDown"),
    ("platform/monitoring/alerts.yaml", "NodeMemoryPressureHigh"),
    ("platform/monitoring/alerts.yaml", "NodeDiskFillingUp"),
    ("platform/monitoring/alerts.yaml", "NodeCPUPressureHigh"),
    ("platform/monitoring/alerts.yaml", "NodeCPULoadHigh"),
    ("platform/monitoring/alerts.yaml", "FluxReconciliationFailure"),
    ("platform/monitoring/alerts.yaml", "FluxResourceSuspended"),
    ("platform/monitoring/alerts.yaml", "CertificateExpiringSoon"),
    ("platform/monitoring/alerts.yaml", "CertificateExpiryImminent"),
    ("platform/monitoring/alerts.yaml", "CertificateNotReady"),
    ("platform/monitoring/alerts.yaml", "HubbleDeniedFlowSpike"),
    ("platform/monitoring/alerts.yaml", "KeycloakDown"),
    ("platform/monitoring/alerts.yaml", "KeycloakDatabaseDown"),
    ("platform/monitoring/alerts.yaml", "KeycloakClientSecretReconcilerFailing"),
    ("platform/monitoring/alerts.yaml", "KeycloakClientSecretReconcilerStale"),
    ("platform/monitoring/alerts.yaml", "PrometheusTargetsMissing"),
    ("platform/monitoring/alerts.yaml", "ImageUpdateAutomationFailing"),
    ("platform/monitoring/alerts.yaml", "ImagePolicyStalled"),
    ("platform/monitoring/alerts.yaml", "UpdateHealthGateFailed"),
    ("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeFillingUpPredicted"),
}

# --------------------------------------------------------------- (1) the pins, in lockstep
def env_pin(rel: str, key: str) -> str | None:
    f = ROOT / rel
    if not f.is_file():
        return None
    m = re.search(rf'^{re.escape(key)}=["\']?([^"\'\s#]+)', f.read_text(), re.M)
    return m.group(1) if m else None


DEV_PIN = env_pin("infra/devcontainer/versions.env", "PROMTOOL_VERSION")
CI_PIN = env_pin("ci/versions.env", "PROMTOOL_VERSION")
check(DEV_PIN is not None,
      "infra/devcontainer/versions.env declares no PROMTOOL_VERSION — promtool is not part of the "
      "baked toolchain, so the alert unit tests are evaluated by whatever the host happens to "
      "have, or by nothing at all (which is where this repo was on 2026-09-12)")
check(CI_PIN is not None,
      "ci/versions.env declares no PROMTOOL_VERSION — the gate's own version file does not pin the "
      "rule engine, so CI and the devcontainer can silently diverge")
check(DEV_PIN is None or DEV_PIN == CI_PIN,
      f"PROMTOOL_VERSION is NOT in lockstep: infra/devcontainer/versions.env={DEV_PIN!r} vs "
      f"ci/versions.env={CI_PIN!r}. The repo's rule is that the conformance-tool versions in the "
      f"two files stay identical, so a laptop and CI run the same engine")

CF = ROOT / "infra/devcontainer/Containerfile"
cf_text = CF.read_text() if CF.is_file() else ""
check(re.search(r"^ARG PROMTOOL_VERSION\s*$", cf_text, re.M),
      "infra/devcontainer/Containerfile has no `ARG PROMTOOL_VERSION` — the pin is declared but "
      "the image build does not consume it")
check("prometheus/prometheus/releases/download" in cf_text and "promtool" in cf_text,
      "infra/devcontainer/Containerfile does not install promtool from the pinned prometheus "
      "release tarball (promtool ships inside it)")
check('echo "promtool=${PROMTOOL_VERSION}"' in cf_text,
      "infra/devcontainer/Containerfile does not record promtool in /etc/dev-toolchain-versions — "
      "that provenance line is how an operator tells a stale image from a current one")

# The build recipe moved from dev.sh into build-image.sh on 2026-10-29, so that `dev.sh build`
# and the CI image pipeline (.gitea/workflows/images.yaml) share ONE recipe. Check the recipe.
BUILDSH = ROOT / "infra/devcontainer/build-image.sh"
check(BUILDSH.is_file() and '"PROMTOOL_VERSION=$PROMTOOL_VERSION"' in BUILDSH.read_text(),
      "infra/devcontainer/build-image.sh does not pass PROMTOOL_VERSION in its BUILD_ARGS — the ARG "
      "would be empty and the build would fetch a bogus URL")

INSTALL = ROOT / ".gitea/workflows/scripts/install-tools.sh"
if INSTALL.is_file():
    itext = INSTALL.read_text()
    check("PROMTOOL_VERSION" in itext
          and re.search(r"^\s*l0\)[^\n]*install_promtool", itext, re.M),
          ".gitea/workflows/scripts/install-tools.sh does not install promtool for the l0 stage — "
          "on the CI runner this check would exit 3 (UNWIRED) forever and the alert unit tests "
          "would be enforced nowhere, which is the exact gap it exists to close")

# --------------------------------------------- (3)(4) one harness, and it fetches no engine
check(HARNESS.is_file() and os.access(HARNESS, os.X_OK),
      f"the rule-unit harness is missing or not executable at {HARNESS.relative_to(ROOT)} — "
      f"nothing would run the alert unit tests")

# Any script that invokes `promtool test rules` is a rule-unit harness. There must be exactly
# one: two harnesses were written independently on 2026-09-12 (a cnpg-only script and this one),
# and the one nothing runs is the one that rots out of sync with the rules.
harnesses: list[str] = []
fetchers: list[str] = []
FETCH_RE = re.compile(
    r"(curl|wget)[^\n]*prometheus/prometheus/releases"          # GitHub release download
    r"|(podman|docker)\s+run[^\n]*prometheus",                  # container-image pull
    re.I)
# Two exclusions, both the repo's existing "reference, not declaration" rule (see
# ci/touched-requirements.sh changed_diff() rules 2-3):
#   * THIS CHECK'S OWN DIRECTORY. It names every forbidden pattern in order to forbid it, so
#     scanning itself would make both guards permanently red — the self-referential trap.
#   * THE PINNED TOOLCHAIN INSTALLERS. Downloading the pinned promtool is their JOB, at a
#     reviewed seam (an image build, or the runner's tool-install step), from a version in
#     versions.env. What is forbidden is a RULE-UNIT RUN acquiring its own engine: then the
#     version, the network and the machine are decided silently at test time, and a missing
#     toolchain becomes a green run instead of an exit 3.
# NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such
# as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs, so
# a bare rglob from ROOT audited up to 18 duplicates of every file — which both invents FAILs
# and, worse, can satisfy an "something declares X" check from a copy rather than real source.
# Same fix as egress-paths-selected-by-rules. .claude/ is gitignored (.gitignore:43).
SELF_DIR = "conformance/tests/l0-static/alert-rules-unit-tested/"
INSTALLERS = (".gitea/workflows/scripts/install-tools.sh", "infra/devcontainer/")
for p in sorted(ROOT.rglob("*.sh")):
    rel = str(p.relative_to(ROOT))
    if rel.startswith((".git/", ".claude/", "conformance/runs/", SELF_DIR)):
        continue
    try:
        text = p.read_text()
    except (OSError, UnicodeDecodeError):
        continue
    body = "\n".join(l for l in text.splitlines() if not l.lstrip().startswith("#"))
    if "promtool" in body and re.search(r"test\s+rules", body):
        harnesses.append(rel)
    if FETCH_RE.search(body) and not rel.startswith(INSTALLERS):
        fetchers.append(rel)

check(harnesses == [str(HARNESS.relative_to(ROOT))],
      f"expected exactly ONE rule-unit harness ({HARNESS.relative_to(ROOT)}) but found "
      f"{harnesses}. A second harness means two behaviours and two things to keep in step; the "
      f"one the gate does not run is the one that rots. Fold it in instead")
check(not fetchers,
      f"{fetchers} acquire a rule ENGINE at run time (a prometheus release download or a "
      f"prometheus container image). promtool must come from the pinned toolchain "
      f"(PROMTOOL_VERSION): a run-time fetch decides the version, the network and the machine "
      f"silently at test time, makes the tests non-hermetic (R-TEST-02), and quietly turns 'the "
      f"toolchain has no promtool' into a green run — which is how it stayed missing")

# ------------------------------------------------------------------ discover the alert files
# Alert files: tracked YAML carrying a PrometheusRule. conformance/ is excluded deliberately —
# the PrometheusRules under tests/l1-rendering/*/golden/ are EXPECTED-RENDER fixtures for the
# render checks, not rules any cluster evaluates. Everything a cluster actually gets lives under
# apps/, platform/, platform-config/ or clusters/.
def tracked_yaml() -> list[Path]:
    out = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z", "*.yaml", "*.yml"],
                         capture_output=True, text=True)
    if out.returncode == 0 and out.stdout:
        return [ROOT / p for p in out.stdout.split("\0") if p]
    # No usable index (the gate container mounts a worktree whose .git points outside the
    # mount). Fall back to a filesystem walk rather than auditing nothing.
    return sorted(p for p in ROOT.rglob("*.y*ml") if not any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts))


KIND_RE = re.compile(r"^\s*kind:\s*PrometheusRule\s*$", re.M)
ALERT_RE = re.compile(r"^\s*-\s*alert:\s*(\S+)", re.M)

alert_files: dict[str, list[str]] = {}
for p in tracked_yaml():
    rel = str(p.relative_to(ROOT))
    if rel.startswith("conformance/") or not p.is_file():
        continue
    try:
        text = p.read_text()
    except (OSError, UnicodeDecodeError):
        continue
    if KIND_RE.search(text):
        alert_files[rel] = ALERT_RE.findall(text)

check(len(alert_files) > 0,
      "no PrometheusRule found outside conformance/ — discovery or the repo layout changed, and "
      "this check would silently audit nothing")

# ------------------- ask the HARNESS which manifests are under test (never re-derive it here)
# Two implementations of "which rules does this test cover" would eventually disagree, and the
# disagreement shows up as a file the gate believes is tested and the harness never touches.
listing = subprocess.run(["bash", str(HARNESS), "--list"], capture_output=True, text=True,
                         cwd=str(ROOT))
check(listing.returncode == 0,
      f"`{HARNESS.relative_to(ROOT)} --list` failed (rc={listing.returncode}): "
      f"{(listing.stderr or listing.stdout).strip()[-600:]}")
covering: dict[str, list[str]] = defaultdict(list)   # manifest -> [test files]
for line in listing.stdout.splitlines():
    if "\t" in line:
        tfile, manifest = line.split("\t", 1)
        covering[manifest.strip()].append(tfile.strip())

for manifest in sorted(covering):
    check(manifest in alert_files,
          f"the harness reports {manifest} as under test, but it is not one of the "
          f"PrometheusRule files found in the repo — the mapping and the tree disagree")

# --------------------------------------------------------- (2) the engine is the pinned one
version_line = ""
if PROMTOOL:
    vp = subprocess.run([PROMTOOL, "--version"], capture_output=True, text=True)
    out = (vp.stdout + vp.stderr).strip()
    version_line = out.splitlines()[0] if out else ""
    m = re.search(r"version\s+(\d+\.\d+\.\d+)", version_line)
    found = m.group(1) if m else None
    check(found is not None,
          f"could not parse a version from `promtool --version`: {version_line!r}")
    check(found is None or DEV_PIN is None or found == DEV_PIN,
          f"promtool reports {found}, but the toolchain pins PROMTOOL_VERSION={DEV_PIN} (the "
          f"Prometheus version prod runs). promtool IS the rule engine: a unit test green under "
          f"one version and red under another is worse than no test. Run the gate in the pinned "
          f"devcontainer (rebuild it: ./infra/devcontainer/dev.sh build), or set PROMTOOL= to a "
          f"v{DEV_PIN} binary. A FAILURE, not a skip, because the wrong engine still prints "
          f"SUCCESS")

# ------------------------------------------------- (5) every rule-unit test file must pass
if PROMTOOL:
    env = dict(os.environ, PROMTOOL=PROMTOOL)
    run = subprocess.run(["bash", str(HARNESS)], capture_output=True, text=True,
                         cwd=str(ROOT), env=env)
    tailed = "\n        ".join((run.stdout + run.stderr).strip().splitlines()[-40:])
    check(run.returncode == 0,
          f"`{HARNESS.relative_to(ROOT)}` FAILED (rc={run.returncode}) — the committed alert "
          f"rules do not behave as their own unit tests say:\n        {tailed}")

    # ------------------------------------------- (6) the harness is proven non-vacuous
    # A synthetic PrometheusRule + companion test through the SAME harness, outside the repo so
    # discovery never picks it up. Intact must PASS; a mutated expectation must FAIL. Without
    # this, a harness that stopped evaluating anything (an empty extraction, a rule file written
    # under the wrong name) would report green forever — which is the disease being treated.
    work = tempfile.mkdtemp(prefix="alert-rules-selftest.")
    try:
        (Path(work) / "synth.yaml").write_text("""apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: selftest
spec:
  groups:
    - name: selftest
      rules:
        - alert: SelfTestCanary
          expr: selftest_metric > 0
          for: 5m
          annotations:
            runbook_url: https://${BASE_DOMAIN}/runbooks/selftest
""")
        test_yaml = """rule_files:
  - generated.rules.yaml
evaluation_interval: 1m
tests:
  - interval: 1m
    name: canary fires only after `for` elapses
    input_series:
      - series: 'selftest_metric'
        values: '1x10'
    promql_expr_test:
      - expr: 'ALERTS{alertname="SelfTestCanary",alertstate="firing"}'
        eval_time: EVAL_TIME
        exp_samples:
          - labels: 'ALERTS{alertname="SelfTestCanary",alertstate="firing"}'
            value: 1
"""
        synth_test = Path(work) / "synth.promtool-test.yaml"
        synth_test.write_text(test_yaml.replace("EVAL_TIME", "6m"))
        good = subprocess.run(["bash", str(HARNESS), str(synth_test)],
                              capture_output=True, text=True, cwd=str(ROOT), env=env)
        check(good.returncode == 0,
              "self-test: the INTACT synthetic unit test did not pass, so the harness itself is "
              "broken and every green result above is meaningless:\n        "
              + "\n        ".join((good.stdout + good.stderr).strip().splitlines()[-20:]))
        # Mutation: assert firing at 2m, which `for: 5m` forbids.
        synth_test.write_text(test_yaml.replace("EVAL_TIME", "2m"))
        bad = subprocess.run(["bash", str(HARNESS), str(synth_test)],
                             capture_output=True, text=True, cwd=str(ROOT), env=env)
        check(bad.returncode != 0,
              "self-test: a MUTATED expectation (firing before `for:` elapsed) still PASSED — "
              "the harness is not actually evaluating anything, so every green result above is "
              "vacuous")
    finally:
        shutil.rmtree(work, ignore_errors=True)

# --------------------------------------------------------- (7) the gap inventory + ratchet
INVENTORY: list[str] = []
GAP_FILES: list[str] = []
alerts_total = covered_total = 0

for rel in sorted(alert_files):
    names = alert_files[rel]
    alerts_total += len(names)
    check(len(names) > 0,
          f"{rel} declares a PrometheusRule but no `alert:` rule was parsed out of it — the "
          f"parser or the file shape changed, and any unit test over it would pass vacuously")
    tests = covering.get(rel, [])
    if not tests:
        GAP_FILES.append(rel)
        INVENTORY.append(f"  {rel:<46} alerts={len(names):<3} tested=0/{len(names):<3} NO TEST FILE")
        check(rel in UNTESTED_FILES,
              f"{rel} carries {len(names)} alert rule(s) and NO test file covers it. A new "
              f"PrometheusRule must ship promtool unit tests: this repo has shipped both a "
              f"permanently-firing critical and a structurally inert alert, and neither was "
              f"visible without them. Add `{rel[:-5]}.promtool-test.yaml` (companion convention) "
              f"or a <component>/tests/*.test.yaml suite. If this is deliberate debt it must be "
              f"DECLARED in UNTESTED_FILES in this check — the register is what keeps the gap "
              f"visible instead of silent")
        continue

    check(rel not in UNTESTED_FILES,
          f"{rel} is listed in UNTESTED_FILES but is now covered by {tests} — the register is "
          f"shrink-only: delete the line so the next missing test is caught")
    text = "".join((ROOT / t).read_text() for t in tests if (ROOT / t).is_file())
    # "COVERED" MUST MEAN ASSERTED, NOT MENTIONED (tightened 2026-09-19).
    # This used to be a bare word match over the whole test file, so an alert named anywhere —
    # including in a PROSE COMMENT — counted as tested. Hit for real while adding
    # platform/monitoring/tests/vpn-egress.test.yaml: its header comment explains that VPNTunnelDown
    # stays quiet during an egress-steering failure, and that sentence alone was enough to make this
    # check declare VPNTunnelDown "asserted on" and demand its removal from UNTESTED_ALERTS. Deleting
    # the register entry would then have recorded coverage that did not exist — the exact "looks fixed"
    # outcome this check exists to prevent, produced BY the check.
    # Now an alert counts only when a test file names it in an `alertname:` position, with comment
    # lines stripped first so a commented-out assertion does not count either.
    # MEASURED BLAST RADIUS BEFORE TIGHTENING: of 61 alerts across the PrometheusRule manifests, 32 are
    # asserted in one of the two idioms below, 29 are not mentioned at all (already declared in the
    # registers), and ZERO are prose-only — so this change failed nothing on the day it landed. That
    # mattered because the registers are shrink-only: had any alert been prose-only, tightening would
    # have needed a register entry it is not allowed to add. (The first measurement said 31/30/0 because
    # it scanned only tests/*.test.yaml and missed the <manifest>.promtool-test.yaml companions.)
    text_no_comments = "\n".join(
        l for l in text.split("\n") if not l.lstrip().startswith("#")
    )
    # BOTH promtool idioms count, because both are real assertions:
    #   alert_rule_test:   - alertname: CNPGPrimariesCoLocated
    #   promql_expr_test:  - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}'
    # Matching only the first produced a FALSE NEGATIVE on CNPGPrimariesCoLocated, which
    # platform/cnpg/alerts.promtool-test.yaml pins across FOUR scenarios entirely through the second
    # form. Had that been believed, the "fix" would have been a redundant duplicate test for an alert
    # already better covered than most.
    covered = [n for n in names
               if re.search(rf'(?:alertname:\s*|alertname="){re.escape(n)}\b', text_no_comments)]
    covered_total += len(covered)
    for n in names:
        if n in covered:
            check((rel, n) not in UNTESTED_ALERTS,
                  f"{rel}: alert {n} is listed in UNTESTED_ALERTS but IS asserted on now — "
                  f"remove the stale entry (the register is shrink-only)")
        else:
            check((rel, n) in UNTESTED_ALERTS,
                  f"{rel}: alert {n} has NO unit test in {tests}. This file is already covered, "
                  f"so a new alert in it must be tested too — otherwise a 'tested' alert file "
                  f"quietly grows untested rules")
    INVENTORY.append(f"  {rel:<46} alerts={len(names):<3} tested={len(covered)}/{len(names):<3} "
                     f"{','.join(tests)}")

# stale register entries
for rel in sorted(UNTESTED_FILES - set(alert_files)):
    check(False, f"UNTESTED_FILES lists {rel}, which carries no PrometheusRule in this repo any "
                 f"more — a stale exemption hides the next real gap; delete the line")
for rel in sorted({f for f, _ in UNTESTED_ALERTS} - set(alert_files)):
    check(False, f"UNTESTED_ALERTS lists alerts in {rel}, which carries no PrometheusRule in this "
                 f"repo any more — delete those lines")

# --------------------------------------------------------------------------------- report
print("Alert-rule unit-test inventory (PrometheusRule -> covering test file(s)):")
for line in INVENTORY:
    print(line)
print(f"\nharness: {HARNESS.relative_to(ROOT)}   promtool: {version_line or 'NOT FOUND'}")
print(f"pinned:  PROMTOOL_VERSION={DEV_PIN} (devcontainer) / {CI_PIN} (ci) — prod runs Prometheus "
      f"v{DEV_PIN}")
print(f"alert files: {len(alert_files)}   covered by a test file: "
      f"{len(alert_files) - len(GAP_FILES)}   with NO test file: {len(GAP_FILES)}")
print(f"alert rules: {alerts_total}   unit-tested: {covered_total}   "
      f"NOT unit-tested: {alerts_total - covered_total}")

if GAP_FILES:
    print("\nDECLARED GAPS — alert files nothing tests at all (shrink-only register; each is a")
    print("rule nothing has ever evaluated, NOT a rule known to work):")
    for g in GAP_FILES:
        print(f"  - {g}  ({len(alert_files[g])} alert(s))")
if UNTESTED_ALERTS:
    print("\nDECLARED GAPS — individual alerts inside a covered file that nothing asserts on:")
    for f, n in sorted(UNTESTED_ALERTS):
        print(f"  - {f}  {n}")

print()
if not PROMTOOL and not FAILURES:
    print("check SKIPPED (exit 3, NOT a pass): promtool is not on PATH, so NO alert rule was "
          "parsed and NO unit test was evaluated.", file=sys.stderr)
    print(f"  promtool is pinned at v{DEV_PIN} in infra/devcontainer/versions.env + "
          f"ci/versions.env.", file=sys.stderr)
    print("  Run the gate in the pinned toolbox (./infra/devcontainer/dev.sh build, then "
          "make -C ci gate), or set PROMTOOL=/path/to/promtool.", file=sys.stderr)
    print(f"  The {CHECKS} offline assertions above (pins, lockstep, one harness, no run-time "
          f"engine fetch, the gap register) DID run and passed.", file=sys.stderr)
    print("  This test declares scope: whole-repo, so the runner reports rc 3 — UNWIRED, never "
          "green.", file=sys.stderr)
    raise SystemExit(3)

if not FAILURES:
    print(f"check OK: alert-rules-unit-tested — {CHECKS} assertions "
          f"({covered_total}/{alerts_total} alert rules unit-tested, "
          f"{len(GAP_FILES)} files with no test file, all declared) (R-OBS-04, R-TEST-02)")
    raise SystemExit(0)
print(f"CHECK FAIL (alert-rules-unit-tested): {len(FAILURES)} failed, "
      f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
for f in FAILURES:
    print(f"  FAIL {f}", file=sys.stderr)
raise SystemExit(1)
PY