merge: alert when a backup schedule has NEVER run, and when one is paused

VeleroNoRecentBackup could not see the failure that actually happened:
velero-daily sat paused having never produced a successful backup, so
velero_backup_last_successful_timestamp{schedule=...} did not exist and
time() - <absent series> yields no series. Prod had no volume backups and
nothing alerted, further masked by a resources-only schedule succeeding
daily. No velero_* metric describes a Schedule OBJECT, proven by scraping a
throwaway paused Schedule and finding zero lines, so this exposes the CRs
via kube-state-metrics customresource_state and keys on
defaultVolumesToFsBackup — the property, not a name list. Adds the repo's
first prometheus rule-unit-test harness, mutation-checked both ways.

ci/Makefile

@@ -12,7 +12,7 @@ CI_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 .DEFAULT_GOAL := help
 
-.PHONY: help spec-immutability lint schema l0 render-golden l1 conformance gate all touched versions tools test
+.PHONY: help spec-immutability lint schema l0 render-golden l1 prom-rules conformance gate all touched versions tools test
 
 help: ## Show this help
 	@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf "  \033[36m%-14s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST)
@@ -32,6 +32,9 @@ render-golden: ## render==golden: canonical render reproduces committed apps/ (n
 l1: ## L1 rendering checks (no cluster; live now)
 	@"$(CI_DIR)l1.sh"
 
+prom-rules: ## promtool rule-unit tests for platform/monitoring alerts (no cluster; live now)
+	@"$(CI_DIR)prom-rules.sh"
+
 conformance: ## L2/L3 in-cluster Chainsaw Job (SKIPS w/o CLUSTER_AVAILABLE=true)
 	@"$(CI_DIR)conformance.sh"
 

ci/README.md

@@ -22,6 +22,7 @@ Individual stages:
 make -C ci lint        # yaml/lint                          (no cluster; live now)
 make -C ci l0          # schema + Kyverno + L0 static conformance (no cluster; live now)
 make -C ci l1          # L1 rendering checks                 (no cluster; live now)
+make -C ci prom-rules  # promtool rule-unit tests on alerts  (no cluster; live now)
 make -C ci conformance # L2/L3 in-cluster Chainsaw Job       (SKIPS unless CLUSTER_AVAILABLE=true)
 make -C ci test        # unit tests for these scripts themselves (ci/tests/*.test.sh)
 ```
@@ -102,13 +103,16 @@ in that case — nothing goes untested — and says plainly that the *scoped* ch
 
 ```
 ci/
-  Makefile                 targets: lint · l0 · l1 · conformance · gate · touched · test · tools · versions
+  Makefile                 targets: lint · l0 · l1 · prom-rules · conformance · gate · touched · test · tools · versions
   versions.env             PINNED tool versions (R-TEST-02) — single source of truth
   lib.sh                   shared helpers: logging, tool checks, cluster detection, artifact paths
   yamllint.yaml            lint config for the yaml/lint stage
   lint.sh                  Stage: yaml/lint
   l0.sh                    Stage: schema validation + Kyverno L0 gate + L0 static conformance
   l1.sh                    Stage: L1 rendering checks (Chainsaw L1 via the conformance runner)
+  prom-rules.sh            Stage: `promtool test rules` over platform/monitoring's PrometheusRules
+                           (harness: platform/monitoring/tests/). Catches the two alerting bugs no
+                           other stage can see — permanently-firing and structurally INERT rules.
   conformance.sh           Stage: L2/L3 in-cluster Chainsaw Job (trigger + collect)
   touched-requirements.sh  compute the R-* subset a change touches (drives the R-TEST-03 filter);
                            exits 3 LOUDLY rather than guessing when the diff base is unreliable

ci/gate.sh

@@ -42,6 +42,7 @@ run_stage lint          "${CI_DIR}/lint.sh"
 run_stage l0            "${CI_DIR}/l0.sh"
 run_stage render-golden "${CI_DIR}/render-golden.sh"
 run_stage l1            "${CI_DIR}/l1.sh"
+run_stage prom-rules    "${CI_DIR}/prom-rules.sh"
 run_stage conformance   "${CI_DIR}/conformance.sh"
 
 echo >&2
@@ -86,7 +87,7 @@ esac
 # Phase 0 that is tolerated (placeholders), but surfaced. render-golden is a
 # no-cluster reproducibility gate (canonical render == committed apps/ goldens):
 # a FAIL means a golden drifted from its ServiceDefinition source — always blocks.
-for s in lint l0 render-golden l1; do
+for s in lint l0 render-golden l1 prom-rules; do
   case "${OUTCOME[$s]}" in
     fail) err  "${s}: FAILED -> blocks merge"; BLOCK=1 ;;
     skip) warn "${s}: skipped (unwired seam / missing tool) — not green" ;;

ci/prom-rules.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# ci/prom-rules.sh — Stage: Prometheus RULE-UNIT tests (no cluster; live now).
+#
+# Thin wrapper over platform/monitoring/tests/run.sh, which runs `promtool test rules` against the
+# COMMITTED PrometheusRule manifests. It is a gate stage because the two alerting defects this repo
+# has actually shipped — a permanently-firing rule and a structurally INERT one — are both invisible
+# to every other stage: yamllint parses them, Kyverno admits them, and a healthy cluster cannot tell
+# a correct alert from one that can never fire. Only the bad state, constructed on purpose, can.
+#
+# Exit codes per ci/lib.sh: 0 pass / 1 fail / 3 skipped (never green).
+set -euo pipefail
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck disable=SC1091
+source "${CI_DIR}/lib.sh"
+
+log "Stage: Prometheus rule-unit tests (promtool test rules; no cluster)"
+
+RUNNER="${REPO_ROOT}/platform/monitoring/tests/run.sh"
+if [ ! -x "$RUNNER" ]; then
+  skip "rule-unit harness not present at ${RUNNER} — UNWIRED, not green"
+  summary "- prom-rules: **SKIPPED** (harness missing)"
+  exit 3
+fi
+
+set +e
+"$RUNNER"
+rc=$?
+set -e
+
+case "$rc" in
+  0)
+    ok "Prometheus rule-unit tests passed"
+    summary "- prom-rules: **passed**"
+    ;;
+  3)
+    skip "promtool unavailable (no promtool on PATH, no podman/docker) — NOT green"
+    summary "- prom-rules: **SKIPPED** (promtool toolchain unavailable)"
+    ;;
+  *)
+    err "Prometheus rule-unit tests FAILED"
+    summary "- prom-rules: **FAILED**"
+    ;;
+esac
+exit "$rc"