diff --git a/ci/Makefile b/ci/Makefile index 3d426da..c51a1ee 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -12,7 +12,7 @@ CI_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) .DEFAULT_GOAL := help -.PHONY: help spec-immutability lint schema l0 render-golden l1 conformance gate all touched versions tools test +.PHONY: help spec-immutability lint schema l0 render-golden l1 prom-rules conformance gate all touched versions tools test help: ## Show this help @awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " \033[36m%-14s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST) @@ -32,6 +32,9 @@ render-golden: ## render==golden: canonical render reproduces committed apps/ (n l1: ## L1 rendering checks (no cluster; live now) @"$(CI_DIR)l1.sh" +prom-rules: ## promtool rule-unit tests for platform/monitoring alerts (no cluster; live now) + @"$(CI_DIR)prom-rules.sh" + conformance: ## L2/L3 in-cluster Chainsaw Job (SKIPS w/o CLUSTER_AVAILABLE=true) @"$(CI_DIR)conformance.sh" diff --git a/ci/README.md b/ci/README.md index bec4949..9588fe9 100644 --- a/ci/README.md +++ b/ci/README.md @@ -22,6 +22,7 @@ Individual stages: make -C ci lint # yaml/lint (no cluster; live now) make -C ci l0 # schema + Kyverno + L0 static conformance (no cluster; live now) make -C ci l1 # L1 rendering checks (no cluster; live now) +make -C ci prom-rules # promtool rule-unit tests on alerts (no cluster; live now) make -C ci conformance # L2/L3 in-cluster Chainsaw Job (SKIPS unless CLUSTER_AVAILABLE=true) make -C ci test # unit tests for these scripts themselves (ci/tests/*.test.sh) ``` @@ -80,13 +81,16 @@ in that case — nothing goes untested — and says plainly that the *scoped* ch ``` ci/ - Makefile targets: lint · l0 · l1 · conformance · gate · touched · test · tools · versions + Makefile targets: lint · l0 · l1 · prom-rules · conformance · gate · touched · test · tools · versions versions.env PINNED tool versions (R-TEST-02) — single source of truth lib.sh shared helpers: logging, tool checks, cluster detection, artifact paths yamllint.yaml lint config for the yaml/lint stage lint.sh Stage: yaml/lint l0.sh Stage: schema validation + Kyverno L0 gate + L0 static conformance l1.sh Stage: L1 rendering checks (Chainsaw L1 via the conformance runner) + prom-rules.sh Stage: `promtool test rules` over platform/monitoring's PrometheusRules + (harness: platform/monitoring/tests/). Catches the two alerting bugs no + other stage can see — permanently-firing and structurally INERT rules. conformance.sh Stage: L2/L3 in-cluster Chainsaw Job (trigger + collect) touched-requirements.sh compute the R-* subset a change touches (drives the R-TEST-03 filter); exits 3 LOUDLY rather than guessing when the diff base is unreliable diff --git a/ci/gate.sh b/ci/gate.sh index 8df27b4..d798cc7 100755 --- a/ci/gate.sh +++ b/ci/gate.sh @@ -42,6 +42,7 @@ run_stage lint "${CI_DIR}/lint.sh" run_stage l0 "${CI_DIR}/l0.sh" run_stage render-golden "${CI_DIR}/render-golden.sh" run_stage l1 "${CI_DIR}/l1.sh" +run_stage prom-rules "${CI_DIR}/prom-rules.sh" run_stage conformance "${CI_DIR}/conformance.sh" echo >&2 @@ -86,7 +87,7 @@ esac # Phase 0 that is tolerated (placeholders), but surfaced. render-golden is a # no-cluster reproducibility gate (canonical render == committed apps/ goldens): # a FAIL means a golden drifted from its ServiceDefinition source — always blocks. -for s in lint l0 render-golden l1; do +for s in lint l0 render-golden l1 prom-rules; do case "${OUTCOME[$s]}" in fail) err "${s}: FAILED -> blocks merge"; BLOCK=1 ;; skip) warn "${s}: skipped (unwired seam / missing tool) — not green" ;; diff --git a/ci/prom-rules.sh b/ci/prom-rules.sh new file mode 100755 index 0000000..a849645 --- /dev/null +++ b/ci/prom-rules.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# ci/prom-rules.sh — Stage: Prometheus RULE-UNIT tests (no cluster; live now). +# +# Thin wrapper over platform/monitoring/tests/run.sh, which runs `promtool test rules` against the +# COMMITTED PrometheusRule manifests. It is a gate stage because the two alerting defects this repo +# has actually shipped — a permanently-firing rule and a structurally INERT one — are both invisible +# to every other stage: yamllint parses them, Kyverno admits them, and a healthy cluster cannot tell +# a correct alert from one that can never fire. Only the bad state, constructed on purpose, can. +# +# Exit codes per ci/lib.sh: 0 pass / 1 fail / 3 skipped (never green). +set -euo pipefail +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${CI_DIR}/lib.sh" + +log "Stage: Prometheus rule-unit tests (promtool test rules; no cluster)" + +RUNNER="${REPO_ROOT}/platform/monitoring/tests/run.sh" +if [ ! -x "$RUNNER" ]; then + skip "rule-unit harness not present at ${RUNNER} — UNWIRED, not green" + summary "- prom-rules: **SKIPPED** (harness missing)" + exit 3 +fi + +set +e +"$RUNNER" +rc=$? +set -e + +case "$rc" in + 0) + ok "Prometheus rule-unit tests passed" + summary "- prom-rules: **passed**" + ;; + 3) + skip "promtool unavailable (no promtool on PATH, no podman/docker) — NOT green" + summary "- prom-rules: **SKIPPED** (promtool toolchain unavailable)" + ;; + *) + err "Prometheus rule-unit tests FAILED" + summary "- prom-rules: **FAILED**" + ;; +esac +exit "$rc" diff --git a/platform/monitoring/README.md b/platform/monitoring/README.md index 2e24805..7c98a73 100644 --- a/platform/monitoring/README.md +++ b/platform/monitoring/README.md @@ -18,6 +18,7 @@ | `loki.yaml` | `HelmRelease` — Loki single-binary, filesystem, compactor-enforced retention | | `alloy.yaml` | `HelmRelease` — Grafana Alloy DaemonSet, tails pod logs → Loki | | `alerts.yaml` | Platform-health `PrometheusRule`s (R-OBS-04) | +| `tests/` | `promtool test rules` unit tests over the committed `PrometheusRule`s — each rule pinned quiet on the good state and firing on the bad one (`tests/run.sh`, or `make -C ci prom-rules`) | | `kustomization.yaml` | builds all of the above | ## Requirement mapping diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index daa1eb5..af66d5e 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -378,6 +378,9 @@ spec: summary: "Velero backup partially failed" description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial" + # Covers "backed up before, then STOPPED". It is deliberately per-schedule and + # un-aggregated, but it is blind to a schedule that has never succeeded ONCE — see the + # block below, which closes that hole rather than complicating this expression. - alert: VeleroNoRecentBackup expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600) for: 1h @@ -388,6 +391,95 @@ spec: description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + # --- THE "NEVER BACKED UP" BLIND SPOT (closed 2026-09-13) ------------------------------ + # VeleroNoRecentBackup cannot fire for a schedule that has NEVER succeeded. Velero creates + # velero_backup_last_successful_timestamp{schedule="X"} only when a backup for X actually + # succeeds, so for a never-successful schedule the series is ABSENT and + # `time() - ` evaluates to no series at all. The rule detects "stopped + # backing up"; it is structurally inert on "never backed up". + # + # That is not hypothetical. Measured on prod 2026-09-12/13: `velero-daily` — the schedule + # that carries VOLUME DATA — sat `paused: true` with zero successful backups, so the cluster + # had NO volume backups and nothing alerted. It was doubly masked, because + # `velero-daily-resources-only` ran cleanly every 03:00: every other backup signal + # (failure_total, partial_failure_total, "last backup" in the UI) looked green while only + # Kubernetes OBJECTS were being protected. + # + # WHY NOT A PROMQL-ONLY FIX (verified, prod 2026-09-13, velero v1.18.1): the Velero metrics + # endpoint exports 35 velero_* metric names, and EVERY one carrying a `schedule` label is + # created by a backup ATTEMPT, not by the Schedule object. A throwaway Schedule created with + # `paused: true` produced ZERO velero_* series (`curl velero:8085/metrics | grep -c ` + # -> 0). There is no "schedule exists" metric and no `spec.paused` metric, so no expression + # over velero_* can see either failure mode. The Schedule CRs are therefore exported to + # Prometheus by kube-state-metrics' custom-resource-state collector as + # kube_velero_schedule_{created,paused,protects_volume_data,info} — configured in + # kube-prometheus-stack.yaml, which is the same repo-owned exporter already used for + # kube_node_labels, not a new sidecar. + # + # MASKING, and why the rules key on a PROPERTY rather than a schedule NAME: + # `kube_velero_schedule_protects_volume_data` is read straight from + # `spec.template.defaultVolumesToFsBackup`, which IS the definition of "this schedule backs + # up volume data". So VeleroNoVolumeDataBackup asks the operator's real question — "is + # anything at all protecting volume data right now?" — and a resources-only schedule can + # never answer it, whatever it is called. Same stance as platform/velero/volume-policies.yaml + # (test the property, not a name list a future object would slip past): renaming, adding or + # removing schedules cannot re-open the hole. + - alert: VeleroScheduleNeverBackedUp + expr: | + (time() - kube_velero_schedule_created) > (36 * 3600) + unless on (schedule) velero_backup_last_successful_timestamp{schedule!=""} + for: 1h + labels: + severity: critical + annotations: + summary: "Velero schedule has NEVER produced a successful backup: {{ $labels.schedule }}" + description: "Schedule {{ $labels.schedule }} has existed over 36h and Velero has never recorded a successful backup for it — the case VeleroNoRecentBackup is structurally blind to (prod ran with a paused velero-daily and NO volume backups this way). Check `kubectl -n velero get schedules.velero.io` for paused, and `backups.velero.io` (never the bare `backup` short name)." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + # The direct root-cause signal. A paused backup schedule is a legitimate but strictly + # TEMPORARY state; it must never be silently indefinite. Warning, not critical, because the + # outcome alerts above escalate to critical on their own if the pause outlasts a day — + # this one exists to name the cause while it is still cheap to fix. + - alert: VeleroSchedulePaused + expr: kube_velero_schedule_paused == 1 + for: 30m + labels: + severity: warning + annotations: + summary: "Velero schedule is PAUSED: {{ $labels.schedule }}" + description: "Schedule {{ $labels.schedule }} has spec.paused=true for over 30m and is creating no backups. Unpause it, or delete the Schedule if it is genuinely retired — an indefinitely paused schedule is a silent gap in R-STATE-03 off-cluster DR." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + # THE ANTI-MASKING RULE. Fires unless at least one schedule that actually captures volume + # data has succeeded in the last 36h, so a healthy resources-only schedule can never make + # the platform look protected. `or vector(0)` is load-bearing: `count() == 0` returns + # NO series and would itself be inert — exactly the bug being fixed — so the empty case is + # given an explicit 0. It also means this rule fires if kube-state-metrics stops exporting + # the Schedule CRs at all: ABSENCE OF THE SIGNAL IS THE ALARM, by design. + # Corollary, measured: evaluated against prod BEFORE the exporter exists this expression + # already returns `{} = 0`, i.e. it would fire. That is correct — there is genuinely no + # proof of volume-data protection without it — and the `for: 1h` is what keeps it from + # paging during the reconcile: the PrometheusRule and the kube-state-metrics values ship in + # the SAME Flux Kustomization (platform/monitoring), and the KSM rollout completes in + # minutes. If this ever pages an hour after a monitoring reconcile, the exporter did not + # come up — check `kubectl -n monitoring logs deploy/...-kube-state-metrics` for a + # `customresourcedefinitions ... is forbidden` RBAC error before suspecting Velero. + - alert: VeleroNoVolumeDataBackup + expr: | + ( + count( + (kube_velero_schedule_protects_volume_data == 1) + and on (schedule) + ((time() - velero_backup_last_successful_timestamp{schedule!=""}) < (36 * 3600)) + ) + or vector(0) + ) == 0 + for: 1h + labels: + severity: critical + annotations: + summary: "NO Velero schedule has backed up volume DATA in 36h" + description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + # ===================================================================================== # Identity — Keycloak (SSO is a critical dependency — docs/15) # ===================================================================================== diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index bd72b3b..da34233 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -232,6 +232,102 @@ spec: # `label_homelab_keithjackson_dev_egress_gateway` (`label_` + non-alphanumerics -> `_`). metricLabelsAllowlist: - "nodes=[homelab.keithjackson.dev/egress-gateway]" + # --- VELERO SCHEDULE OBJECTS AS METRICS (added 2026-09-13) --------------------------------- + # Velero exports NOTHING that enumerates Schedule objects and nothing that exposes + # `spec.paused`. Verified on prod against velero v1.18.1: all 35 velero_* metric names that + # carry a `schedule` label are created by a backup ATTEMPT, so a Schedule that has never run + # has NO series at all — a throwaway `paused: true` Schedule produced zero lines in + # `curl velero:8085/metrics`. Consequence: `VeleroNoRecentBackup` + # (`time() - velero_backup_last_successful_timestamp{schedule!=""}`) was structurally INERT for + # a never-successful schedule, because `time() - ` yields no series. prod sat + # with `velero-daily` (the VOLUME-DATA schedule) paused and unbacked-up, and nothing alerted. + # + # No PromQL can fix that — the fact is not in the TSDB — so the Schedule CRs themselves are + # exported here. KSM's custom-resource-state collector is the right mechanism precisely + # because it is ALREADY the platform's CRD-to-metrics path (this same block exports + # kube_node_labels for VPNTunnelDown); it needs no new exporter, image, ServiceMonitor or + # scrape job, and the series land on the existing kube-state-metrics target. + # + # Consumed by VeleroScheduleNeverBackedUp / VeleroSchedulePaused / VeleroNoVolumeDataBackup + # (platform/monitoring/alerts.yaml). Cost: 4 series per Schedule — 8 on prod today. + # + # Chart mechanics, verified by rendering kube-prometheus-stack 88.6.5 (KSM subchart 8.4.1, + # KSM v2.20.0) and by running that exact KSM image against prod's API: + # * `customResourceState.enabled` mounts this `config` as a ConfigMap at + # /etc/customresourcestate/config.yaml and adds --custom-resource-state-config-file. + # `--custom-resource-state-only` is NOT set, so the standard kube_* metrics keep flowing. + # * the chart auto-adds the apiextensions.k8s.io/customresourcedefinitions list+watch rule + # (rbac.customResourceState.createClusterRoleRules, default true). That rule is MANDATORY, + # not cosmetic: without it KSM logs "customresourcedefinitions ... is forbidden" and emits + # the CRS metrics as an EMPTY SET — silent, exactly the inert-alert failure mode again. + # Measured: the first probe run, with velero.io access but no CRD access, produced 0 series. + # * `rbac.extraRules` below is the only RBAC this repo must supply (read-only, one CRD). + # * `metricNamePrefix` deliberately yields `kube_velero_schedule_*`, NOT `velero_*`: these are + # kube-state-metrics series about Kubernetes objects, and must not be mistaken for + # something Velero itself exports. + # * `nilIsZero` matters — a Schedule that simply omits `spec.paused` (the chart renders + # `velero-daily-resources-only` that way) must still publish paused=0 rather than nothing, + # or `== 1` would be evaluated against a missing series. + # * the KSM Deployment carries NO checksum annotation over this ConfigMap, so a change to + # `config` alone does not restart the pod: after editing it, `kubectl -n monitoring rollout + # restart deploy/kube-prometheus-stack-kube-state-metrics`. Enabling it the first time + # changes the pod spec (new arg + volume), so the initial rollout is automatic. + rbac: + extraRules: + - apiGroups: ["velero.io"] + resources: ["schedules"] + verbs: ["get", "list", "watch"] + customResourceState: + enabled: true + config: + spec: + resources: + - groupVersionKind: + group: velero.io + version: v1 + kind: Schedule + metricNamePrefix: kube_velero_schedule + labelsFromPath: + # `schedule` (not `name`) so these join 1:1 with Velero's own `schedule` label. + schedule: [metadata, name] + namespace: [metadata, namespace] + metrics: + - name: created + help: "Unix creation time of the Velero Schedule object." + each: + type: Gauge + gauge: + # KSM parses the RFC3339 timestamp to a unix float (verified live). + # This is what gives VeleroScheduleNeverBackedUp a grace period, so a + # newly added schedule is not alerted before its first run is even due. + path: [metadata, creationTimestamp] + - name: paused + help: "1 if the Velero Schedule is paused and creating no backups." + each: + type: Gauge + gauge: + path: [spec, paused] + nilIsZero: true + - name: protects_volume_data + help: "1 if the Schedule captures pod volume DATA, not just Kubernetes objects." + each: + type: Gauge + gauge: + # THE anti-masking signal: read from the field that defines it, so a + # resources-only schedule can never stand in for a volume-data one. + path: [spec, template, defaultVolumesToFsBackup] + nilIsZero: true + - name: info + help: "Descriptive metadata for the Velero Schedule (cron, phase)." + each: + type: Info + info: + # NOTE: KSM OMITS a label whose path is absent (a Schedule with no + # status.phase yet yields no `phase` label at all), so label sets differ + # between series here. Context for dashboards — never join on this metric. + labelsFromPath: + cron: [spec, schedule] + phase: [status, phase] resources: requests: cpu: 20m diff --git a/platform/monitoring/tests/backups.test.yaml b/platform/monitoring/tests/backups.test.yaml new file mode 100644 index 0000000..7fbf8f2 --- /dev/null +++ b/platform/monitoring/tests/backups.test.yaml @@ -0,0 +1,357 @@ +# promtool rule-unit tests for the platform-health.backups group (platform/monitoring/alerts.yaml). +# Run with: platform/monitoring/tests/run.sh (or `make -C ci prom-rules`) +# +# Every rule is pinned in BOTH directions, because this repo has shipped both failure modes to +# prod: a permanently-firing alert (CNPG, 2026-09-12) and a structurally inert one +# (VeleroNoRecentBackup, which could not fire for a schedule that had never backed up — prod ran +# with NO volume backups and nothing alerted). +# +# CLOCK CONVENTION: promtool's test clock starts at the unix epoch, so `time()` == the eval_time in +# seconds. "This schedule has existed for 48h" is therefore expressed as a creationTimestamp of +# -172800 (48h BEFORE the epoch) rather than by running the test for two days. Same for +# last-successful-backup timestamps. Sample interval is 1m so nothing is ever stale inside +# Prometheus' 5m lookback. +rule_files: + - generated.rules.yaml + +evaluation_interval: 1m + +tests: + # =========================================================================================== + # 1. TODAY'S HEALTHY PROD STATE -> the whole group must be SILENT. + # This is the "not permanently firing" half of the proof, and it is also the test that + # catches a broken join: kube_velero_schedule_created is >36h old for both schedules, so + # VeleroScheduleNeverBackedUp's left-hand side is NON-EMPTY here and only the + # `unless on (schedule)` suppresses it. Mis-spell that label and this test goes red. + # =========================================================================================== + - interval: 1m + name: healthy prod state (both schedules succeeded recently) - everything quiet + input_series: + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-172800+0x180' + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-172800+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '1+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' + values: '0+0x180' + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' + values: '0+0x180' + # Velero also exports an un-scheduled (ad-hoc backup) series. Present on purpose: every rule + # must ignore it, or an ad-hoc backup would satisfy the schedule alerts. + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero"}' + values: '0+0x180' + alert_rule_test: + - eval_time: 2h + alertname: VeleroNoRecentBackup + exp_alerts: [] + - eval_time: 2h + alertname: VeleroScheduleNeverBackedUp + exp_alerts: [] + - eval_time: 2h + alertname: VeleroSchedulePaused + exp_alerts: [] + - eval_time: 2h + alertname: VeleroNoVolumeDataBackup + exp_alerts: [] + - eval_time: 2h + alertname: VeleroBackupFailure + exp_alerts: [] + - eval_time: 2h + alertname: VeleroBackupPartialFailure + exp_alerts: [] + promql_expr_test: + # The anti-masking counter really does count the volume-data schedule (a `1`, not the + # `or vector(0)` fallback) — so the rule is quiet for the right reason. + - expr: | + count( + (kube_velero_schedule_protects_volume_data == 1) + and on (schedule) + ((time() - velero_backup_last_successful_timestamp{schedule!=""}) < (36 * 3600)) + ) + or vector(0) + eval_time: 2h + exp_samples: + - labels: '{}' + value: 1 + + # =========================================================================================== + # 2. THE PROD DEFECT, RECONSTRUCTED (2026-09-12): `velero-daily` paused with no successful + # backup ever, while `velero-daily-resources-only` runs fine every 03:00 and masks it. + # Note what VeleroNoRecentBackup does here: NOTHING. That assertion is the regression test + # for the original bug — it documents that the old rule alone can never see this state. + # =========================================================================================== + - interval: 1m + name: velero-daily paused and NEVER backed up, resources-only healthy (the prod defect) + input_series: + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-172800+0x180' + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-172800+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '1+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '1+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + # NO velero_backup_last_successful_timestamp for velero-daily: it never succeeded, so Velero + # never created the series. That absence IS the bug. + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' + values: '0+0x180' + alert_rule_test: + # The old rule is blind to this state. Asserted, not assumed. + - eval_time: 2h + alertname: VeleroNoRecentBackup + exp_alerts: [] + # `for: 1h` is real: not yet firing at 30m. + - eval_time: 30m + alertname: VeleroScheduleNeverBackedUp + exp_alerts: [] + - eval_time: 2h + alertname: VeleroScheduleNeverBackedUp + exp_alerts: + - exp_labels: + severity: critical + job: kube-state-metrics + namespace: velero + schedule: velero-daily + customresource_group: velero.io + customresource_kind: Schedule + customresource_version: v1 + exp_annotations: + summary: "Velero schedule has NEVER produced a successful backup: velero-daily" + description: "Schedule velero-daily has existed over 36h and Velero has never recorded a successful backup for it — the case VeleroNoRecentBackup is structurally blind to (prod ran with a paused velero-daily and NO volume backups this way). Check `kubectl -n velero get schedules.velero.io` for paused, and `backups.velero.io` (never the bare `backup` short name)." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + # `for: 30m` is real: not yet firing at 20m. + - eval_time: 20m + alertname: VeleroSchedulePaused + exp_alerts: [] + - eval_time: 2h + alertname: VeleroSchedulePaused + exp_alerts: + - exp_labels: + severity: warning + job: kube-state-metrics + namespace: velero + schedule: velero-daily + customresource_group: velero.io + customresource_kind: Schedule + customresource_version: v1 + exp_annotations: + summary: "Velero schedule is PAUSED: velero-daily" + description: "Schedule velero-daily has spec.paused=true for over 30m and is creating no backups. Unpause it, or delete the Schedule if it is genuinely retired — an indefinitely paused schedule is a silent gap in R-STATE-03 off-cluster DR." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + - eval_time: 2h + alertname: VeleroNoVolumeDataBackup + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "NO Velero schedule has backed up volume DATA in 36h" + description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + promql_expr_test: + # The `or vector(0)` fallback is load-bearing: without it `count() == 0` returns no + # series and the rule would be inert exactly when it matters. + - expr: | + count( + (kube_velero_schedule_protects_volume_data == 1) + and on (schedule) + ((time() - velero_backup_last_successful_timestamp{schedule!=""}) < (36 * 3600)) + ) + or vector(0) + eval_time: 2h + exp_samples: + - labels: '{}' + value: 0 + # And the old expression genuinely produces nothing for a never-successful schedule. + - expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600) + eval_time: 2h + exp_samples: [] + + # =========================================================================================== + # 3. NEVER BACKED UP BUT **NOT** PAUSED — a wedged schedule controller, a cron that never + # comes due, a storage location that rejects every backup before one ever succeeds. + # Proves VeleroScheduleNeverBackedUp does not secretly depend on `paused`. + # =========================================================================================== + - interval: 1m + name: schedule enabled but has never succeeded - still alerts, paused rule stays quiet + input_series: + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-172800+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '1+0x180' + alert_rule_test: + - eval_time: 2h + alertname: VeleroScheduleNeverBackedUp + exp_alerts: + - exp_labels: + severity: critical + job: kube-state-metrics + namespace: velero + schedule: velero-daily + customresource_group: velero.io + customresource_kind: Schedule + customresource_version: v1 + exp_annotations: + summary: "Velero schedule has NEVER produced a successful backup: velero-daily" + description: "Schedule velero-daily has existed over 36h and Velero has never recorded a successful backup for it — the case VeleroNoRecentBackup is structurally blind to (prod ran with a paused velero-daily and NO volume backups this way). Check `kubectl -n velero get schedules.velero.io` for paused, and `backups.velero.io` (never the bare `backup` short name)." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + - eval_time: 2h + alertname: VeleroSchedulePaused + exp_alerts: [] + + # =========================================================================================== + # 4. GRACE PERIOD — a Schedule added 12h ago whose first run is not due yet must NOT alert. + # This is the "not permanently firing" guard for the new rule: without the + # kube_velero_schedule_created term it would page on every newly added schedule. + # =========================================================================================== + - interval: 1m + name: freshly created schedule inside its 36h grace window - no alert + input_series: + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-weekly-new",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-43200+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-weekly-new",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-weekly-new",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + # An established, healthy volume-data schedule alongside it, so the anti-masking rule is + # quiet and this test isolates the grace period. + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-172800+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '1+0x180' + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' + values: '0+0x180' + alert_rule_test: + - eval_time: 2h + alertname: VeleroScheduleNeverBackedUp + exp_alerts: [] + - eval_time: 2h + alertname: VeleroNoVolumeDataBackup + exp_alerts: [] + - eval_time: 2h + alertname: VeleroSchedulePaused + exp_alerts: [] + + # =========================================================================================== + # 5. BACKED UP ONCE, THEN STOPPED (48h stale) — the case the ORIGINAL rule was written for. + # Kept so the old rule is proven live too, and so the new rule is proven NOT to + # double-page: the success series exists, so `unless` suppresses VeleroScheduleNeverBackedUp. + # =========================================================================================== + - interval: 1m + name: volume-data schedule last succeeded 48h ago - stale alerts fire, never-backed-up does not + input_series: + - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '-864000+0x180' + - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '0+0x180' + - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' + values: '1+0x180' + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' + values: '-172800+0x180' + alert_rule_test: + - eval_time: 2h + alertname: VeleroNoRecentBackup + exp_alerts: + - exp_labels: + severity: critical + job: velero + namespace: velero + schedule: velero-daily + exp_annotations: + summary: "No successful Velero backup in 24h: velero-daily" + description: "Schedule velero-daily has not produced a successful backup in over 24h (missed schedule)." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + - eval_time: 2h + alertname: VeleroScheduleNeverBackedUp + exp_alerts: [] + - eval_time: 2h + alertname: VeleroNoVolumeDataBackup + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "NO Velero schedule has backed up volume DATA in 36h" + description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + + # =========================================================================================== + # 6. THE EXPORTER ITSELF DISAPPEARS — kube-state-metrics stops publishing the Schedule CRs + # (removed values, broken RBAC, KSM down) while Velero's own metrics still look fine. + # The two rules that need the exporter go quiet, which is unavoidable; VeleroNoVolumeDataBackup + # FIRES instead. Asserted deliberately: absence of the signal must be the alarm, so this + # family can never regress to "silent because the input vanished" a second time. + # =========================================================================================== + - interval: 1m + name: kube-state-metrics schedule metrics absent - absence itself alerts + input_series: + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' + values: '0+0x180' + - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' + values: '0+0x180' + alert_rule_test: + - eval_time: 2h + alertname: VeleroNoVolumeDataBackup + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "NO Velero schedule has backed up volume DATA in 36h" + description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" + - eval_time: 2h + alertname: VeleroScheduleNeverBackedUp + exp_alerts: [] + - eval_time: 2h + alertname: VeleroSchedulePaused + exp_alerts: [] + + # =========================================================================================== + # 7. The two pre-existing counter rules, so the whole group is covered both ways rather than + # just the parts that were touched. + # =========================================================================================== + - interval: 1m + name: backup failures increment - failure rules fire + input_series: + - series: 'velero_backup_failure_total{job="velero",namespace="velero",schedule="velero-daily"}' + values: '0+0x100 1+0x80' + - series: 'velero_backup_partial_failure_total{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' + values: '0+0x100 1+0x80' + alert_rule_test: + - eval_time: 2h + alertname: VeleroBackupFailure + exp_alerts: + - exp_labels: + severity: critical + job: velero + namespace: velero + schedule: velero-daily + exp_annotations: + summary: "Velero backup failed: schedule velero-daily" + description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-failure" + - eval_time: 2h + alertname: VeleroBackupPartialFailure + exp_alerts: + - exp_labels: + severity: warning + job: velero + namespace: velero + schedule: velero-daily-resources-only + exp_annotations: + summary: "Velero backup partially failed" + description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped." + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial" diff --git a/platform/monitoring/tests/extract_rules.py b/platform/monitoring/tests/extract_rules.py new file mode 100755 index 0000000..c527a18 --- /dev/null +++ b/platform/monitoring/tests/extract_rules.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Flatten this repo's PrometheusRule CRs into a plain Prometheus rule file. + +`promtool test rules` consumes a rule file (`{groups: [...]}`), not a Kubernetes +PrometheusRule object, so the committed manifests cannot be fed to it directly. This +script is the (deliberately dumb) adapter: it walks the given YAML files, picks out every +`kind: PrometheusRule` document, and concatenates their `spec.groups` into one rule file. + +Why an adapter instead of a second copy of the rules: a rule-unit test is only worth +anything if it tests the EXACT expression that ships. Copying expressions into a fixture +would let the fixture stay green while the deployed rule rots -- which is the class of bug +this harness exists to catch (a permanently-firing CNPG alert and a structurally inert +Velero alert both reached prod in 2026-09). + +Flux `postBuild` placeholders (`${BASE_DOMAIN}`) are left verbatim: no expression in +platform/monitoring uses one, and leaving them alone keeps annotation text identical to +the file under test. If a placeholder ever appears inside an `expr`, that is refused +loudly rather than silently producing an untestable rule file. + +Group names are made unique per source file so two files may legally reuse a name. + +Usage: + extract_rules.py --out generated.rules.yaml alerts.yaml [more.yaml ...] +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover - reported by the runner as a SKIP, never a pass + print( + "extract_rules.py: PyYAML is required (it ships transitively with the pinned " + "yamllint used by ci/lint.sh: pip install 'yamllint==1.35.1')", + file=sys.stderr, + ) + raise SystemExit(4) + +PLACEHOLDER = re.compile(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}") + + +def extract(paths: list[Path]) -> dict: + groups: list[dict] = [] + seen: set[str] = set() + for path in paths: + docs = [d for d in yaml.safe_load_all(path.read_text()) if d] + rules_docs = [d for d in docs if d.get("kind") == "PrometheusRule"] + if not rules_docs: + raise SystemExit(f"{path}: contains no PrometheusRule documents") + for doc in rules_docs: + for group in doc.get("spec", {}).get("groups", []) or []: + for rule in group.get("rules", []) or []: + expr = str(rule.get("expr", "")) + if PLACEHOLDER.search(expr): + raise SystemExit( + f"{path}: rule {rule.get('alert') or rule.get('record')!r} has a " + f"postBuild placeholder inside its expr; it cannot be unit-tested " + f"as written" + ) + # Namespace the group by source file so names cannot collide. + group = dict(group) + name = f"{path.stem}::{group['name']}" + if name in seen: + raise SystemExit(f"duplicate rule group {name!r}") + seen.add(name) + group["name"] = name + groups.append(group) + if not groups: + raise SystemExit("no rule groups found") + return {"groups": groups} + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--out", required=True, type=Path, help="rule file to write") + ap.add_argument("manifests", nargs="+", type=Path, help="PrometheusRule YAML files") + args = ap.parse_args() + + out = extract(args.manifests) + args.out.write_text(yaml.safe_dump(out, sort_keys=False, default_flow_style=False)) + n = sum(len(g.get("rules", []) or []) for g in out["groups"]) + print( + f"extract_rules.py: {len(out['groups'])} group(s), {n} rule(s) -> {args.out}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/platform/monitoring/tests/run.sh b/platform/monitoring/tests/run.sh new file mode 100755 index 0000000..a024b9d --- /dev/null +++ b/platform/monitoring/tests/run.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# platform/monitoring/tests/run.sh — Prometheus RULE-UNIT-TEST harness (no cluster). +# +# WHY THIS EXISTS +# This repo has now shipped both halves of the alerting failure space to prod: +# * a PERMANENTLY-FIRING rule (the CNPG alert fixed 2026-09-12), and +# * a STRUCTURALLY INERT rule (VeleroNoRecentBackup: `time() - ` yields no +# series, so a schedule that had NEVER backed up could not raise it — prod ran with no +# volume backups at all and nothing alerted). +# Neither is visible by reading the expression, by `kubectl apply`, or by looking at a healthy +# live cluster: an inert rule and a correct rule are indistinguishable when everything is fine. +# `promtool test rules` is the tool that separates them, because it lets the BAD state be +# constructed on purpose and asserted on, offline and in CI. Every rule here is therefore +# pinned twice: quiet on the good state, firing on the bad one. +# +# The rules under test are the COMMITTED manifests (extract_rules.py flattens the +# PrometheusRule CRs into a rule file) — never a copy, so a fixture cannot drift green while +# the deployed rule rots. +# +# Usage: ./run.sh # run every *.test.yaml here +# ./run.sh foo.test.yaml +# +# Exit codes follow ci/lib.sh: 0 pass, 1 fail, 3 SKIPPED (toolchain unavailable — never green). +# promtool is taken from $PATH if present, otherwise run from the pinned Prometheus image via +# podman/docker. The image tag matches the version prod runs, so "passes here" means "passes +# with the engine that will evaluate it". +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONITORING_DIR="$(cd "${HERE}/.." && pwd)" +PROM_IMAGE="${PROM_IMAGE:-quay.io/prometheus/prometheus:v3.14.0}" + +MANIFESTS=( + "${MONITORING_DIR}/alerts.yaml" + "${MONITORING_DIR}/pvc-capacity-alerts.yaml" +) + +if [ "$#" -gt 0 ]; then + TESTS=("$@") +else + mapfile -t TESTS < <(find "${HERE}" -maxdepth 1 -name '*.test.yaml' | sort) +fi +if [ "${#TESTS[@]}" -eq 0 ]; then + echo "run.sh: no *.test.yaml found in ${HERE}" >&2 + exit 3 +fi + +# ---------------------------------------------------------------- toolchain resolution +RUNNER="" +if command -v promtool >/dev/null 2>&1; then + RUNNER="native" +elif command -v podman >/dev/null 2>&1; then + RUNNER="podman" +elif command -v docker >/dev/null 2>&1; then + RUNNER="docker" +else + echo "run.sh: SKIP — no promtool on PATH and no podman/docker to run ${PROM_IMAGE}" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# The test files reference the generated rule file by bare name, so everything must sit in one +# directory (that is also the only directory the container needs mounted). +rc=0 +python3 "${HERE}/extract_rules.py" --out "${WORK}/generated.rules.yaml" "${MANIFESTS[@]}" || rc=$? +if [ "$rc" -eq 4 ]; then + echo "run.sh: SKIP — PyYAML unavailable, cannot flatten the PrometheusRule manifests" >&2 + exit 3 +elif [ "$rc" -ne 0 ]; then + exit "$rc" +fi + +names=() +for t in "${TESTS[@]}"; do + cp "$t" "${WORK}/" + names+=("$(basename "$t")") +done +chmod -R a+rX "${WORK}" + +echo "run.sh: promtool test rules (${RUNNER}) ${names[*]}" >&2 +case "$RUNNER" in + native) + ( cd "${WORK}" && promtool test rules "${names[@]}" ) ;; + podman) + podman run --rm --user 0 -v "${WORK}:/rules:z" -w /rules \ + --entrypoint /bin/promtool "${PROM_IMAGE}" test rules "${names[@]}" ;; + docker) + docker run --rm --user 0 -v "${WORK}:/rules" -w /rules \ + --entrypoint /bin/promtool "${PROM_IMAGE}" test rules "${names[@]}" ;; +esac