# promtool rule-unit tests for the platform-health.backups group (platform/monitoring/alerts.yaml). # Run with: platform/monitoring/tests/run.sh (the merge gate runs the same harness via # conformance/tests/l0-static/alert-rules-unit-tested) # # Every rule is pinned in BOTH directions, because this repo has shipped both failure modes to # prod: a permanently-firing alert (CNPG, 2026-09-12) and a structurally inert one # (VeleroNoRecentBackup, which could not fire for a schedule that had never backed up — prod ran # with NO volume backups and nothing alerted). # # CLOCK CONVENTION: promtool's test clock starts at the unix epoch, so `time()` == the eval_time in # seconds. "This schedule has existed for 48h" is therefore expressed as a creationTimestamp of # -172800 (48h BEFORE the epoch) rather than by running the test for two days. Same for # last-successful-backup timestamps. Sample interval is 1m so nothing is ever stale inside # Prometheus' 5m lookback. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # =========================================================================================== # 1. TODAY'S HEALTHY PROD STATE -> the whole group must be SILENT. # This is the "not permanently firing" half of the proof, and it is also the test that # catches a broken join: kube_velero_schedule_created is >36h old for both schedules, so # VeleroScheduleNeverBackedUp's left-hand side is NON-EMPTY here and only the # `unless on (schedule)` suppresses it. Mis-spell that label and this test goes red. # =========================================================================================== - interval: 1m name: healthy prod state (both schedules succeeded recently) - everything quiet input_series: - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-172800+0x180' - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-172800+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '1+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' values: '0+0x180' - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' values: '0+0x180' # Velero also exports an un-scheduled (ad-hoc backup) series. Present on purpose: every rule # must ignore it, or an ad-hoc backup would satisfy the schedule alerts. - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero"}' values: '0+0x180' alert_rule_test: - eval_time: 2h alertname: VeleroNoRecentBackup exp_alerts: [] - eval_time: 2h alertname: VeleroScheduleNeverBackedUp exp_alerts: [] - eval_time: 2h alertname: VeleroSchedulePaused exp_alerts: [] - eval_time: 2h alertname: VeleroNoVolumeDataBackup exp_alerts: [] - eval_time: 2h alertname: VeleroBackupFailure exp_alerts: [] - eval_time: 2h alertname: VeleroBackupPartialFailure exp_alerts: [] promql_expr_test: # The anti-masking counter really does count the volume-data schedule (a `1`, not the # `or vector(0)` fallback) — so the rule is quiet for the right reason. - expr: | count( (kube_velero_schedule_protects_volume_data == 1) and on (schedule) ((time() - velero_backup_last_successful_timestamp{schedule!=""}) < (36 * 3600)) ) or vector(0) eval_time: 2h exp_samples: - labels: '{}' value: 1 # =========================================================================================== # 2. THE PROD DEFECT, RECONSTRUCTED (2026-09-12): `velero-daily` paused with no successful # backup ever, while `velero-daily-resources-only` runs fine every 03:00 and masks it. # Note what VeleroNoRecentBackup does here: NOTHING. That assertion is the regression test # for the original bug — it documents that the old rule alone can never see this state. # =========================================================================================== - interval: 1m name: velero-daily paused and NEVER backed up, resources-only healthy (the prod defect) input_series: - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-172800+0x180' - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-172800+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '1+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '1+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily-resources-only",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' # NO velero_backup_last_successful_timestamp for velero-daily: it never succeeded, so Velero # never created the series. That absence IS the bug. - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' values: '0+0x180' alert_rule_test: # The old rule is blind to this state. Asserted, not assumed. - eval_time: 2h alertname: VeleroNoRecentBackup exp_alerts: [] # `for: 1h` is real: not yet firing at 30m. - eval_time: 30m alertname: VeleroScheduleNeverBackedUp exp_alerts: [] - eval_time: 2h alertname: VeleroScheduleNeverBackedUp exp_alerts: - exp_labels: severity: critical job: kube-state-metrics namespace: velero schedule: velero-daily customresource_group: velero.io customresource_kind: Schedule customresource_version: v1 exp_annotations: summary: "Velero schedule has NEVER produced a successful backup: velero-daily" description: "Schedule velero-daily has existed over 36h and Velero has never recorded a successful backup for it — the case VeleroNoRecentBackup is structurally blind to (prod ran with a paused velero-daily and NO volume backups this way). Check `kubectl -n velero get schedules.velero.io` for paused, and `backups.velero.io` (never the bare `backup` short name)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # `for: 30m` is real: not yet firing at 20m. - eval_time: 20m alertname: VeleroSchedulePaused exp_alerts: [] - eval_time: 2h alertname: VeleroSchedulePaused exp_alerts: - exp_labels: severity: warning job: kube-state-metrics namespace: velero schedule: velero-daily customresource_group: velero.io customresource_kind: Schedule customresource_version: v1 exp_annotations: summary: "Velero schedule is PAUSED: velero-daily" description: "Schedule velero-daily has spec.paused=true for over 30m and is creating no backups. Unpause it, or delete the Schedule if it is genuinely retired — an indefinitely paused schedule is a silent gap in R-STATE-03 off-cluster DR." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" - eval_time: 2h alertname: VeleroNoVolumeDataBackup exp_alerts: - exp_labels: severity: critical exp_annotations: summary: "NO Velero schedule has backed up volume DATA in 36h" description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" promql_expr_test: # The `or vector(0)` fallback is load-bearing: without it `count() == 0` returns no # series and the rule would be inert exactly when it matters. - expr: | count( (kube_velero_schedule_protects_volume_data == 1) and on (schedule) ((time() - velero_backup_last_successful_timestamp{schedule!=""}) < (36 * 3600)) ) or vector(0) eval_time: 2h exp_samples: - labels: '{}' value: 0 # And the old expression genuinely produces nothing for a never-successful schedule. - expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600) eval_time: 2h exp_samples: [] # =========================================================================================== # 3. NEVER BACKED UP BUT **NOT** PAUSED — a wedged schedule controller, a cron that never # comes due, a storage location that rejects every backup before one ever succeeds. # Proves VeleroScheduleNeverBackedUp does not secretly depend on `paused`. # =========================================================================================== - interval: 1m name: schedule enabled but has never succeeded - still alerts, paused rule stays quiet input_series: - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-172800+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '1+0x180' alert_rule_test: - eval_time: 2h alertname: VeleroScheduleNeverBackedUp exp_alerts: - exp_labels: severity: critical job: kube-state-metrics namespace: velero schedule: velero-daily customresource_group: velero.io customresource_kind: Schedule customresource_version: v1 exp_annotations: summary: "Velero schedule has NEVER produced a successful backup: velero-daily" description: "Schedule velero-daily has existed over 36h and Velero has never recorded a successful backup for it — the case VeleroNoRecentBackup is structurally blind to (prod ran with a paused velero-daily and NO volume backups this way). Check `kubectl -n velero get schedules.velero.io` for paused, and `backups.velero.io` (never the bare `backup` short name)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" - eval_time: 2h alertname: VeleroSchedulePaused exp_alerts: [] # =========================================================================================== # 4. GRACE PERIOD — a Schedule added 12h ago whose first run is not due yet must NOT alert. # This is the "not permanently firing" guard for the new rule: without the # kube_velero_schedule_created term it would page on every newly added schedule. # =========================================================================================== - interval: 1m name: freshly created schedule inside its 36h grace window - no alert input_series: - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-weekly-new",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-43200+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-weekly-new",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-weekly-new",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' # An established, healthy volume-data schedule alongside it, so the anti-masking rule is # quiet and this test isolates the grace period. - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-172800+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '1+0x180' - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' values: '0+0x180' alert_rule_test: - eval_time: 2h alertname: VeleroScheduleNeverBackedUp exp_alerts: [] - eval_time: 2h alertname: VeleroNoVolumeDataBackup exp_alerts: [] - eval_time: 2h alertname: VeleroSchedulePaused exp_alerts: [] # =========================================================================================== # 5. BACKED UP ONCE, THEN STOPPED (48h stale) — the case the ORIGINAL rule was written for. # Kept so the old rule is proven live too, and so the new rule is proven NOT to # double-page: the success series exists, so `unless` suppresses VeleroScheduleNeverBackedUp. # =========================================================================================== - interval: 1m name: volume-data schedule last succeeded 48h ago - stale alerts fire, never-backed-up does not input_series: - series: 'kube_velero_schedule_created{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '-864000+0x180' - series: 'kube_velero_schedule_paused{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '0+0x180' - series: 'kube_velero_schedule_protects_volume_data{job="kube-state-metrics",namespace="velero",schedule="velero-daily",customresource_group="velero.io",customresource_kind="Schedule",customresource_version="v1"}' values: '1+0x180' - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' values: '-172800+0x180' alert_rule_test: - eval_time: 2h alertname: VeleroNoRecentBackup exp_alerts: - exp_labels: severity: critical job: velero namespace: velero schedule: velero-daily exp_annotations: summary: "No successful Velero backup in 24h: velero-daily" description: "Schedule velero-daily has not produced a successful backup in over 24h (missed schedule)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" - eval_time: 2h alertname: VeleroScheduleNeverBackedUp exp_alerts: [] - eval_time: 2h alertname: VeleroNoVolumeDataBackup exp_alerts: - exp_labels: severity: critical exp_annotations: summary: "NO Velero schedule has backed up volume DATA in 36h" description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # =========================================================================================== # 6. THE EXPORTER ITSELF DISAPPEARS — kube-state-metrics stops publishing the Schedule CRs # (removed values, broken RBAC, KSM down) while Velero's own metrics still look fine. # The two rules that need the exporter go quiet, which is unavoidable; VeleroNoVolumeDataBackup # FIRES instead. Asserted deliberately: absence of the signal must be the alarm, so this # family can never regress to "silent because the input vanished" a second time. # =========================================================================================== - interval: 1m name: kube-state-metrics schedule metrics absent - absence itself alerts input_series: - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily"}' values: '0+0x180' - series: 'velero_backup_last_successful_timestamp{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' values: '0+0x180' alert_rule_test: - eval_time: 2h alertname: VeleroNoVolumeDataBackup exp_alerts: - exp_labels: severity: critical exp_annotations: summary: "NO Velero schedule has backed up volume DATA in 36h" description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" - eval_time: 2h alertname: VeleroScheduleNeverBackedUp exp_alerts: [] - eval_time: 2h alertname: VeleroSchedulePaused exp_alerts: [] # =========================================================================================== # 7. The two pre-existing counter rules, so the whole group is covered both ways rather than # just the parts that were touched. # =========================================================================================== - interval: 1m name: backup failures increment - failure rules fire input_series: - series: 'velero_backup_failure_total{job="velero",namespace="velero",schedule="velero-daily"}' values: '0+0x100 1+0x80' - series: 'velero_backup_partial_failure_total{job="velero",namespace="velero",schedule="velero-daily-resources-only"}' values: '0+0x100 1+0x80' alert_rule_test: - eval_time: 2h alertname: VeleroBackupFailure exp_alerts: - exp_labels: severity: critical job: velero namespace: velero schedule: velero-daily exp_annotations: summary: "Velero backup failed: schedule velero-daily" description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-failure" - eval_time: 2h alertname: VeleroBackupPartialFailure exp_alerts: - exp_labels: severity: warning job: velero namespace: velero schedule: velero-daily-resources-only exp_annotations: summary: "Velero backup partially failed" description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial"