# promtool rule-unit tests for the wave-0 per-app `-down` SLI rules. # Manifests under test: every PrometheusRule in apps/wave-0/ — gitea, jackett, pihole, plex, # qbittorrent (container/Deployment), pihole (container/StatefulSet — `availability.replicas: 2`, so # its SLI reads kube_statefulset_status_replicas_ready; see schemas/renderer-contract.md # `### availability`) and vm-demo (KubeVirt VMI). The harness discovers this file by # the SUITE convention (`/tests/*.test.yaml`) and flattens those RENDERED manifests, so these # tests run against the exact expressions Flux applies, never a copy. # # Run with: platform/monitoring/tests/run.sh apps/wave-0/tests/app-slis.test.yaml # (the merge gate runs the same harness via # conformance/tests/l0-static/alert-rules-unit-tested) # # WHY THESE EXIST. Until 2026-09-13 every one of these rules read `up{job=""} == 0` while the # rendered ServiceMonitor named a Service port (`metrics`) that no rendered Service defined. The # monitor discovered ZERO targets, so no `up{job=""}` series existed and the rule could not # fire — on prod, `count by (job) (up)` listed 24 jobs and not one of the twelve appeared. # `PrometheusTargetsMissing` could not see it either: a target that was never discovered has no `up` # series to be 0. The rules now read kube-state-metrics / kubevirt series that DO exist for these # workloads (verified on prod), and every rule below is pinned in BOTH directions: # * QUIET on the healthy state -> it is not permanently firing; # * FIRING when the workload has no ready replica -> it is not inert; # * FIRING when the signal is ABSENT -> the failure mode that produced this whole defect class # (a rule that goes quiet because its series vanished) is itself alerted on. # # CLOCK/`for` CONVENTION: sample interval 1m, series 21 samples long, evaluated at 10m — comfortably # past every rule's `for: 5m` and well inside Prometheus' 5m lookback. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # =========================================================================================== # 1. HEALTHY WAVE-0 (what prod looks like right now) -> every rule SILENT. # This is the "not permanently firing" half. It is also the test that catches a mis-spelled # label: get `deployment=` wrong and the `== 0` half matches nothing while the `absent()` half # matches everything, so this test goes red instead of the rule going quietly wrong. # =========================================================================================== - interval: 1m name: wave-0 healthy - every app has a ready replica, the demo VM runs - all quiet input_series: - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' values: '1+0x20' - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}' values: '1+0x20' - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running", node="k3s-prod-1"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: gitea-down exp_alerts: [] - eval_time: 10m alertname: jackett-down exp_alerts: [] - eval_time: 10m alertname: pihole-down exp_alerts: [] - eval_time: 10m alertname: plex-down exp_alerts: [] - eval_time: 10m alertname: qbittorrent-down exp_alerts: [] - eval_time: 10m alertname: vm-demo-down exp_alerts: [] # =========================================================================================== # 2. EACH APP DOWN IN TURN (0 available replicas) -> ONLY that app's rule fires. # The per-app isolation matters: a rule that forgot its `namespace=`/`deployment=` matchers # would light up every app at once and be useless for routing. # =========================================================================================== - interval: 1m name: wave-0 plex has no available replica - only plex-down fires input_series: - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' values: '1+0x20' - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}' values: '1+0x20' - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: plex-down exp_alerts: - exp_labels: severity: critical service: plex namespace: plex deployment: plex exp_annotations: summary: "plex is down" description: "plex: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: gitea-down exp_alerts: [] - eval_time: 10m alertname: jackett-down exp_alerts: [] - eval_time: 10m alertname: pihole-down exp_alerts: [] - eval_time: 10m alertname: qbittorrent-down exp_alerts: [] - eval_time: 10m alertname: vm-demo-down exp_alerts: [] # Before `for: 5m` has elapsed the rule must be PENDING, not firing — proof the 5m is real and # a single scrape blip does not page anyone. promql_expr_test: - expr: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"} == 0 or absent(kube_deployment_status_replicas_available{namespace="plex", deployment="plex"})' eval_time: 2m exp_samples: - labels: 'kube_deployment_status_replicas_available{deployment="plex", namespace="plex"}' value: 0 - interval: 1m name: wave-0 gitea/jackett/pihole/qbittorrent each down - each own rule fires input_series: - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' values: '0+0x20' - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}' values: '0+0x20' - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: gitea-down exp_alerts: - exp_labels: severity: critical service: gitea namespace: gitea deployment: gitea exp_annotations: summary: "gitea is down" description: "gitea: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: jackett-down exp_alerts: - exp_labels: severity: critical service: jackett namespace: jackett deployment: jackett exp_annotations: summary: "jackett is down" description: "jackett: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: pihole-down exp_alerts: - exp_labels: severity: critical service: pihole namespace: pihole statefulset: pihole exp_annotations: summary: "pihole is down" description: "pihole: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)." - eval_time: 10m alertname: qbittorrent-down exp_alerts: - exp_labels: severity: critical service: qbittorrent namespace: qbittorrent deployment: qbittorrent exp_annotations: summary: "qbittorrent is down" description: "qbittorrent: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: plex-down exp_alerts: [] - eval_time: 10m alertname: vm-demo-down exp_alerts: [] # =========================================================================================== # 3. THE SIGNAL VANISHES -> the rule FIRES. This is the defect class itself, asserted on. # The old `up{job=...} == 0` rule had exactly this shape of hole and was silent for it. Here # the Deployment series for plex is simply not in the input at all, and plex-down must still # fire (via the `absent()` half), with the labels the selector's equality matchers carry. # =========================================================================================== - interval: 1m name: wave-0 plex Deployment series ABSENT (never scraped / workload gone) - plex-down fires input_series: - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' values: '1+0x20' - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: plex-down exp_alerts: - exp_labels: severity: critical service: plex namespace: plex deployment: plex exp_annotations: summary: "plex is down" description: "plex: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: gitea-down exp_alerts: [] # =========================================================================================== # 4. THE VM: a VMI carries no replica count, so liveness is the EXISTENCE of a running-phase # series. Two ways to be down, both asserted: the VMI is gone entirely, and the VMI exists in # some OTHER phase. The second is the one a naive `kubevirt_vmi_info == 1` would miss, and it # is why the phase value is matched — LOWERCASE `running`, the value KubeVirt actually writes # (a capital-R matcher selects nothing and would be just as inert). # =========================================================================================== - interval: 1m name: wave-0 demo VM has no running-phase VMI - vm-demo-down fires input_series: - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: vm-demo-down exp_alerts: - exp_labels: severity: critical service: vm-demo namespace: vm-demo name: vm-demo phase: running exp_annotations: summary: "vm-demo is down" description: "vm-demo: no KubeVirt VMI in phase running for 5m (SLI: up, via kubevirt_vmi_info)." - interval: 1m name: wave-0 demo VM exists but is Failed, not running - vm-demo-down still fires input_series: - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="failed"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: vm-demo-down exp_alerts: - exp_labels: severity: critical service: vm-demo namespace: vm-demo name: vm-demo phase: running exp_annotations: summary: "vm-demo is down" description: "vm-demo: no KubeVirt VMI in phase running for 5m (SLI: up, via kubevirt_vmi_info)."