# promtool rule-unit tests for the wave-1 per-app `-down` SLI rules. # Manifests under test: every PrometheusRule in apps/wave-1/ — nextcloud, radarr, semaphore, sonarr # (container/Deployment) and keycloak (StatefulSet, see below). Discovered by the SUITE convention; # the flattened manifests are the RENDERED ones Flux applies. # # Run with: platform/monitoring/tests/run.sh apps/wave-1/tests/app-slis.test.yaml # # See apps/wave-0/tests/app-slis.test.yaml for the full history: these rules all read # `up{job=""} == 0` until 2026-09-13 against a job no scrape target produced, and were # structurally inert on prod. Each is now pinned QUIET on the healthy state, FIRING on no-ready- # replica, and FIRING when the signal is ABSENT. # # KEYCLOAK IS THE INTERESTING ONE. Its server is a StatefulSet run by the Keycloak operator, NOT the # Deployment this ServiceDefinition would otherwise imply — so its rule reads # kube_statefulset_status_replicas_ready (catalog/keycloak.yaml declares # `observability.livenessWorkload: {kind: StatefulSet, name: keycloak}`). Had it been left to the # default, the `absent()` half would have matched forever and the alert would have been PERMANENTLY # FIRING, which is the other failure mode this repo has shipped to prod. Verified on prod # 2026-09-13: kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"} # = 2, and the only Deployment in that namespace is keycloak-operator. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # =========================================================================================== # 1. HEALTHY WAVE-1 -> every rule SILENT ("not permanently firing"). # =========================================================================================== - interval: 1m name: wave-1 healthy - every app ready, keycloak StatefulSet at 2 replicas - all quiet input_series: - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}' values: '2+0x20' - series: 'kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}' values: '1+0x20' # The namespace's OTHER workloads are present on purpose: keycloak-operator is a Deployment in # the keycloak namespace, and keycloak-db is a StatefulSet in it. A rule that matched the # namespace only (not the object) would be satisfied — or triggered — by these instead of by # the IdP itself. That is the exact masking KeycloakDown had to be fixed for. - series: 'kube_deployment_status_replicas_available{namespace="keycloak", deployment="keycloak-operator"}' values: '1+0x20' - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak-db"}' values: '3+0x20' alert_rule_test: - eval_time: 10m alertname: keycloak-down exp_alerts: [] - eval_time: 10m alertname: nextcloud-down exp_alerts: [] - eval_time: 10m alertname: radarr-down exp_alerts: [] - eval_time: 10m alertname: semaphore-down exp_alerts: [] - eval_time: 10m alertname: sonarr-down exp_alerts: [] # =========================================================================================== # 2. THE IdP ITSELF IS DOWN while its operator and DB are fine -> ONLY keycloak-down fires. # This is the anti-masking assertion: the neighbours in that namespace stay ready. # =========================================================================================== - interval: 1m name: wave-1 keycloak StatefulSet has no ready replica (operator + DB fine) - keycloak-down fires input_series: - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="keycloak", deployment="keycloak-operator"}' values: '1+0x20' - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak-db"}' values: '3+0x20' - series: 'kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}' values: '1+0x20' alert_rule_test: - eval_time: 10m alertname: keycloak-down exp_alerts: - exp_labels: severity: critical service: keycloak namespace: keycloak statefulset: keycloak exp_annotations: summary: "keycloak is down" description: "keycloak: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)." - eval_time: 10m alertname: nextcloud-down exp_alerts: [] - eval_time: 10m alertname: radarr-down exp_alerts: [] - eval_time: 10m alertname: semaphore-down exp_alerts: [] - eval_time: 10m alertname: sonarr-down exp_alerts: [] # =========================================================================================== # 3. EACH DEPLOYMENT-BACKED APP DOWN -> its own rule fires, keycloak stays quiet. # =========================================================================================== - interval: 1m name: wave-1 nextcloud/radarr/semaphore/sonarr have no available replica - each own rule fires input_series: - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}' values: '2+0x20' - series: 'kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}' values: '0+0x20' alert_rule_test: - eval_time: 10m alertname: nextcloud-down exp_alerts: - exp_labels: severity: critical service: nextcloud namespace: nextcloud deployment: nextcloud exp_annotations: summary: "nextcloud is down" description: "nextcloud: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: radarr-down exp_alerts: - exp_labels: severity: critical service: radarr namespace: radarr deployment: radarr exp_annotations: summary: "radarr is down" description: "radarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: semaphore-down exp_alerts: - exp_labels: severity: critical service: semaphore namespace: semaphore deployment: semaphore exp_annotations: summary: "semaphore is down" description: "semaphore: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: sonarr-down exp_alerts: - exp_labels: severity: critical service: sonarr namespace: sonarr deployment: sonarr exp_annotations: summary: "sonarr is down" description: "sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: keycloak-down exp_alerts: [] # =========================================================================================== # 4. THE SIGNAL VANISHES -> the rules FIRE. The defect class itself, asserted on: a rule that can # only fire on `== 0` goes SILENT when kube-state-metrics or the workload object disappears, # which is indistinguishable from healthy. Here NOTHING in wave-1 is scraped. # =========================================================================================== - interval: 1m name: wave-1 all liveness series ABSENT (kube-state-metrics gone) - every rule fires input_series: # One unrelated series so the test has an input at all; no wave-1 liveness series exists. - series: 'up{job="kube-state-metrics"}' values: '0+0x20' alert_rule_test: - eval_time: 10m alertname: keycloak-down exp_alerts: - exp_labels: severity: critical service: keycloak namespace: keycloak statefulset: keycloak exp_annotations: summary: "keycloak is down" description: "keycloak: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)." - eval_time: 10m alertname: nextcloud-down exp_alerts: - exp_labels: severity: critical service: nextcloud namespace: nextcloud deployment: nextcloud exp_annotations: summary: "nextcloud is down" description: "nextcloud: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: radarr-down exp_alerts: - exp_labels: severity: critical service: radarr namespace: radarr deployment: radarr exp_annotations: summary: "radarr is down" description: "radarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: semaphore-down exp_alerts: - exp_labels: severity: critical service: semaphore namespace: semaphore deployment: semaphore exp_annotations: summary: "semaphore is down" description: "semaphore: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." - eval_time: 10m alertname: sonarr-down exp_alerts: - exp_labels: severity: critical service: sonarr namespace: sonarr deployment: sonarr exp_annotations: summary: "sonarr is down" description: "sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)."