# promtool rule-unit tests for KeycloakPasskeyReconcilerFailing (platform/monitoring/alerts.yaml, # group platform-health.identity). # Run with: platform/monitoring/tests/run.sh (the merge gate runs the same harness via # conformance/tests/l0-static/alert-rules-unit-tested) # # WHAT THE RULE CLAIMS, PINNED IN BOTH DIRECTIONS. The passkey reconciler # (platform/keycloak/passkey-reconciler.yaml) runs every 15 minutes with backoffLimit 0, so one # failed tick is one Job with kube_job_status_failed = 1, kept for an hour (ttlSecondsAfterFinished) # and then garbage-collected. Its steady state is GREEN whether or not git lets it bind the browser # flow, so a failed Job is always news. What can go wrong with the rule: # * it never fires — a job_name regex that misses the CronJob's Job names (`-`); # * it fires on the SIBLING — the client-secret reconciler lives in the same namespace, and its # failures already have their own alert and runbook; one cause, one alert; # * it fires on a single transient failure that the next tick cleared — `for: 15m` exists for that. # Fixture names are prod's real shape: CronJob Jobs are named -. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # ============================================================================================= # 1. HEALTHY: every passkey Job succeeded -> SILENT. The series exist with failed = 0, so a rule # that forgot the `> 0` would fire here. # ============================================================================================= - interval: 1m name: passkey reconciler succeeding - quiet input_series: - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889000"}' values: '0+0x60' - series: 'kube_job_status_succeeded{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889000"}' values: '1+0x60' alert_rule_test: - eval_time: 60m alertname: KeycloakPasskeyReconcilerFailing exp_alerts: [] # ============================================================================================= # 2. THE FAILURE: a passkey Job failed and stays failed -> FIRES after 15m (and not before). # ============================================================================================= - interval: 1m name: passkey reconciler Job failed - fires after 15m input_series: - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889015"}' values: '1+0x60' alert_rule_test: - eval_time: 10m alertname: KeycloakPasskeyReconcilerFailing exp_alerts: [] - eval_time: 20m alertname: KeycloakPasskeyReconcilerFailing exp_alerts: - exp_labels: severity: warning exp_annotations: summary: "Keycloak passkey reconciler failing" description: "A keycloak-passkey-reconciler Job has been failing for 15m. The realm's passkey policy, the passkey enrolment action or the browser login flow may differ from git, or a default required action (CONFIGURE_TOTP) may be missing. The reconciler never writes the flow users are logging in through, so a failure here does NOT by itself mean login is broken. Read the Job log: every PROBLEM line names the pass and the exact divergence." runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-passkey-reconciler" # ============================================================================================= # 3. THE SIBLING'S FAILURE IS NOT THIS ALERT. The client-secret reconciler's Job fails in the # same namespace; KeycloakClientSecretReconcilerFailing owns it. This one must stay quiet. # ============================================================================================= - interval: 1m name: only the client-secret reconciler failing - passkey alert quiet input_series: - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-client-secret-reconciler-29889015"}' values: '1+0x60' - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889015"}' values: '0+0x60' alert_rule_test: - eval_time: 60m alertname: KeycloakPasskeyReconcilerFailing exp_alerts: [] # ============================================================================================= # 4. ONE TRANSIENT FAILURE, cleared by the next tick: the failed Job is garbage-collected (its # series ends) after 5m here and the next Job succeeds -> never held for 15m -> QUIET. # ============================================================================================= - interval: 1m name: single transient failure cleared within 15m - quiet input_series: - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889030"}' values: '1+0x5' - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889045"}' values: '_x15 0+0x45' alert_rule_test: - eval_time: 30m alertname: KeycloakPasskeyReconcilerFailing exp_alerts: []