From 7c4eb439b2ad6bfa22620994b13d373aaaa8f720 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Thu, 29 Oct 2026 21:41:15 -0500 Subject: [PATCH] feat(monitoring): alert when the keycloak passkey reconciler's Job fails KeycloakPasskeyReconcilerFailing (warning, for 15m) on kube_job_status_failed{namespace="keycloak", job_name=~"keycloak-passkey-reconciler-.*"}, following the sibling client-secret reconciler's rule. promtool unit tests pin it in both directions: quiet when succeeding, firing after 15m on a failed Job, quiet on the SIBLING's failure, and quiet on a single transient failure. Runbook added and projected by the runbooks ConfigMap generator. Co-Authored-By: Claude Opus 5.5 --- platform/monitoring/alerts.yaml | 30 +++++++ .../keycloak-passkey-reconciler.test.yaml | 90 +++++++++++++++++++ runbooks/README.md | 2 + runbooks/keycloak-passkey-reconciler.md | 69 ++++++++++++++ runbooks/kustomization.yaml | 1 + 5 files changed, 192 insertions(+) create mode 100644 platform/monitoring/tests/keycloak-passkey-reconciler.test.yaml create mode 100644 runbooks/keycloak-passkey-reconciler.md diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index e007f2d..d56bad8 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -1445,6 +1445,36 @@ spec: redirect-URI drift would go uncorrected and unnoticed (docs/15). This is the rule that would catch the healer being quietly absent during a domain migration. runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-client-secrets" + # --- The passkey reconciler (platform/keycloak/passkey-reconciler.yaml) --- + # The sibling CronJob that converges the WebAuthn passwordless policy, the passkey + # enrolment required action, and the browser login flow (blue/green; bound only when + # git's bind-browser-flow is "true"). There is no failure that is "expected by + # design": with the bind switch off, an unbound verified candidate is a SUCCESSFUL + # run. So a failed Job is always news — drift it may not repair (e.g. a missing + # CONFIGURE_TOTP, which locks every platform-admin out), a built flow that did not + # verify, an exhausted candidate-alias budget, or an admin API it cannot reach. + # backoffLimit is 0, so one failed tick is one failed Job; `for: 15m` waits out a + # single transient failure while still firing within about two ticks. + # A separate rule from the client reconciler's on purpose: different CronJob, + # different job_name, different runbook — one cause, one alert. + # UNVERIFIED-ON-CLUSTER at authoring time: the CronJob had never run, so its series + # did not exist yet. The metric and label are the ones the sibling rule above uses. + - alert: KeycloakPasskeyReconcilerFailing + expr: | + max(kube_job_status_failed{namespace="keycloak", job_name=~"keycloak-passkey-reconciler-.*"}) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Keycloak passkey reconciler failing" + description: >- + A keycloak-passkey-reconciler Job has been failing for 15m. The realm's passkey + policy, the passkey enrolment action or the browser login flow may differ from + git, or a default required action (CONFIGURE_TOTP) may be missing. The reconciler + never writes the flow users are logging in through, so a failure here does NOT + by itself mean login is broken. Read the Job log: every PROBLEM line names the + pass and the exact divergence. + runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-passkey-reconciler" # ===================================================================================== # The monitor's own capacity — don't let the monitor blind itself (docs/10) diff --git a/platform/monitoring/tests/keycloak-passkey-reconciler.test.yaml b/platform/monitoring/tests/keycloak-passkey-reconciler.test.yaml new file mode 100644 index 0000000..6bb3a43 --- /dev/null +++ b/platform/monitoring/tests/keycloak-passkey-reconciler.test.yaml @@ -0,0 +1,90 @@ +# promtool rule-unit tests for KeycloakPasskeyReconcilerFailing (platform/monitoring/alerts.yaml, +# group platform-health.identity). +# Run with: platform/monitoring/tests/run.sh (the merge gate runs the same harness via +# conformance/tests/l0-static/alert-rules-unit-tested) +# +# WHAT THE RULE CLAIMS, PINNED IN BOTH DIRECTIONS. The passkey reconciler +# (platform/keycloak/passkey-reconciler.yaml) runs every 15 minutes with backoffLimit 0, so one +# failed tick is one Job with kube_job_status_failed = 1, kept for an hour (ttlSecondsAfterFinished) +# and then garbage-collected. Its steady state is GREEN whether or not git lets it bind the browser +# flow, so a failed Job is always news. What can go wrong with the rule: +# * it never fires — a job_name regex that misses the CronJob's Job names (`-`); +# * it fires on the SIBLING — the client-secret reconciler lives in the same namespace, and its +# failures already have their own alert and runbook; one cause, one alert; +# * it fires on a single transient failure that the next tick cleared — `for: 15m` exists for that. +# Fixture names are prod's real shape: CronJob Jobs are named -. +rule_files: + - generated.rules.yaml + +evaluation_interval: 1m + +tests: + # ============================================================================================= + # 1. HEALTHY: every passkey Job succeeded -> SILENT. The series exist with failed = 0, so a rule + # that forgot the `> 0` would fire here. + # ============================================================================================= + - interval: 1m + name: passkey reconciler succeeding - quiet + input_series: + - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889000"}' + values: '0+0x60' + - series: 'kube_job_status_succeeded{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889000"}' + values: '1+0x60' + alert_rule_test: + - eval_time: 60m + alertname: KeycloakPasskeyReconcilerFailing + exp_alerts: [] + + # ============================================================================================= + # 2. THE FAILURE: a passkey Job failed and stays failed -> FIRES after 15m (and not before). + # ============================================================================================= + - interval: 1m + name: passkey reconciler Job failed - fires after 15m + input_series: + - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889015"}' + values: '1+0x60' + alert_rule_test: + - eval_time: 10m + alertname: KeycloakPasskeyReconcilerFailing + exp_alerts: [] + - eval_time: 20m + alertname: KeycloakPasskeyReconcilerFailing + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + summary: "Keycloak passkey reconciler failing" + description: "A keycloak-passkey-reconciler Job has been failing for 15m. The realm's passkey policy, the passkey enrolment action or the browser login flow may differ from git, or a default required action (CONFIGURE_TOTP) may be missing. The reconciler never writes the flow users are logging in through, so a failure here does NOT by itself mean login is broken. Read the Job log: every PROBLEM line names the pass and the exact divergence." + runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-passkey-reconciler" + + # ============================================================================================= + # 3. THE SIBLING'S FAILURE IS NOT THIS ALERT. The client-secret reconciler's Job fails in the + # same namespace; KeycloakClientSecretReconcilerFailing owns it. This one must stay quiet. + # ============================================================================================= + - interval: 1m + name: only the client-secret reconciler failing - passkey alert quiet + input_series: + - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-client-secret-reconciler-29889015"}' + values: '1+0x60' + - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889015"}' + values: '0+0x60' + alert_rule_test: + - eval_time: 60m + alertname: KeycloakPasskeyReconcilerFailing + exp_alerts: [] + + # ============================================================================================= + # 4. ONE TRANSIENT FAILURE, cleared by the next tick: the failed Job is garbage-collected (its + # series ends) after 5m here and the next Job succeeds -> never held for 15m -> QUIET. + # ============================================================================================= + - interval: 1m + name: single transient failure cleared within 15m - quiet + input_series: + - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889030"}' + values: '1+0x5' + - series: 'kube_job_status_failed{job="kube-state-metrics",namespace="keycloak",job_name="keycloak-passkey-reconciler-29889045"}' + values: '_x15 0+0x45' + alert_rule_test: + - eval_time: 30m + alertname: KeycloakPasskeyReconcilerFailing + exp_alerts: [] diff --git a/runbooks/README.md b/runbooks/README.md index 1b9493c..070f485 100644 --- a/runbooks/README.md +++ b/runbooks/README.md @@ -141,6 +141,8 @@ Each affected runbook says so on its own page and gives the by-hand equivalent. - **`keycloak-db`** — Keycloak's CloudNativePG database is unavailable. - **`keycloak-client-secrets`** — the client/URL reconciler failing or silent. The 2026-09-06 lockout that every green signal missed. +- **`keycloak-passkey-reconciler`** — the passkey policy / enrolment action / browser-flow reconciler + failing. Never writes the flow you log in through; a missing `CONFIGURE_TOTP` is the cause that bites. - **`oauth2-proxy-auth-failures`** — a gated app "is up but does not work". Linked from `OAuth2ProxyGateUnobserved`. The gate answers an unauthenticated **XHR** with a 302 to Keycloak rather than a 401, the browser follows it silently against a live SSO session, and the call diff --git a/runbooks/keycloak-passkey-reconciler.md b/runbooks/keycloak-passkey-reconciler.md new file mode 100644 index 0000000..0f4f051 --- /dev/null +++ b/runbooks/keycloak-passkey-reconciler.md @@ -0,0 +1,69 @@ +# keycloak-passkey-reconciler — the passkey reconciler's Job is failing + +**Alert:** `KeycloakPasskeyReconcilerFailing` (warning, 15m) +**File:** `platform/monitoring/alerts.yaml`, group `platform-health.identity` +**Expression:** +``` +max(kube_job_status_failed{namespace="keycloak", job_name=~"keycloak-passkey-reconciler-.*"}) > 0 +``` + +## What this means here + +`platform/keycloak/passkey-reconciler.yaml` is a CronJob (every 15 minutes, `backoffLimit: 0`) that +converges three things the one-shot `KeycloakRealmImport` can never carry to the realm once it rests +at `Done=True`: + +1. the realm's **WebAuthn passwordless policy** (`webAuthnPolicyPasswordless*`); +2. the **`webauthn-register-passwordless`** required action (enabled, *not* a default action); +3. the **browser login flow**, blue/green: it builds the declared tree under a new alias + (`browser-mfa-`), verifies it by readback, and **binds it only if git says so** — + `bind-browser-flow: "true"` in `platform/keycloak/passkey-desired-state.yaml`. With `"false"` (the + default) a run that builds and verifies a candidate but does not bind it is a **success**. + +So this alert never means "the bind switch is off". It means a run reported a PROBLEM. The reconciler +is structurally unable to write the flow users are currently logging in through, so **a failing Job +does not by itself mean login is broken** — but one cause it reports does lock people out: a missing +`CONFIGURE_TOTP` required action (see below). + +## First commands + +Every command runs in the pinned toolbox — kubectl is not on the workstation: + +```sh +./infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n keycloak get cronjob keycloak-passkey-reconciler' +./infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n keycloak get jobs -l app.kubernetes.io/name=keycloak-passkey-reconciler' +# The log of the most recent run. Every failure is a line starting `PROBLEM`; nothing secret is printed. +./infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n keycloak logs -l app.kubernetes.io/name=keycloak-passkey-reconciler --tail=300' +# What git currently asks for (the bind switch): +grep -n 'bind-browser-flow:' platform/keycloak/passkey-desired-state.yaml +``` + +## Likely causes, most likely first + +1. **`could not find an unused alias ... after 5 attempt(s)`** — earlier builds left half-built + flows under every candidate alias (`browser-mfa-`, `-r2` … `-r5`). Nothing here deletes, by + design. Delete the **unbound** `browser-mfa-*` flows named in the log in the admin console + (Authentication → Flows), never the one the realm is bound to. The next tick rebuilds. +2. **`was built but does NOT match git, so it was NOT BOUND`** — the admin API accepted the build + but the readback differs (the log names the step). Login is unaffected. If it names the + conditional-OTP config (`condUserRole`), that is the regression that once 403'd every + platform-admin, caught before it went live. +3. **`MISSING the default required action(s) ... CONFIGURE_TOTP`** — someone declared a + `requiredActions` block in the realm import (26.0.6 treats it as REPLACE). This one IS a live + lock-out for platform-admins: re-register the action in the admin console + (Authentication → Required actions). The reconciler cannot repair it. +4. **`Keycloak admin API not usable` / `FATAL`** — Keycloak is down (check `KeycloakDown` first), the + `keycloak-admin-credentials` Secret changed, or the desired-state ConfigMap is malformed + (`bind-browser-flow` must be the quoted string `"true"` or `"false"`). +5. **`FAIL-CLOSED`** — the bound flow's executions could not be read, so nothing was built. Usually + transient; if persistent, the realm is bound to a flow that no longer exists. + +## When to escalate or stop + +- **Never** fix this by editing the bound flow in the admin console while `bind-browser-flow` is + `"true"`: the next run will build a fresh copy and rebind it. Flip the switch to `"false"` in a PR + first; see `platform/keycloak/README.md`, "Passkeys", for the rollback procedure. +- If people **cannot log in**, this alert is not the cause to chase — start at `keycloak-down` and + roll the browser flow back (PR setting `bind-browser-flow: "false"`, then rebind the previous + alias printed in the Job log). +- Do not `kubectl edit` the CronJob or the ConfigMaps; Flux reverts it. Every change is a PR. diff --git a/runbooks/kustomization.yaml b/runbooks/kustomization.yaml index c6deba9..e260cc1 100644 --- a/runbooks/kustomization.yaml +++ b/runbooks/kustomization.yaml @@ -77,6 +77,7 @@ configMapGenerator: - image-automation=image-automation.md - image-policy-stalled=image-policy-stalled.md - keycloak-client-secrets=keycloak-client-secrets.md + - keycloak-passkey-reconciler=keycloak-passkey-reconciler.md - keycloak-db=keycloak-db.md - keycloak-down=keycloak-down.md - kubevirt-control-plane=kubevirt-control-plane.md