diff --git a/ci/render-keycloak-passkey-policy.py b/ci/render-keycloak-passkey-policy.py new file mode 100644 index 0000000..b99c50a --- /dev/null +++ b/ci/render-keycloak-passkey-policy.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +# ============================================================================= +# ci/render-keycloak-passkey-policy.py +# +# Render platform/keycloak/passkey-desired-state.yaml FROM +# platform/keycloak/keycloak-realm-import.yaml. +# +# WHY THIS EXISTS +# --------------- +# The Keycloak realm import is a ONE-SHOT that fails OPEN: once it rests at +# Done=True / Started=False it never re-runs. Measured on prod 2026-10-18, hours +# after the passkey declaration merged: +# +# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath=...status +# Done=True Started=False HasErrors=False +# +# So the WebAuthn passwordless policy and the restructured `browser-mfa` flow that +# keycloak-realm-import.yaml declares are INERT. They are in git, Flux applied +# them, and they reach the realm never. platform/keycloak/passkey-reconciler.yaml +# is the level-triggered loop that closes that; this renderer supplies the DESIRED +# STATE it converges towards (and, for the flow, audits against). +# +# THE CONSTRAINT THAT SHAPES IT: the reconciler pod runs with +# automountServiceAccountToken: false, exactly like its client-secret sibling. It +# has NO Kubernetes API access at all, by design (R-SEC-02) — so it cannot read the +# KeycloakRealmImport CR to learn what the realm is supposed to look like. Its input +# must be MOUNTED. Hence a ConfigMap. +# +# THAT IS ALSO WHY THIS IS A RENDERER AND NOT A HAND-WRITTEN ConfigMap. A +# hand-authored copy of the policy would be a SECOND SOURCE OF TRUTH for the RP ID, +# the user-verification requirement and the whole flow tree — free to disagree with +# the realm import. "Two declarations that can disagree" is the same shape as the +# bug this whole effort is closing. So the ConfigMap is GENERATED from the realm +# import and the generation is GATED: +# conformance/tests/l0-static/keycloak-passkey-reconciler/check.sh runs this script +# with --check on every merge, and the merge fails if the committed file is not what +# this renderer produces. Same render==golden idiom as +# ci/render-keycloak-client-urls.py, catalog/render.py and ci/render-golden.sh. +# +# STRINGS ARE COPIED VERBATIM, NEVER RE-DERIVED. webAuthnPolicyPasswordlessRpId is +# written in the realm import as the Flux substitution token ${BASE_DOMAIN}, and it +# is carried through into the ConfigMap CHARACTER FOR CHARACTER. Both files are +# rendered by the SAME Flux Kustomization (./platform-config, whose +# postBuild.substituteFrom reads cluster-vars), so both are substituted in the same +# pass with the same value and cannot disagree about what the base domain is. This +# renderer never parses, splits or reconstructs a hostname and never reads +# cluster-vars — there is no second place the domain is decided. +# +# THE ONE THING THIS RENDERER DERIVES RATHER THAN COPIES +# ----------------------------------------------------- +# The enrolment required action, `webauthn-register-passwordless`. It is +# deliberately NOT declared in the realm import, and the import's own header +# explains at length why declaring it there would be actively DANGEROUS: Keycloak +# 26.0.6 treats `requiredActions` in a realm representation as REPLACE, not merge, +# so declaring one action imports exactly that action plus delete_account and skips +# DefaultRequiredActions.addActions entirely — deleting CONFIGURE_TOTP, +# UPDATE_PASSWORD, VERIFY_EMAIL and seven more. Losing CONFIGURE_TOTP is a hard +# lock-out of every platform-admin, because auth-otp-form in the browser flow still +# demands a TOTP they could then never enrol. +# +# So the required action cannot be copied from the import, because it is correctly +# absent from it. It is instead DERIVED, as a consequence of what the import DOES +# declare: +# +# IF the declared browser flow contains a `webauthn-authenticator-passwordless` +# execution, THEN passkey logins are offered, THEREFORE the enrolment action must +# be registered and enabled — and NOT a defaultAction, because the import's whole +# stated posture is "additive and optional" and a defaultAction forces enrolment +# on every user at next login. +# +# This is a derivation, not a second declaration: there is no value here that a +# human could edit into disagreement with the import. If the import stops declaring +# the passkey execution, this renderer stops emitting the required action (and +# fails, because then the whole desired state is empty of passkey intent). +# +# WHAT IS DELIBERATELY NOT EXPORTED +# * the TWO-FACTOR webAuthnPolicy* family. Nothing in this realm uses the +# `webauthn-authenticator` two-factor authenticator; the passwordless +# authenticator reads getWebAuthnPolicyPasswordless() only, so a policy written +# into the two-factor family is a silent no-op. Exporting it would hand the +# reconciler a write surface with no purpose. +# * `clients`, `users`, `groups`, `roles`. Client secrets and client URLs are the +# SIBLING reconciler's surface (platform/keycloak/client-secret-reconciler.yaml), +# and users/groups/roles are nobody's — prod's real user and the media-users / +# platform-admins / viewers groups must survive every run of everything. +# * execution `priority`. It is consumed here to ORDER the exported executions +# (Keycloak returns them in priority order), but the numbers themselves are not +# exported: the reconciler compares the resulting SEQUENCE, which is what +# actually determines behaviour. Exporting the integers would make a +# priority renumbering that changes no ordering read as drift. +# +# USAGE +# python3 ci/render-keycloak-passkey-policy.py # print the rendered file +# python3 ci/render-keycloak-passkey-policy.py --write # write it (regenerate) +# python3 ci/render-keycloak-passkey-policy.py --check # exit 1 if it has drifted +# ============================================================================= +from __future__ import annotations + +import argparse +import difflib +import json +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + print("FATAL: PyYAML is required (same dependency as catalog/render.py)", + file=sys.stderr) + raise SystemExit(1) + +ROOT = Path(__file__).resolve().parent.parent +IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml" +OUT_FILE = "platform/keycloak/passkey-desired-state.yaml" +DATA_KEY = "passkey-desired-state.json" +CONFIGMAP_NAME = "keycloak-passkey-desired-state" +NAMESPACE = "keycloak" + +# The realm keys this renderer exports, and therefore the ONLY realm keys the +# reconciler may write. Every one is a webAuthnPolicyPasswordless* field. Kept in +# lockstep with POLICY_FIELDS in the reconciler script; the L0 check asserts the two +# tuples are identical, so they cannot drift apart silently. +POLICY_STR_FIELDS = ( + "webAuthnPolicyPasswordlessRpId", + "webAuthnPolicyPasswordlessRpEntityName", + "webAuthnPolicyPasswordlessRequireResidentKey", + "webAuthnPolicyPasswordlessUserVerificationRequirement", + "webAuthnPolicyPasswordlessAuthenticatorAttachment", + "webAuthnPolicyPasswordlessAttestationConveyancePreference", +) +POLICY_LIST_FIELDS = ( + "webAuthnPolicyPasswordlessSignatureAlgorithms", + "webAuthnPolicyPasswordlessAcceptableAaguids", +) +POLICY_INT_FIELDS = ("webAuthnPolicyPasswordlessCreateTimeout",) +POLICY_BOOL_FIELDS = ("webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister",) +POLICY_FIELDS = (POLICY_STR_FIELDS + POLICY_LIST_FIELDS + POLICY_INT_FIELDS + + POLICY_BOOL_FIELDS) + +# The field Keycloak 26.0.6 gates the WHOLE passwordless policy on +# (DefaultExportImportManager.updateRealm: the policy is applied only when +# RpEntityName is non-null and non-empty). A desired state without it would produce +# an accepted PUT that changes nothing, so refuse to render one. +RP_ENTITY_NAME = "webAuthnPolicyPasswordlessRpEntityName" + +# The execution whose presence is what makes passkey enrolment necessary. +PASSKEY_AUTHENTICATOR = "webauthn-authenticator-passwordless" +ENROL_ACTION = "webauthn-register-passwordless" + + +def render(root: Path) -> str: + """Return the full text of passkey-desired-state.yaml as it should be committed.""" + docs = list(yaml.safe_load_all((root / IMPORT_FILE).read_text())) + imp = next((d for d in docs if d and d.get("kind") == "KeycloakRealmImport"), None) + if imp is None: + raise SystemExit("FATAL: no KeycloakRealmImport document in %s" % IMPORT_FILE) + realm = (imp.get("spec") or {}).get("realm") or {} + realm_name = realm.get("realm") + if not isinstance(realm_name, str) or not realm_name: + raise SystemExit("FATAL: %s declares no spec.realm.realm name" % IMPORT_FILE) + + # ---- the passwordless policy, copied VERBATIM ---------------------------------- + policy: dict[str, object] = {} + for f in POLICY_STR_FIELDS: + if f not in realm: + continue # not declared => NOT MANAGED + v = realm[f] + # A bare ${BASE_DOMAIN} parses as a plain string; an int or bool here would be + # a YAML quoting mistake that must not be silently stringified. + if not isinstance(v, str): + raise SystemExit("FATAL: %s realm field %s is not a string (got %r) — " + "quote it in the realm import" % (IMPORT_FILE, f, v)) + policy[f] = v + for f in POLICY_LIST_FIELDS: + if f not in realm: + continue + v = realm[f] + if not isinstance(v, list) or not all(isinstance(x, str) for x in v): + raise SystemExit("FATAL: %s realm field %s is not a list of strings" + % (IMPORT_FILE, f)) + policy[f] = list(v) # verbatim, order as authored + for f in POLICY_INT_FIELDS: + if f not in realm: + continue + v = realm[f] + if not isinstance(v, int) or isinstance(v, bool): + raise SystemExit("FATAL: %s realm field %s is not an integer" + % (IMPORT_FILE, f)) + policy[f] = v + for f in POLICY_BOOL_FIELDS: + if f not in realm: + continue + v = realm[f] + if not isinstance(v, bool): + raise SystemExit("FATAL: %s realm field %s is not a boolean" + % (IMPORT_FILE, f)) + policy[f] = v + + if not policy: + # Fail closed. An empty policy would make the reconciler believe every field is + # unmanaged and silently stop reconciling — the realm import's own fail-open. + raise SystemExit("FATAL: %s declares no webAuthnPolicyPasswordless* field — " + "refusing to render an empty desired state" % IMPORT_FILE) + if not policy.get(RP_ENTITY_NAME): + raise SystemExit( + "FATAL: %s does not declare a non-empty %s. Keycloak 26.0.6 applies the " + "passwordless policy ONLY when that field is set, so every realm PUT would " + "be accepted and change nothing" % (IMPORT_FILE, RP_ENTITY_NAME)) + + # ---- the flow tree, flattened per flow and ORDERED BY PRIORITY ----------------- + browser_flow = realm.get("browserFlow") + if not isinstance(browser_flow, str) or not browser_flow: + raise SystemExit("FATAL: %s declares no spec.realm.browserFlow — the flow could " + "be perfect and still not be the one anyone logs in through" + % IMPORT_FILE) + raw_flows = realm.get("authenticationFlows") or [] + if not isinstance(raw_flows, list) or not raw_flows: + raise SystemExit("FATAL: %s declares no authenticationFlows" % IMPORT_FILE) + + flows: dict[str, object] = {} + for fl in raw_flows: + alias = fl.get("alias") + if not isinstance(alias, str) or not alias: + raise SystemExit("FATAL: %s has an authentication flow with no alias" + % IMPORT_FILE) + if alias in flows: + raise SystemExit("FATAL: %s declares the flow %r twice" + % (IMPORT_FILE, alias)) + execs = fl.get("authenticationExecutions") or [] + if not isinstance(execs, list) or not execs: + raise SystemExit("FATAL: %s flow %r declares no authenticationExecutions" + % (IMPORT_FILE, alias)) + ordered = [] + for ex in sorted(execs, key=lambda e: (e.get("priority") or 0)): + is_flow = bool(ex.get("authenticatorFlow")) + name = ex.get("flowAlias") if is_flow else ex.get("authenticator") + if not isinstance(name, str) or not name: + raise SystemExit( + "FATAL: %s flow %r has an execution with neither an authenticator " + "nor a flowAlias" % (IMPORT_FILE, alias)) + requirement = ex.get("requirement") + if not isinstance(requirement, str) or not requirement: + raise SystemExit("FATAL: %s flow %r execution %r declares no requirement" + % (IMPORT_FILE, alias, name)) + entry: dict[str, object] = { + "kind": "flow" if is_flow else "authenticator", + "name": name, + "requirement": requirement, + } + cfg = ex.get("authenticatorConfig") + if cfg is not None: + if not isinstance(cfg, str) or not cfg: + raise SystemExit("FATAL: %s flow %r execution %r has a non-string " + "authenticatorConfig" % (IMPORT_FILE, alias, name)) + entry["configAlias"] = cfg + ordered.append(entry) + flows[alias] = {"topLevel": bool(fl.get("topLevel")), "executions": ordered} + + if browser_flow not in flows: + raise SystemExit("FATAL: %s binds browserFlow to %r but declares no such flow" + % (IMPORT_FILE, browser_flow)) + # Every referenced subflow must be declared, or the reconciler's flattening walks + # into a hole and the audit silently covers less than it claims. + for alias, spec in flows.items(): + for ex in spec["executions"]: + if ex["kind"] == "flow" and ex["name"] not in flows: + raise SystemExit("FATAL: %s flow %r references subflow %r, which is not " + "declared" % (IMPORT_FILE, alias, ex["name"])) + + # ---- authenticator configs, copied verbatim ------------------------------------ + configs: dict[str, dict] = {} + for cfg in realm.get("authenticatorConfig") or []: + alias = cfg.get("alias") + if not isinstance(alias, str) or not alias: + raise SystemExit("FATAL: %s has an authenticatorConfig with no alias" + % IMPORT_FILE) + body = cfg.get("config") or {} + if not isinstance(body, dict) or not body: + raise SystemExit("FATAL: %s authenticatorConfig %r has no config body" + % (IMPORT_FILE, alias)) + # Values are stringified because Keycloak stores authenticator config values as + # strings. `negate: "false"` in the import and a stored "false" are the same + # declaration, and a YAML-unquoted `false` must not become a Python bool that + # then never matches. + configs[alias] = {k: str(v) for k, v in sorted(body.items())} + + # Every configAlias a flow references must be defined, else the reconciler reports + # "git declares config X but defines no key/value for it" on every run forever. + for alias, spec in flows.items(): + for ex in spec["executions"]: + ca = ex.get("configAlias") + if ca and ca not in configs: + raise SystemExit( + "FATAL: %s flow %r execution %r references authenticatorConfig %r, " + "which is not declared" % (IMPORT_FILE, alias, ex["name"], ca)) + + # ---- the enrolment required action, DERIVED (see the header) ------------------- + offers_passkey = any( + ex["kind"] == "authenticator" and ex["name"] == PASSKEY_AUTHENTICATOR + for spec in flows.values() for ex in spec["executions"]) + if not offers_passkey: + raise SystemExit( + "FATAL: %s declares a webAuthnPolicyPasswordless* policy but no %r execution " + "in any flow. The policy decides what a passkey must BE; without that " + "execution nothing ever OFFERS one, so the policy is unreachable " + "configuration and this desired state would be meaningless" + % (IMPORT_FILE, PASSKEY_AUTHENTICATOR)) + + want = { + "realm": realm_name, + "webAuthnPolicyPasswordless": policy, + # enabled: passkey enrolment must be possible, since the flow offers passkeys. + # defaultAction FALSE: enrolment stays OPT-IN. True would force every user to + # enrol a passkey at next login — the platform-wide lock-out the realm import's + # "additive and optional" posture exists to avoid. Derived, not copied; see the + # header for why the import cannot declare this itself. + "requiredAction": {"alias": ENROL_ACTION, "enabled": True, + "defaultAction": False}, + "browserFlow": browser_flow, + "flows": flows, + "authenticatorConfig": configs, + } + + body = json.dumps(want, indent=2, sort_keys=True, ensure_ascii=False) + indented = "\n".join((" " + ln) if ln else "" for ln in body.splitlines()) + return HEADER + indented + "\n" + + +HEADER = """--- +# ============================================================================= +# GENERATED FILE — DO NOT EDIT BY HAND. +# +# source: platform/keycloak/keycloak-realm-import.yaml (spec.realm) +# regenerate: python3 ci/render-keycloak-passkey-policy.py --write +# gated by: conformance/tests/l0-static/keycloak-passkey-reconciler +# (runs this renderer with --check on every merge) +# +# WHAT THIS IS: the DESIRED STATE for the `homelab` realm's PASSKEY configuration — +# the WebAuthn passwordless policy, the enrolment required action, and the declared +# browser authentication flow tree. platform/keycloak/passkey-reconciler.yaml mounts +# it and (a) converges the policy and the required action onto it, (b) AUDITS the +# live browser flow against it and alarms on divergence without rewriting it. +# +# WHY IT IS A SEPARATE OBJECT: the realm import is a ONE-SHOT that never re-runs +# once Done=True, so a realm-level field or a flow edited in git never reaches the +# realm. Measured on prod 2026-10-18, hours after the passkey declaration merged: +# Done=True / Started=False, with the whole passkey block inert. The reconciler is +# the level-triggered loop that closes that; this file is what it reconciles +# TOWARDS. +# +# WHY IT IS GENERATED AND NOT AUTHORED: a hand-written copy of the RP ID, the +# user-verification requirement and the flow tree would be a second source of truth, +# free to disagree with the realm import — the same "two declarations that can +# disagree" shape as the bug being closed. It is rendered from the import and the +# render is gated, so the two cannot drift. +# +# STRINGS ARE VERBATIM: webAuthnPolicyPasswordlessRpId is the ${BASE_DOMAIN} +# substitution token exactly as the realm import writes it. Both files are rendered +# by the SAME Flux Kustomization (./platform-config), so the token resolves to the +# same value in both and the domain is decided in exactly one place. +# +# `requiredAction` is the ONE entry that is DERIVED rather than copied: the realm +# import deliberately declares no `requiredActions` block, because Keycloak 26.0.6 +# treats that as REPLACE and declaring one action would delete CONFIGURE_TOTP and +# nine more (a hard lock-out of every platform-admin). It is derived instead from +# the presence of a webauthn-authenticator-passwordless execution in the flow: +# passkeys are offered, therefore enrolment must be enabled — and defaultAction is +# false, so enrolment stays opt-in. +# +# A field NOT declared on the realm in the import is simply ABSENT here, and the +# reconciler leaves it alone. Absence means UNMANAGED, never "force to a default". +# ============================================================================= +apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s + labels: + app.kubernetes.io/name: %s + app.kubernetes.io/part-of: keycloak +data: + %s: | +""" % (CONFIGMAP_NAME, NAMESPACE, CONFIGMAP_NAME, DATA_KEY) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group() + g.add_argument("--write", action="store_true", help="write %s" % OUT_FILE) + g.add_argument("--check", action="store_true", + help="exit 1 if the committed file has drifted from the source") + args = ap.parse_args() + + want = render(ROOT) + target = ROOT / OUT_FILE + + if args.write: + target.write_text(want, encoding="utf-8") + print("wrote %s" % OUT_FILE) + return 0 + + if args.check: + if not target.is_file(): + print("CHECK FAIL: %s does not exist. Run: python3 " + "ci/render-keycloak-passkey-policy.py --write" % OUT_FILE, + file=sys.stderr) + return 1 + have = target.read_text(encoding="utf-8") + if have == want: + print("passkey-desired-state.yaml is in sync with %s" % IMPORT_FILE) + return 0 + print("CHECK FAIL: %s has DRIFTED from %s — the reconciler would converge the " + "realm on a passkey policy the realm import does not declare, and audit " + "the login flow against a tree nobody authored.\n" + "Regenerate with: python3 ci/render-keycloak-passkey-policy.py --write\n" + % (OUT_FILE, IMPORT_FILE), file=sys.stderr) + for line in difflib.unified_diff(have.splitlines(True), want.splitlines(True), + fromfile="committed " + OUT_FILE, + tofile="rendered from " + IMPORT_FILE): + sys.stderr.write(line) + return 1 + + sys.stdout.write(want) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/conformance/tests/l0-static/keycloak-passkey-reconciler/chainsaw-test.yaml b/conformance/tests/l0-static/keycloak-passkey-reconciler/chainsaw-test.yaml new file mode 100644 index 0000000..7d87164 --- /dev/null +++ b/conformance/tests/l0-static/keycloak-passkey-reconciler/chainsaw-test.yaml @@ -0,0 +1,101 @@ +# ============================================================================= +# L0 STATIC — the passkey configuration is derived from one declaration, continuously +# reconciled, and the browser login flow is auditable but UNWRITABLE +# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster. +# +# R-IDP-03 (identity config is declarative) and R-IDP-04 (MFA available) are both +# implemented through a mechanism that FAILS OPEN: the KeycloakRealmImport is a ONE-SHOT. +# Measured on prod 2026-10-18, hours after the passkey declaration merged: +# +# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath='{.status}' +# Done=True Started=False HasErrors=False +# +# Once it rests at Done=True the operator never re-runs it, so the WebAuthn passwordless +# policy and the restructured `browser-mfa` flow that keycloak-realm-import.yaml declares +# are INERT — in git, applied by Flux, reported Ready, reaching the realm never. Merging +# them changed nothing about how anyone logs in, and nothing anywhere said so. +# Declarative config that silently does not apply is not declarative config. +# +# GIVEN the repository as authored, +# THEN the reconciler's desired state is GENERATED from the realm import (so the RP ID — +# which is baked into every passkey at enrolment and can NEVER be re-keyed — is +# declared exactly once); both files are rendered by the SAME Flux Kustomization, so +# their ${BASE_DOMAIN} token cannot resolve to two values; the renderer's exported +# policy keys and the reconciler's writable keys are the SAME tuple; the realm +# import declares NO requiredActions block (26.0.6 treats it as REPLACE, which would +# delete CONFIGURE_TOTP and lock out every platform-admin); the enrolment action is +# enabled but NOT a defaultAction, so enrolment stays opt-in; the reconciler carries +# no ServiceAccount token, mounts only what it needs, and contains no dollar sign; +# the endpoint allow-list contains NO write for any authentication flow, execution +# or authenticator config, and NO required-action write for any alias other than the +# one named literally; the SIBLING client-secret reconciler's allow-list is NOT +# widened; and the reconciler's own `--selftest` (extracted from the shipped +# ConfigMap and executed) proves compare-then-write, idempotence, strict +# read-modify-write, the RpEntityName gate, flow-is-unwritable, the conditional-OTP +# regression and fail-closed — with mutations.py proving that selftest can FAIL. +# +# WHY THE FLOW IS AUDITED AND NOT CONVERGED, asserted here rather than only argued: +# converging a five-flow, ten-execution restructure needs create-subflow, +# create-execution, set-requirement, set-priority and bind-config as SEPARATE admin API +# calls with no transaction. A run that dies between them leaves a HALF-BUILT FLOW BOUND +# AS THE LOGIN PATH for Grafana, Gitea, Nextcloud, Headlamp, Semaphore, PiHole, Crafty, +# the *arr stack AND kubectl. This repo has already had a total SSO outage from a realm +# change (2026-09-06), and the conditional-OTP sub-flow — which the declared structure +# moves one level deeper than it sits today — once 403'd every platform-admin. A +# reconciler that half-rewrites the bound flow every 15 minutes is a worse failure than +# the one it replaces, because it is a loop rather than a single bad change. +# +# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git + PyYAML + +# stdlib); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw +# Test wraps the SAME script. +# +# NOT proven here: that the LIVE realm's policy, enrolment action or browser flow match +# git (that is cluster state — the reconciler's `--audit` mode is the read-only probe for +# it, and NO L2 test is added by this change, so the live realm is UNVERIFIED by CI); +# that a passkey ceremony actually completes or that the ALTERNATIVE fork degrades to the +# password form (needs a real browser against a real realm); that the five flow-building +# admin-API calls behave against a live Keycloak the way the 26.0.6 source says they do +# (every proof here is against an in-process stub); or that the CronJob's schedule fires. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-passkey-reconciler + labels: + conformance.home.k3s/R-IDP-03: "true" + conformance.home.k3s/R-IDP-04: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04" + conformance.home.k3s/layer: "L0" + # WHOLE-REPO SCOPE, for the same reason the sibling keycloak-client-secret-reconciler + # check declares it: this invariant spans platform/ (the realm import, the reconciler, + # the generated desired state), platform-config/ (which Flux Kustomization renders + # them, and therefore which substitution pass they share), ci/ (the renderer) and + # clusters/ (that both are reachable from both entrypoints). A change that breaks it + # need not attribute R-IDP-03/04 at all, and the requirement-scoped selection would + # then filter this check out of exactly the change that introduces the bug — which is + # how 12 unusable prod OIDC clients sat behind an entirely green cluster on + # 2026-09-06. + conformance.home.k3s/scope: "whole-repo" + # Not safety-critical in the harness's sense (it mutates nothing and needs no + # cluster), but it guards a component that holds WRITE access to the identity + # provider's realm settings. + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + The realm's passkey configuration — policy, enrolment action AND browser login + flow — is declared once in the realm import, derived from it by a gated renderer, + and continuously reconciled; the flow converges blue/green, so a one-shot import + that never re-runs can neither leave the configuration inert nor let a reconciler + half-rewrite the login path of the whole platform. +spec: + steps: + - name: audit-passkey-declaration-renderer-and-reconciler + description: >- + Assert the desired state is derived from the realm import and shares its + substitution pass, that the reconciler's write surface excludes every flow + mutation, that the sibling reconciler is unwidened, and run the reconciler's own + selftest plus the mutation suite that proves it can fail. + try: + - script: + content: bash ./check.sh diff --git a/conformance/tests/l0-static/keycloak-passkey-reconciler/check.sh b/conformance/tests/l0-static/keycloak-passkey-reconciler/check.sh new file mode 100755 index 0000000..bda08e9 --- /dev/null +++ b/conformance/tests/l0-static/keycloak-passkey-reconciler/check.sh @@ -0,0 +1,708 @@ +#!/usr/bin/env bash +# ============================================================================= +# L0 static check (offline, no cluster) — keycloak-passkey-reconciler +# Serves: R-IDP-03 ("Identity config (realms, clients, groups, roles) is declarative." +# — spec docs/15-identity-and-access.md) +# R-IDP-04 ("IdP secrets SOPS-encrypted; MFA available." +# — spec docs/15-identity-and-access.md) +# +# WHY THIS EXISTS — THE REALM IMPORT IS A ONE-SHOT THAT FAILS OPEN +# --------------------------------------------------------------- +# platform/keycloak/keycloak-realm-import.yaml declares the WebAuthn PASSWORDLESS +# policy and a restructured `browser-mfa` browser flow. Measured on prod 2026-10-18, +# hours after that declaration merged: +# +# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath='{.status}' +# Done=True Started=False HasErrors=False +# +# A KeycloakRealmImport is a ONE-SHOT. Once it rests at Done=True the operator never +# re-runs it, so every realm-level field and every flow declared after the FIRST +# import is inert — in git, applied by Flux, reported Ready, reaching the realm +# never. The same shape as the 2026-09-06 client-secret incident (12 prod clients +# holding the literal `(env:...)` placeholder behind an entirely green cluster), and +# the same fix: a level-triggered reconciler. This check is the static half. +# +# THE RECONCILER HOLDS WRITE ACCESS TO THE IDENTITY PROVIDER. That is the reason this +# check is unusually thorough about its BLAST RADIUS rather than only its behaviour. +# +# GIVEN the repository as authored, +# THEN (1) THE DESIRED STATE IS DERIVED, NOT DECLARED TWICE: the reconciler's input +# (platform/keycloak/passkey-desired-state.yaml) is GENERATED from the realm +# import by ci/render-keycloak-passkey-policy.py, and this check RUNS the +# renderer with --check, so a hand edit that makes the two disagree is +# unmergeable. The RP ID is the sharpest case: it is baked into every passkey +# at enrolment and can NEVER be re-keyed, so two declarations that can skew +# about the domain is an unrecoverable-credential bug, not a cosmetic one. +# (2) BOTH FILES ARE RENDERED BY THE SAME FLUX KUSTOMIZATION. +# postBuild.substituteFrom is per-Kustomization, so a split would let the +# ${BASE_DOMAIN} token in the realm import and the one in the desired state +# resolve to two different values — and an RP ID mismatch invalidates every +# enrolled passkey with no error the user can act on. +# (3) THE DESIRED STATE USES NO SUBSTITUTION TOKEN THE REALM IMPORT DOES NOT, and +# declares no realm key outside the reconciler's write surface. +# (4) THE TWO WRITE SURFACES AGREE: the renderer's exported policy keys and the +# reconciler's POLICY_FIELDS are the SAME tuple. If they drifted, the renderer +# could emit a key the reconciler refuses (fail-closed forever) or the +# reconciler could accept a key the renderer never emits (dead surface). +# (5) THE RECONCILER MANIFEST IS SAFE AND WIRED: reconciled from BOTH cluster +# entrypoints, mounts exactly the ConfigMaps its code defaults to, carries NO +# ServiceAccount token, and contains no dollar sign (it ships inside a +# SUBSTITUTED Flux path, where an unescaped dollar-brace expansion is +# rewritten before any container runs). +# (6) THE CURRENTLY BOUND FLOW IS STRUCTURALLY UNWRITABLE — asserted against the +# SOURCE, not the comments. The flow converges BLUE/GREEN (build under a new +# alias, verify, flip browserFlow in one atomic PUT), so flow-create and +# execution-add endpoints ARE on the allow-list and their presence is not the +# hazard. What is asserted instead: nothing is DELETABLE (the superseded flow +# is the rollback target); the execution- and config-id-scoped write endpoints +# stay off the list because they carry no flow alias and so cannot be screened; +# no required-action write reaches any alias but the one named literally; and +# the guard the allow-list CANNOT express exists in the source and is wired +# into the single request choke point, FAIL-CLOSED until the live binding has +# been read. That last one is what the whole design rests on: a reconciler that +# half-rewrites the BOUND browser flow is a total SSO outage on a loop, and +# this repo has had one (2026-09-06). +# (7) THE SIBLING RECONCILER IS UNTOUCHED: client-secret-reconciler.yaml's +# allow-list is NOT widened by this work. Its --selftest asserts it refuses +# /users, /groups, /roles, /authentication and the whole-realm endpoint, and +# its guarantee that prod's real user and the media-users / platform-admins / +# viewers groups survive every run must stay LITERALLY true. +# (8) THE RECONCILER'S BEHAVIOUR IS PROVEN, NOT ASSERTED: this check EXTRACTS the +# script from the ConfigMap and executes its `--selftest`, which drives the +# real code paths against an in-process stub Keycloak that MODELS the three +# Keycloak 26.0.6 behaviours that bite — the RpEntityName gate on the +# passwordless policy, the all-or-nothing policy rebuild that defaults every +# omitted key, and the required-action representation replace. +# (8b) AND THAT PROOF IS FALSIFIABLE: mutations.py injects plausible regressions +# one at a time and requires --selftest to CATCH every one. A passing suite +# proves nothing if it cannot fail, and "reported success while the realm was +# broken" is the exact failure this component exists to end. +# AND (9) THIS CHECK IS LOAD-BEARING: it DECLARES +# `conformance.home.k3s/scope: whole-repo` so ci/l0.sh runs it unconditionally +# via --include-scope, ci/gate.sh blocks the merge when the l0 stage fails, and +# ci/touched-requirements.sh maps platform/keycloak/ to these requirement IDs. +# An unreferenced test is precisely the "looks applied and isn't" failure this +# repo keeps hitting. +# +# NOT proven here (named honestly, in the style of the sibling L0 checks): +# - that the LIVE realm's policy, enrolment action or browser flow match git. That is +# cluster state. The reconciler's own `--audit` mode is the read-only probe for it +# and is what an L2 test should run; no such L2 test is added by this change, so the +# live realm is UNVERIFIED by CI. +# - that the declared browser flow actually WORKS as a login path. Nothing offline can +# prove that a passkey ceremony completes, that the ALTERNATIVE fork degrades to the +# password form, or that a platform-admin is still asked for TOTP. Those need a real +# browser against a real realm. +# - that the reconciler's first real run against prod will succeed. The flow is built +# through five admin-API endpoints whose request shapes were read from the 26.0.6 +# source but never exercised against a live Keycloak. +# - that the admin credential decrypts to a usable password. Key PRESENCE is checkable +# offline; the plaintext is not, and must not be. +# - that the CronJob's image can be pulled or its schedule fires. +# ============================================================================= +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root +PYTHON="${PYTHON:-python3}" + +IMPORT_FILE="platform/keycloak/keycloak-realm-import.yaml" +RECON_FILE="platform/keycloak/passkey-reconciler.yaml" +STATE_FILE="platform/keycloak/passkey-desired-state.yaml" +RENDERER="ci/render-keycloak-passkey-policy.py" +SIBLING_FILE="platform/keycloak/client-secret-reconciler.yaml" + +# Honest SKIP (never green) if the thing under test is absent entirely. +for f in "$IMPORT_FILE" "$RECON_FILE" "$STATE_FILE" "$RENDERER"; do + if [ ! -f "${ROOT}/${f}" ]; then + echo "check SKIPPED: ${f} absent — nothing to audit" >&2 + exit 3 + fi +done + +cd "${ROOT}" +"${PYTHON}" - "${ROOT}" <<'PY' +"""Static audit of the Keycloak passkey declaration, the renderer that derives the +reconciler's desired state from it, and the reconciler's blast radius. PyYAML parses the +manifests; lib/status.py is IMPORTED for the Flux graph walk rather than forked, so there +is one definition of "what a Flux path renders" in the harness.""" +from __future__ import annotations + +import ast +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +ROOT = Path(sys.argv[1]) +# This body is fed to `python3 -` on stdin, so there is no __file__ to derive from; the +# test directory is named relative to the repo root the wrapper passes in. +DIR_PATH = (ROOT / "conformance" / "tests" / "l0-static" + / "keycloak-passkey-reconciler") +sys.path.insert(0, str(ROOT / "conformance" / "lib")) +import status as st # noqa: E402 + +IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml" +RECON_FILE = "platform/keycloak/passkey-reconciler.yaml" +STATE_FILE = "platform/keycloak/passkey-desired-state.yaml" +RENDERER = "ci/render-keycloak-passkey-policy.py" +SIBLING_FILE = "platform/keycloak/client-secret-reconciler.yaml" +STATE_CM_NAME = "keycloak-passkey-desired-state" +STATE_DATA_KEY = "passkey-desired-state.json" +SCRIPT_KEY = "reconcile-keycloak-passkeys.py" +ENROL_ACTION = "webauthn-register-passwordless" +DOLLAR = chr(36) + +failures: list[str] = [] + + +def check(cond, msg): + if not cond: + failures.append(msg) + + +def load(rel): + return list(yaml.safe_load_all((ROOT / rel).read_text())) + + +# ================================================== (1) the desired state is DERIVED +proc = subprocess.run([sys.executable, str(ROOT / RENDERER), "--check"], + capture_output=True, text=True, timeout=120) +check(proc.returncode == 0, + f"{STATE_FILE} has DRIFTED from {IMPORT_FILE} (ci/render-keycloak-passkey-policy" + f".py --check failed). The reconciler would converge the realm on a passkey policy " + f"the realm import does not declare, and audit the login flow against a tree " + f"nobody authored. The RP ID is the sharpest case: it is baked into every passkey " + f"at enrolment and can NEVER be re-keyed, so two declarations that skew about the " + f"domain is an unrecoverable-credential bug.\n " + + "\n ".join((proc.stdout + proc.stderr).strip().splitlines()[-12:])) + +# The realm import must actually declare the things the renderer derives from. If it +# stops, the renderer fails -- but say so HERE too, because a SKIP-shaped failure in the +# renderer would otherwise be the only signal. +imp_docs = [d for d in load(IMPORT_FILE) if d] +imp = next((d for d in imp_docs if d.get("kind") == "KeycloakRealmImport"), None) +check(imp is not None, f"{IMPORT_FILE}: no KeycloakRealmImport document") +realm = ((imp or {}).get("spec") or {}).get("realm") or {} +policy_keys_in_import = sorted(k for k in realm + if k.startswith("webAuthnPolicyPasswordless")) +check(policy_keys_in_import, + f"{IMPORT_FILE} declares no webAuthnPolicyPasswordless* field, so there is no " + f"passkey policy for the reconciler to converge") +check(realm.get("browserFlow"), + f"{IMPORT_FILE} declares no spec.realm.browserFlow — a perfect flow that is not " + f"BOUND changes nothing about how anyone logs in") + +# THE requiredActions TRAP, enforced rather than only commented. The realm import's own +# header explains it at length: Keycloak 26.0.6 treats requiredActions in a realm +# representation as REPLACE, not merge, so declaring ONE action imports exactly that +# action plus delete_account and skips DefaultRequiredActions.addActions entirely -- +# deleting CONFIGURE_TOTP, UPDATE_PASSWORD, VERIFY_EMAIL and seven more. Losing +# CONFIGURE_TOTP is a hard lock-out of every platform-admin, because auth-otp-form in the +# browser flow still demands a TOTP they could then never enrol. The comment saying "DO +# NOT ADD ONE" is advice; this is the gate. +check("requiredActions" not in realm, + f"{IMPORT_FILE} declares spec.realm.requiredActions. Keycloak 26.0.6 treats that " + f"as REPLACE, not merge: declaring any action imports exactly those plus " + f"delete_account and NEVER runs DefaultRequiredActions.addActions, so " + f"CONFIGURE_TOTP, UPDATE_PASSWORD, VERIFY_EMAIL and seven more are DELETED. That " + f"is a permanent lock-out of every platform-admin, because auth-otp-form in this " + f"realm's browser flow still demands a TOTP they could no longer enrol. The " + f"enrolment action is already registered by default (priority 80, enabled, " + f"defaultAction false) and the passkey reconciler converges it through the " + f"per-alias admin API, which has no such hazard") + +# ============================== (2)(3) the two files share one substitution pass +state_docs = [d for d in load(STATE_FILE) if d] +state_cm = next((d for d in state_docs if d.get("kind") == "ConfigMap"), None) +check(state_cm is not None, f"{STATE_FILE}: no ConfigMap") +desired = {} +if state_cm is not None: + check(state_cm["metadata"]["name"] == STATE_CM_NAME, + f"{STATE_FILE}: the ConfigMap is named " + f"{state_cm['metadata']['name']!r}, but the CronJob mounts {STATE_CM_NAME!r}") + data = state_cm.get("data") or {} + check(list(data) == [STATE_DATA_KEY], + f"{STATE_FILE}: expected exactly the data key {STATE_DATA_KEY!r}, got " + f"{sorted(data)} — the reconciler's PASSKEY_STATE_FILE default would not " + f"resolve") + raw = data.get(STATE_DATA_KEY, "") + try: + desired = json.loads(raw) + except ValueError as e: + check(False, f"{STATE_FILE}: the {STATE_DATA_KEY} payload is not valid JSON " + f"({e}) — the reconciler would fail closed on every run") + +# Both files must be rendered by ONE Flux Kustomization. substituteFrom is +# per-Kustomization, so a split lets the same ${BASE_DOMAIN} token resolve to two +# different values -- and an RP ID mismatch invalidates every enrolled passkey silently. +import_owners = set() +state_owners = set() +recon_owners = set() +for env in ("dev", "prod"): + entry = f"clusters/{env}" + if not (ROOT / entry).is_dir(): + continue + applied, _ = st.reconciled_files(ROOT, entry) + check(RECON_FILE in applied, + f"{RECON_FILE} is not reachable from the {entry} entrypoint — the passkey " + f"healer would exist on one cluster only") + check(STATE_FILE in applied, + f"{STATE_FILE} is not reachable from the {entry} entrypoint — the reconciler " + f"would fail closed on that cluster forever") + check(IMPORT_FILE in applied, f"{IMPORT_FILE} is not reachable from {entry}") + +# Which Kustomization DIRECTORY lists each file. Asserted structurally: both must be +# listed as resources of the same kustomization.yaml. +for kpath in ROOT.glob("*/kustomization.yaml"): + try: + k = yaml.safe_load(kpath.read_text()) or {} + except Exception: + continue + res = [str(r) for r in (k.get("resources") or [])] + base = kpath.parent.name + if any(r.endswith("keycloak-realm-import.yaml") for r in res): + import_owners.add(base) + if any(r.endswith("passkey-desired-state.yaml") for r in res): + state_owners.add(base) + if any(r.endswith("passkey-reconciler.yaml") for r in res): + recon_owners.add(base) +check(import_owners and import_owners == state_owners, + f"{IMPORT_FILE} is rendered by {sorted(import_owners)} but {STATE_FILE} by " + f"{sorted(state_owners)}. They MUST be the same Flux Kustomization: " + f"postBuild.substituteFrom is per-Kustomization, so a split would let the " + f"{DOLLAR}{{BASE_DOMAIN}} token in each resolve to a different value. An RP ID is " + f"baked into every passkey at enrolment and can never be re-keyed, so a domain the " + f"two files disagree about is an unrecoverable credential, not a cosmetic drift") +check(recon_owners == state_owners, + f"{RECON_FILE} is rendered by {sorted(recon_owners)} but its desired state by " + f"{sorted(state_owners)} — the reconciler could be applied without its input") + +# (3) no substitution token the realm definition does not also use. +import_text = (ROOT / IMPORT_FILE).read_text() +TOKEN = re.compile(re.escape(DOLLAR) + r"\{([A-Za-z_][A-Za-z0-9_]*)") +state_tokens = set(TOKEN.findall((ROOT / STATE_FILE).read_text())) +import_tokens = set(TOKEN.findall(import_text)) +# The renderer's own HEADER mentions ${BASE_DOMAIN} in prose; that is the same token the +# policy uses, so a subset check is the right assertion rather than an exact match. +check(state_tokens <= import_tokens, + f"{STATE_FILE} uses substitution token(s) {sorted(state_tokens - import_tokens)} " + f"that {IMPORT_FILE} does not. A token the realm definition never uses is either a " + f"hand edit or a renderer bug, and an UNDEFINED one renders as the EMPTY STRING " + f"silently — which for an RP ID means every passkey ceremony fails") + +# ======================================= (4) the two write surfaces are the SAME tuple +recon_docs = [d for d in load(RECON_FILE) if d] +cm = next((d for d in recon_docs if d.get("kind") == "ConfigMap"), None) +cj = next((d for d in recon_docs if d.get("kind") == "CronJob"), None) +check(cm is not None, f"{RECON_FILE}: no ConfigMap carrying the reconciler script") +check(cj is not None, f"{RECON_FILE}: no CronJob running the reconciler") + +script = "" +if cm is not None: + data = cm.get("data") or {} + check(len(data) == 1, + f"{RECON_FILE}: the reconciler ConfigMap must carry exactly one script key " + f"(found {sorted(data)}) — a second copy would be untested by this gate") + check(SCRIPT_KEY in data, + f"{RECON_FILE}: expected the script under {SCRIPT_KEY!r}, got {sorted(data)}") + script = next(iter(data.values()), "") + check(len(script.splitlines()) > 200, + f"{RECON_FILE}: the extracted script is implausibly short " + f"({len(script.splitlines())} lines) — the ConfigMap shape changed and the " + f"selftest below would prove nothing") + # THE nextcloud TRAP. This file is rendered by a Flux Kustomization with + # postBuild.substituteFrom, so envsubst rewrites it before any container runs. + check(DOLLAR not in script, + f"{RECON_FILE}: the reconciler script contains a dollar sign. It ships inside " + f"a SUBSTITUTED Flux path, where a dollar-brace expansion is rewritten at " + f"reconcile time (see apps/wave-1/nextcloud.yaml). Rewrite without it — Python " + f"never needs one; chr(36) spells it where a literal is genuinely required") + + +def tuple_literal(src, name): + """The value of a module-level tuple assignment, via ast. Parsing the SOURCE rather + than importing it keeps this check offline and side-effect free, and means the thing + asserted is the text that ships.""" + try: + tree = ast.parse(src) + except SyntaxError: + return None + for node in tree.body: + if isinstance(node, ast.Assign): + for t in node.targets: + if isinstance(t, ast.Name) and t.id == name: + try: + return ast.literal_eval(node.value) + except ValueError: + return None + return None + + +renderer_src = (ROOT / RENDERER).read_text() + + +# POLICY_FIELDS is a CONCATENATION of other tuples in both files, so literal_eval of the +# assignment itself cannot evaluate it. Resolve it from its parts instead. Both files name +# their parts from the same vocabulary; a part present in one and not the other simply +# contributes nothing, which is why the comparison below is of the resolved SETS. +def resolve_policy(src): + parts = ("POLICY_STR_FIELDS", "POLICY_ORDERED_LIST_FIELDS", + "POLICY_UNORDERED_LIST_FIELDS", "POLICY_LIST_FIELDS", "POLICY_INT_FIELDS", + "POLICY_BOOL_FIELDS") + out = [] + for p in parts: + v = tuple_literal(src, p) + if v: + out.extend(v) + return sorted(set(out)) + + +recon_keys = resolve_policy(script) +rend_keys = resolve_policy(renderer_src) +check(recon_keys and recon_keys == rend_keys, + f"the reconciler's writable policy keys {recon_keys} and the renderer's exported " + f"keys {rend_keys} DISAGREE. If the renderer emits a key the reconciler refuses, " + f"every run fails closed forever; if the reconciler accepts a key the renderer " + f"never emits, that is dead write surface on the identity provider") +# Every exported key must actually be a passwordless key. The two-factor +# webAuthnPolicy* family is a silent no-op for passkeys (the passwordless authenticator +# reads getWebAuthnPolicyPasswordless() only), so exporting one would be a write surface +# with no effect. +check(all(k.startswith("webAuthnPolicyPasswordless") for k in recon_keys), + f"the reconciler's writable keys include a non-passwordless field: " + f"{[k for k in recon_keys if not k.startswith('webAuthnPolicyPasswordless')]}. " + f"The two-factor webAuthnPolicy* family is never consulted by a passkey login, so " + f"writing it is a silent no-op") +# And the desired state must declare only those. +if isinstance(desired.get("webAuthnPolicyPasswordless"), dict): + extra = sorted(set(desired["webAuthnPolicyPasswordless"]) - set(recon_keys)) + check(not extra, + f"{STATE_FILE} declares policy key(s) {extra} outside the reconciler's write " + f"surface — it would fail closed on every run") + # The 26.0.6 gate: the policy is applied ONLY when RpEntityName is non-empty. + check(desired["webAuthnPolicyPasswordless"].get( + "webAuthnPolicyPasswordlessRpEntityName"), + f"{STATE_FILE} carries no non-empty " + f"webAuthnPolicyPasswordlessRpEntityName. Keycloak 26.0.6 applies the " + f"passwordless policy only when that field is set " + f"(DefaultExportImportManager.updateRealm), so every realm PUT would be " + f"accepted with 204 and change nothing") + +# THE ENROLMENT POSTURE. defaultAction true would force every user to enrol a passkey at +# next login -- a platform-wide lock-out risk, and the exact opposite of the realm +# import's stated "additive and optional" decision. It is one boolean, so it is gated. +ra = desired.get("requiredAction") or {} +check(ra.get("alias") == ENROL_ACTION, + f"{STATE_FILE}: requiredAction.alias is {ra.get('alias')!r}, expected " + f"{ENROL_ACTION!r} — the reconciler's allow-list names that alias literally, so " + f"any other value is unreachable and the pass would fail closed") +check(ra.get("enabled") is True, + f"{STATE_FILE}: requiredAction.enabled must be true, else passkey ENROLMENT is " + f"impossible while the login flow offers passkeys") +check(ra.get("defaultAction") is False, + f"{STATE_FILE}: requiredAction.defaultAction is {ra.get('defaultAction')!r}, which " + f"must be false. True forces EVERY user to enrol a passkey at next login — this " + f"IdP gates Grafana, Gitea, Nextcloud, Headlamp, Semaphore, PiHole, Crafty, the " + f"*arr stack AND kubectl, so a misfiring mandatory enrolment is a total platform " + f"lock-out. The realm import's whole stated posture is additive and OPTIONAL") + +# ========================================= (5) the reconciler manifest is safe + wired +if cj is not None: + pod = cj["spec"]["jobTemplate"]["spec"]["template"]["spec"] + ctr = pod["containers"][0] + vols = {v["name"]: v for v in pod.get("volumes", [])} + mounts = {m["name"]: m for m in ctr.get("volumeMounts", [])} + env = {e["name"]: e for e in ctr.get("env", [])} + + # NO KUBERNETES API ACCESS. This is what makes the mounted-ConfigMap design strictly + # narrower than reading the KeycloakRealmImport CR through the API: no Role, no + # RoleBinding, no token. It is also why this change adds no **/rbac.yaml. + check(pod.get("automountServiceAccountToken") is False, + f"{RECON_FILE}: the reconciler must set automountServiceAccountToken: false. " + f"It talks only to the Keycloak admin API and reads a mounted ConfigMap, so a " + f"ServiceAccount token is pure blast radius (R-SEC-02)") + check("serviceAccountName" not in pod, + f"{RECON_FILE}: the reconciler must not name a ServiceAccount") + + cm_vols = {n: v for n, v in vols.items() if "configMap" in v} + names = {(v["configMap"] or {}).get("name") for v in cm_vols.values()} + check(STATE_CM_NAME in names, + f"{RECON_FILE}: the CronJob does not mount the {STATE_CM_NAME} ConfigMap, so " + f"the reconciler would fail closed on every run. Mounted: {sorted(names)}") + check(cm["metadata"]["name"] in names, + f"{RECON_FILE}: the CronJob does not mount its own script ConfigMap") + # NO SECRET IS MOUNTED AS A VOLUME. The admin credential is consumed via + # secretKeyRef only; projecting it as a file would widen what a readable mount leaks. + check(not [v for v in vols.values() if "secret" in v], + f"{RECON_FILE}: the reconciler mounts a Secret as a volume. The admin " + f"credential is referenced by secretKeyRef only; it must never be projected " + f"as a file here") + + # The mount paths must be the ones the code DEFAULTS to, or the container starts and + # fails closed forever while every manifest looks correct. + state_vol = next((n for n, v in cm_vols.items() + if (v["configMap"] or {}).get("name") == STATE_CM_NAME), None) + if state_vol: + mp = (mounts.get(state_vol) or {}).get("mountPath") + check(mp == "/etc/keycloak-passkey", + f"{RECON_FILE}: the desired-state ConfigMap is mounted at {mp!r}, but the " + f"script's PASSKEY_STATE_FILE default is " + f"/etc/keycloak-passkey/{STATE_DATA_KEY}") + check(f'"/etc/keycloak-passkey/{STATE_DATA_KEY}"' in script, + f"{RECON_FILE}: the script's PASSKEY_STATE_FILE default no longer matches " + f"the mount path — they are a pair and must move together") + check((mounts.get(state_vol) or {}).get("readOnly") is True, + f"{RECON_FILE}: the desired-state mount must be readOnly") + + # The admin credential, referenced BY NAME. Never decrypted, never a literal. + for key in ("KEYCLOAK_ADMIN_USER", "KEYCLOAK_ADMIN_PASSWORD"): + e = env.get(key) or {} + ref = ((e.get("valueFrom") or {}).get("secretKeyRef") or {}) + check(ref.get("name") == "keycloak-admin-credentials", + f"{RECON_FILE}: {key} must come from the keycloak-admin-credentials " + f"Secret by secretKeyRef, got {e!r}") + check("value" not in e, + f"{RECON_FILE}: {key} carries a literal `value`. A credential in a " + f"manifest is a plaintext secret in git") + check((env.get("KEYCLOAK_REALM") or {}).get("value") == desired.get("realm"), + f"{RECON_FILE}: KEYCLOAK_REALM is " + f"{(env.get('KEYCLOAK_REALM') or {}).get('value')!r} but the desired state " + f"declares realm {desired.get('realm')!r}. The reconciler would converge the " + f"wrong realm, or none") + + check(ctr["securityContext"]["readOnlyRootFilesystem"] is True, + f"{RECON_FILE}: readOnlyRootFilesystem must be true") + check(ctr["securityContext"]["allowPrivilegeEscalation"] is False, + f"{RECON_FILE}: allowPrivilegeEscalation must be false") + check(pod["securityContext"].get("runAsNonRoot") is True, + f"{RECON_FILE}: runAsNonRoot must be true") + check(cj["spec"].get("concurrencyPolicy") == "Forbid", + f"{RECON_FILE}: concurrencyPolicy must be Forbid — two concurrent reconcilers " + f"would double the write attempts against the IdP during an outage recovery") + check(re.fullmatch(r"\S+ \S+ \S+ \S+ \S+", cj["spec"].get("schedule", "")), + f"{RECON_FILE}: schedule {cj['spec'].get('schedule')!r} is not a 5-field cron") + +# ===== (6) THE BOUND FLOW IS STRUCTURALLY UNWRITABLE — asserted against the SOURCE +# +# This component converges the browser flow BLUE/GREEN: it builds the target structure +# under a NEW, unused alias, verifies it, then flips the realm's browserFlow in ONE +# atomic PUT. It therefore NEEDS flow-create and execution-add endpoints, so unlike the +# first design of this check the mere presence of a flow write is no longer the thing to +# forbid. What must hold instead is narrower and stronger: +# +# (a) nothing may be DELETED, ever — the superseded flow is the rollback target; +# (b) the single-execution and config-edit endpoints stay off the list — requirements +# are set through the flow-scoped endpoint, which the guard in (d) screens; +# (c) no required-action write may reach any alias but the one named LITERALLY; +# (d) and the guard the allow-list CANNOT express must exist in the source: a +# method-aware screen that refuses any flow write targeting the CURRENTLY BOUND +# flow's subtree, FAIL-CLOSED until that binding has been read. +# +# (d) is the load-bearing one. Without it the reconciler could mutate the flow users are +# logging in through, and a run dying mid-restructure would leave a HALF-BUILT FLOW BOUND +# AS THE LOGIN PATH for Grafana, Gitea, Nextcloud, Headlamp, Semaphore, PiHole, Crafty, +# the *arr stack AND kubectl. This repo has already had a total SSO outage from a realm +# change (2026-09-06). +allow = tuple_literal(script, "_ALLOWED") or () +check(bool(allow), + f"{RECON_FILE}: could not parse the _ALLOWED endpoint allow-list out of the " + f"script. This check's central assertions about the write surface cannot be made, " + f"so it must FAIL rather than pass silently") +if allow: + check(all(isinstance(e, tuple) and len(e) == 2 for e in allow), + f"{RECON_FILE}: _ALLOWED is not a tuple of (method, path) pairs. A path-only " + f"allow-list cannot distinguish reading a flow from writing one, which this " + f"component's whole safety argument rests on") + writes = [(m, p) for m, p in allow if m in ("PUT", "POST", "PATCH", "DELETE")] + # (a) nothing is ever deleted. + check(not [(m, p) for m, p in allow if m == "DELETE"], + f"{RECON_FILE}: the allow-list contains a DELETE method. Nothing this program " + f"does requires deleting anything: a superseded flow is left unbound as the " + f"ROLLBACK TARGET, and DELETE /admin/realms/ deletes the realm") + # (b) the endpoints that would let it edit live objects in place. + banned = [(m, p) for m, p in writes + if re.fullmatch(r"/admin/realms/\[\^/\]\+/authentication/executions/" + r"\[\^/\]\+", p) + or re.fullmatch(r"/admin/realms/\[\^/\]\+/authentication/config/\[\^/\]\+", + p) + or p.endswith("/authentication/register-required-action")] + check(not banned, + f"{RECON_FILE}: the allow-list permits {banned}. The single-execution update " + f"and config-edit endpoints must stay off it: requirements are set through the " + f"FLOW-SCOPED endpoint, which carries the flow alias in its path and is " + f"therefore screenable against the bound subtree. An execution- or " + f"config-id-scoped write is not, so it would be an unscreened hole") + # (c) no required-action write may reach any other alias. + ra_writes = [(m, p) for m, p in writes if "required-actions" in p] + check(all(p.endswith("required-actions/" + ENROL_ACTION) for _, p in ra_writes), + f"{RECON_FILE}: the allow-list permits a required-action write to an alias " + f"other than {ENROL_ACTION!r}: {ra_writes}. Disabling CONFIGURE_TOTP is a " + f"permanent lock-out of every platform-admin, because auth-otp-form in the " + f"browser flow still demands a TOTP that could no longer be enrolled") + check(not [p for _, p in ra_writes if "[^/]+" in p.split("required-actions/")[-1]], + f"{RECON_FILE}: a required-action write path uses a WILDCARD alias. The alias " + f"must be named LITERALLY so no other action is writable by construction") + # Nothing outside the realm + authentication surface at all. + stray = [(m, p) for m, p in allow + if "/users" in p or "/groups" in p or "/roles" in p or "/clients" in p] + check(not stray, + f"{RECON_FILE}: the allow-list reaches {stray}. Users, groups and roles must " + f"be unreachable (prod's real user and the media-users / platform-admins / " + f"viewers groups must survive every run); clients are the SIBLING " + f"reconciler's surface and the two must not overlap") + +# ---- (d) THE PROTECTED-SUBTREE GUARD EXISTS, IS WIRED, AND IS FAIL-CLOSED ---------- +# Asserted against the SOURCE, because this is the property the blue/green design rests +# on and a comment claiming it is worth nothing. Three things must all be true. +check(re.search(r"^def _screen_write\(", script, re.M) is not None, + f"{RECON_FILE}: the script has no _screen_write() function. That is the guard " + f"that refuses any flow write targeting the CURRENTLY BOUND flow's subtree — the " + f"one thing the allow-list cannot express, because its write entries are " + f"alias-parameterised. Without it this component can mutate the live login path") +# It must be CALLED from the single choke point, not merely defined. +check(re.search(r"_screen_write\(method, path", script) is not None, + f"{RECON_FILE}: _screen_write() is defined but never called from _http() with the " + f"request's method and path. A guard that is not on the request path is decoration") +# FAIL-CLOSED: the protected sets must start undetermined, and the reset that runs at +# the top of every run must put them back there. +reset_fn = re.search(r"def _reset_protected\(\):(.*?)(?=\n\S)", script, re.S) +check(reset_fn is not None + and "_PROTECTED_FLOW_ALIASES = None" in reset_fn.group(1) + and "_PROTECTED_EXECUTION_IDS = None" in reset_fn.group(1), + f"{RECON_FILE}: _reset_protected() does not set both protected sets to None. They " + f"must start each run UNDETERMINED, because while they are, every flow write is " + f"refused — that is the fail-closed half of the guard, and it is what stops a " + f"build running before the live binding has been read") +check(re.search(r"_reset_protected\(\)", script.split("def _selftest")[0]) is not None, + f"{RECON_FILE}: _reset_protected() is never called outside the selftest, so the " + f"protected sets are not reset per run and a previous run's binding could be " + f"trusted for this one") +# The create-flow endpoint carries NO alias in its path, so it must be screened on its +# BODY or it escapes the guard entirely. (Found by the selftest; recorded here so a +# future edit cannot silently undo it.) +check("_FLOW_CREATE_PATH" in script, + f"{RECON_FILE}: there is no separate screen for POST .../authentication/flows. " + f"That endpoint carries no alias in its path — the alias is in the request BODY — " + f"so without a body-based screen it bypasses the protected-subtree guard " + f"altogether") +# And the one function allowed to change the live binding must be distinct from the +# policy write, which must refuse to. +check("def bind_browser_flow(" in script, + f"{RECON_FILE}: there is no bind_browser_flow() function. The atomic flip must be " + f"one named, auditable operation, not an incidental field in some other write") +check(re.search(r"never rebinds the browser flow", script) is not None, + f"{RECON_FILE}: the policy write no longer refuses to change browserFlow. " + f"PUT /admin/realms/ DOES read rep.getBrowserFlow(), so the policy pass " + f"can rebind the login path by accident — to an UNVERIFIED flow") + +# ======================================== (7) the SIBLING reconciler is NOT widened +if (ROOT / SIBLING_FILE).is_file(): + sib_docs = [d for d in load(SIBLING_FILE) if d] + sib_cm = next((d for d in sib_docs if d.get("kind") == "ConfigMap"), None) + sib_script = next(iter((sib_cm.get("data") or {}).values()), "") if sib_cm else "" + sib_allow = tuple_literal(sib_script, "_ALLOWED") or () + check(bool(sib_allow), + f"{SIBLING_FILE}: could not parse its _ALLOWED list, so this check cannot " + f"prove the sibling was not widened") + if sib_allow: + # The sibling's list is PATH-ONLY by design. Its selftest asserts it refuses the + # whole-realm endpoint and the /authentication tree; assert the same here so a + # future edit that "just adds the realm endpoint there instead" is caught. + bad = [p for p in sib_allow + if isinstance(p, str) + and (re.fullmatch(r"/admin/realms/\[\^/\]\+", p) + or "authentication" in p or "/users" in p or "/groups" in p + or "/roles" in p)] + check(not bad, + f"{SIBLING_FILE}: its allow-list was WIDENED to {bad}. That reconciler's " + f"stated guarantee — prod's real user and the media-users / " + f"platform-admins / viewers groups survive every run — is expressed as " + f"endpoints it CANNOT reach, and its own --selftest asserts the refusal of " + f"/users, /groups, /roles, /authentication and the whole-realm endpoint. " + f"The passkey work is a SIBLING program with its own narrow list precisely " + f"so that guarantee stays literally true. Revert the widening") + +# ============================== (8) the reconciler's BEHAVIOUR, proven by its selftest +if script: + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "reconcile.py" + p.write_text(script) + proc = subprocess.run([sys.executable, str(p), "--selftest"], + capture_output=True, text=True, timeout=300) + tail = (proc.stdout + proc.stderr).strip().splitlines() + check(proc.returncode == 0, + "the passkey reconciler's --selftest FAILED (compare-then-write / " + "idempotence / strict read-modify-write / the RpEntityName gate / " + "flow-is-unwritable / the conditional-OTP regression / fail-closed):\n" + " " + "\n ".join(tail[-14:])) + if proc.returncode == 0: + print(" passkey reconciler --selftest: " + (tail[-1] if tail else "(none)")) + + # ---- (8b) THE SELFTEST IS FALSIFIABLE, not decorative ---------------------- + # A passing suite proves nothing unless it can be shown to FAIL. mutations.py + # injects plausible regressions one at a time -- a partial policy PUT, a + # rebind of browserFlow, a widened allow-list that makes the flow writable, a + # fail-open on an unreadable input -- and requires --selftest to catch every + # one. This matters more than usual here: the whole incident class is a + # component that REPORTED SUCCESS while the realm was broken, and a green test + # that cannot fail is the same lie one level up. + mut = DIR_PATH / "mutations.py" + if mut.is_file(): + mproc = subprocess.run([sys.executable, str(mut), str(p)], + capture_output=True, text=True, timeout=900) + mtail = (mproc.stdout + mproc.stderr).strip().splitlines() + check(mproc.returncode == 0, + "the passkey reconciler's --selftest is NOT FALSIFIABLE — at least " + "one injected regression survived it, so the suite does not prove " + "what it claims:\n " + "\n ".join(mtail[-16:])) + if mproc.returncode == 0: + print(" passkey reconciler mutations: " + + (mtail[-1] if mtail else "(no output)")) + else: + check(False, + "conformance/tests/l0-static/keycloak-passkey-reconciler/mutations.py " + "is missing — nothing proves the --selftest can fail, and an " + "unfalsifiable proof is the failure mode this component exists to end") + +# ======================================================= (9) this check is load-bearing +L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else "" +GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else "" +TOUCHED = ((ROOT / "ci" / "touched-requirements.sh").read_text() + if (ROOT / "ci" / "touched-requirements.sh").is_file() else "") +SELF_YAML = DIR_PATH / "chainsaw-test.yaml" +SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else "" +check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) + is not None, + "keycloak-passkey-reconciler/chainsaw-test.yaml does not declare " + 'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check ' + "only when a change happens to attribute one of the requirements it is labelled " + "with. This invariant spans platform/, platform-config/, ci/ and clusters/, so a " + "change that breaks it need not attribute R-IDP-03/04 — which is how 12 unusable " + "prod OIDC clients sat behind an entirely green cluster on 2026-09-06") +check("--include-scope" in L0, + "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 " + "stage is a purely requirement-scoped run again and the scope annotation is inert " + "decoration — this check would then run only by coincidence") +check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) + and re.search(r"fail\).*BLOCK=1", GATE), + "ci/gate.sh does not set the merge block when the l0 stage fails") +check(re.search(r"^\s*platform/keycloak/\*\)", TOUCHED, re.M), + "ci/touched-requirements.sh has no platform/keycloak/* arm, so editing the realm " + "import or the reconciler would attribute NO requirement and the keycloak tests " + "would be filtered out of the merge gate's touched subset") + +# ============================================================================= +if failures: + print("CHECK FAIL: %d assertion(s) failed" % len(failures), file=sys.stderr) + for f in failures: + print(" FAIL %s" % f, file=sys.stderr) + raise SystemExit(1) +print("keycloak-passkey-reconciler: all assertions passed " + "(desired state derived from the realm import, one substitution pass, write " + "surfaces agree, the BOUND flow is unwritable and fail-closed, nothing is " + "deletable, sibling allow-list unwidened, selftest green and falsifiable)") +PY diff --git a/conformance/tests/l0-static/keycloak-passkey-reconciler/mutations.py b/conformance/tests/l0-static/keycloak-passkey-reconciler/mutations.py new file mode 100644 index 0000000..b552bf1 --- /dev/null +++ b/conformance/tests/l0-static/keycloak-passkey-reconciler/mutations.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +# ============================================================================= +# mutations.py -- prove the passkey reconciler's --selftest IS FALSIFIABLE. +# +# WHY THIS EXISTS +# --------------- +# A selftest that passes proves nothing on its own: a selftest asserting `True == +# True` also passes. This reconciler exists because a one-shot realm import +# REPORTED SUCCESS while the realm was never converged (Done=True / Started=False +# with the whole passkey block inert, measured on prod 2026-10-18), and the same +# trap applies one level up -- a green test suite that cannot fail is the same lie +# in a different place. +# +# It matters more here than for the sibling client reconciler, because this program +# CONSTRUCTS AUTHENTICATION FLOWS and then REBINDS THE REALM'S LOGIN PATH. An +# untested safety property on that surface is not a documentation gap, it is an +# outage waiting for a schedule tick. +# +# So this injects a series of PLAUSIBLE regressions into the shipped script, one at +# a time, and requires that --selftest FAILS for every one. A mutation that +# SURVIVES means the suite does not actually check that property, and the check +# fails, naming it. +# +# THE FIRST SIX ARE THE BLUE/GREEN SAFETY ARGUMENT ITSELF. The design builds a new +# flow under an unused alias, verifies it, and flips `browserFlow` in one atomic +# write -- never mutating the flow users are logging in through. Each of those +# properties is enforced in code; these mutations are the proof that the enforcement +# is load-bearing rather than commentary. Several model the obvious "simplification" +# someone will propose: +# * "just build it under the declared alias, the suffix is ugly" +# * "bind it first, then check -- the check is just belt and braces" +# * "the protected-alias screen is redundant, the allow-list covers it" +# Every one of those would be a total SSO outage waiting to happen. +# +# ANCHORS ARE AUTHORED AGAINST THE EXTRACTED SCRIPT, not against the YAML manifest. +# The sibling mutations.py dedents by four because its anchors are copied out of the +# ConfigMap block scalar; these are copied out of the script as `python3 -` sees it, +# which is one less transformation to get wrong. check.sh passes the extracted file, +# so what is proven falsifiable is byte-for-byte what runs on the cluster either way. +# +# python3 mutations.py +# +# Runtime is ~60s (one subprocess per mutation, each running the full selftest +# against an in-process stub Keycloak; no network, no cluster). +# ============================================================================= +from __future__ import annotations + +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +# (name, the mistake being modelled, anchor, replacement) +MUTATIONS: list[tuple[str, str, str, str]] = [ + # ---------------------------------------------------------------- blue/green safety + ("build INTO the currently bound flow", + "'the suffix is ugly, just build under the declared alias' -- every " + "create-execution and set-requirement call would then land on the flow users are " + "logging in through, and a run dying part-way leaves a HALF-BUILT LOGIN PATH for " + "every SSO app and kubectl. This is the single regression the whole design exists " + "to make impossible", + " def mapped(alias):\n" + " return \"%s-%s\" % (alias, suffix)", + " def mapped(alias):\n return alias"), + + ("the protected-subtree screen is never called", + "'the allow-list already covers it' -- it does not: the allow-list's write entries " + "are alias-parameterised, so only this screen can say 'but not the flow that is " + "bound'. Without it the reconciler may write anywhere in the authentication tree", + " _screen_write(method, path, body if isinstance(body, dict) else None)", + " pass"), + + ("the per-run reset makes the protected sets EMPTY, not UNDETERMINED", + "reset them to frozenset() so the code reads more simply -- it silently removes " + "the fail-closed property, so a build that ran before the live binding had been " + "read would be PERMITTED, and could write straight into the live flow. (The " + "module-level initial value is deliberately NOT the anchor here: run() resets " + "these at the start of every run, so the initialiser is unreachable and mutating " + "it would prove nothing.)", + " global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS\n" + " _PROTECTED_FLOW_ALIASES = None\n" + " _PROTECTED_EXECUTION_IDS = None", + " global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS\n" + " _PROTECTED_FLOW_ALIASES = frozenset()\n" + " _PROTECTED_EXECUTION_IDS = frozenset()"), + + ("bind BEFORE verifying the build", + "'bind it first, the readback is just belt and braces' -- it is not: the readback " + "is what catches a build that was accepted but came out wrong, including the " + "conditional-OTP config landing under the wrong key. Binding first makes that the " + "live login path before anyone notices", + " verify = compare_shape(final_shape, expected, base, realm, token, configs,\n" + " \"the newly built flow %r\" % target)", + " verify = []"), + + ("a failed build is reported as SUCCESS", + "swallow the build error and return a converged result -- the realm is then left " + "on the old flow (which blue/green makes safe) but the run says it converged, so " + "nobody ever learns the passkey flow was never built. That is the one-shot " + "import's fail-open, reintroduced. NOTE: simply removing the early return here " + "is NOT a regression -- verify-before-bind catches a half-built flow anyway, which " + "is defence in depth working as intended -- so the mutation targets the reporting", + " problems.append(\n" + " \"flow: building %r FAILED (%s). Nothing was bound, so login is \"\n" + " \"UNAFFECTED -- the old flow is still live. The partial flow is left in \"\n" + " \"place, unbound and harmless\" % (candidate, e))\n" + " return 0, problems", + " return 1, []"), + + ("an existing candidate flow is reused WITHOUT verifying it", + "'if the alias exists we already built it' -- a half-built leftover from a crashed " + "run lives under exactly that alias, so this binds a partially constructed login " + "path", + " if cand_shape is not None and not compare_shape(\n" + " cand_shape, expected, base, realm, token, configs,\n" + " \"the existing flow %r\" % candidate):", + " if cand_shape is not None:"), + + # ------------------------------------------------------------------- idempotence + ("sub-flow aliases are compared as names", + "compare the sub-flow alias like any other field -- the flows this program builds " + "carry a hash suffix on every alias, so a built flow could NEVER be recognised as " + "converged: it would be rebuilt and the login path reflipped every fifteen " + "minutes, forever", + " out.append((level, ex[\"kind\"], \"\" if is_flow else ex[\"name\"],\n" + " ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))", + " out.append((level, ex[\"kind\"], ex[\"name\"],\n" + " ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))"), + + ("the converged short-circuit is removed", + "drop the 'bound flow already matches git' early return -- every run rebuilds and " + "reflips, which is a permanent write loop against the IdP and churns the realm's " + "login path on a schedule", + " if not drift:\n" + " log(\"flow ok (the bound flow %r matches git: %d execution(s), no \"\n" + " \"write)\" % (bound, len(expected)))\n" + " return 0, problems", + " if False:\n return 0, problems"), + + ("the execution priority is dropped from the requirement write", + "omit `priority` from the body -- Keycloak's updateExecutions deserialises a " + "missing int as 0 and compares it against the stored priority, so this RESHUFFLES " + "the execution order while only a requirement change was intended. The passkey " + "branch would end up in the wrong place in the tree", + " body={\"id\": ex.get(\"id\"), \"requirement\": want_req,\n" + " \"priority\": ex.get(\"priority\") or 0})", + " body={\"id\": ex.get(\"id\"), \"requirement\": want_req})"), + + ("the alias budget is unbounded", + "loop forever looking for a free alias -- a persistently failing build then creates " + "unbounded flow sprawl in the realm, and nothing here can delete any of it", + " for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):", + " for attempt in range(1, 10000):"), + + ("fail-open when the bound flow cannot be read", + "carry on when the live flow's executions are unreadable -- the program then cannot " + "know which flows are live, so it cannot prove a build would miss them, and it " + "would build with the protected sets still undetermined", + " return 0, [\n" + " \"flow: the realm binds browserFlow to %r but its executions could not \"\n" + " \"be read, so this program cannot establish which flows are live. Flow \"\n" + " \"writes stay FAIL-CLOSED for this run -- nothing was built or bound\"\n" + " % bound]", + " _set_protected(set(), set())\n bound_shape, bound_raw = None, None"), + + # ------------------------------------------------------- the conditional-OTP regression + ("authenticator config contents no longer compared", + "check only that a config is BOUND, not what it says -- this is THE conditional-OTP " + "regression: the original bug wrote the role under " + "`condition.evaluated.user.role` instead of `condUserRole`, so the role resolved " + "to null, the REQUIRED conditional step threw 'Invalid role name submitted: null' " + "and every platform-admin got a 403 from every SSO-gated app", + " if str(have_cfg.get(key)) != str(want_cfg[key]):", + " if False:"), + + ("undeclared config keys are ignored", + "stop reporting keys the realm holds and git does not -- the stale " + "`condition.evaluated.user.role` key is the signature of the original null-role " + "bug, and a flow carrying both keys would look correct", + " extra = sorted(k for k in have_cfg if k not in want_cfg)", + " extra = []"), + + ("flow structure divergence detected but not reported", + "leave the per-step comparison out -- the realm's login flow could be the old " + "pre-passkey shape forever and every run would report the realm converged, which " + "is the exact fail-open of the one-shot import this program replaces", + " if observed[idx][:4] != expected[idx][:4]:", + " if False:"), + + ("the browserFlow BINDING is no longer compared", + "stop noticing that a correct flow is not the one bound -- a perfect, fully-built " + "passkey flow that nothing logs in through changes nothing at all", + " if bound_shape is not None:\n" + " drift = compare_shape(bound_shape, expected, base, realm, token, configs,", + " if False:\n" + " drift = compare_shape(bound_shape, expected, base, realm, token, configs,"), + + # ------------------------------------------------------------- realm / policy writes + ("partial policy PUT instead of read-modify-write", + "send only the drifted policy keys -- Keycloak 26.0.6 rebuilds the WHOLE " + "passwordless policy from the representation and substitutes its own DEFAULT for " + "every key the body omits, so a partial PUT silently resets everything it did not " + "mention while returning 204", + " body = dict(realm_rep)\n" + " for key in drift:\n" + " body[key] = wanted[key]", + " body = {RP_ENTITY_NAME: realm_rep.get(RP_ENTITY_NAME),\n" + " BROWSER_FLOW: realm_rep.get(BROWSER_FLOW)}\n" + " for key in drift:\n" + " body[key] = wanted[key]"), + + ("the policy pass rebinds browserFlow as a side effect", + "'while we are here, let the policy write fix the flow binding too' -- the realm " + "PUT does read rep.getBrowserFlow(), so this is genuinely in reach, and it would " + "rebind the login path to an UNVERIFIED flow from a pass whose stated surface is " + "the passkey policy", + " if body.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW):\n" + " raise RuntimeError(", + " body[BROWSER_FLOW] = \"browser\"\n" + " if False:\n" + " raise RuntimeError("), + + ("the RpEntityName gate removed from desired-state validation", + "accept a desired state with an empty RpEntityName -- Keycloak 26.0.6 applies the " + "passwordless policy ONLY when that field is non-empty, so every realm PUT would " + "be accepted with 204 and change nothing: reported success, zero convergence", + " if not policy.get(RP_ENTITY_NAME):\n" + " raise ValueError(", + " if False:\n raise ValueError("), + + ("the RpEntityName gate removed from the write path", + "drop the second, defence-in-depth guard -- it is unreachable while the " + "desired-state validation holds, which is exactly why deleting it would otherwise " + "be an invisible regression", + " if not body.get(RP_ENTITY_NAME):\n" + " raise RuntimeError(", + " if False:\n raise RuntimeError("), + + ("no readback verification on the policy write", + "trust the 204 -- a server that accepts the PUT and does not apply it is reported " + "as REPAIRED, which is 'reports success while the realm never converged' again, " + "and it is precisely what the RpEntityName gate does when it is missed", + " still = policy_drift(after, wanted)", + " still = {}"), + + ("AAGUID comparison made order-sensitive", + "compare the AAGUID allow-list in the order Keycloak returns it -- an allow-list " + "is a set membership test, so an order-sensitive comparison rewrites a CONVERGED " + "realm on every run forever while every run still reports success", + " if stored != sorted(declared_value):\n" + " drift[key] = (stored, sorted(declared_value))", + " if stored != declared_value:\n" + " drift[key] = (stored, declared_value)"), + + ("signature-algorithm order treated as insignificant", + "sort the signature algorithms before comparing -- it is a PREFERENCE LIST handed " + "to the authenticator, so ES256-then-RS256 is not the same declaration as " + "RS256-then-ES256, and a reorder in the realm would never be corrected", + " if stored != list(declared_value):", + " if sorted(stored) != sorted(declared_value):"), + + ("unmanaged policy keys forced to a value", + "iterate every writable key instead of only the declared ones -- a " + "webAuthnPolicyPasswordless field the realm holds and git does not declare would " + "be overwritten, turning 'not declared' into 'must equal my default'", + " for key in sorted(wanted):\n declared_value = wanted[key]", + " for key in POLICY_FIELDS:\n declared_value = wanted.get(key, \"\")"), + + # ------------------------------------------------------------------ required actions + ("required-action PUT sends only the two drifted fields", + "a minimal body looks safer -- but updateRequiredAction REPLACES the provider from " + "the representation, so name, priority and config are all blanked: the enrolment " + "action keeps working but loses its priority and display name", + " body = dict(rep)\n for key in drift:\n body[key] = wanted[key]", + " body = {}\n for key in drift:\n body[key] = wanted[key]"), + + ("the guarded default required actions are no longer audited", + "stop checking that CONFIGURE_TOTP / UPDATE_PASSWORD / delete_credential still " + "exist -- their disappearance is the signature of a requiredActions block in the " + "realm import, which 26.0.6 treats as REPLACE, and it is a silent permanent " + "lock-out of every platform-admin", + " missing = [a for a in GUARDED_ACTIONS if a not in aliases]", + " missing = []"), + + # ----------------------------------------------------------------- inputs / coupling + ("the three passes are coupled", + "abort the run at the first failing pass -- a policy write that fails for an " + "unrelated reason would then suppress the browser-flow pass, which is the most " + "safety-critical thing here", + " problems.extend(found)\n if found:\n rc = 1", + " problems.extend(found)\n if found:\n rc = 1\n" + " break"), + + ("policy-key validation removed from the desired state", + "accept any realm key from the ConfigMap -- an arbitrary realm attribute becomes " + "writable through a file edit, so a hand-edited ConfigMap could set sslRequired, " + "bruteForceProtected or any other top-level realm field on the live IdP", + " if key not in POLICY_FIELDS:", + " if False:"), + + ("fail-open on an unreadable desired state", + "treat a missing ConfigMap as 'nothing declared' -- the run reports success while " + "reconciling nothing, which is precisely the fail-open of the realm import this " + "whole component exists to replace", + " if not isinstance(raw, dict) or not raw:\n" + " raise ValueError(\"%s does not hold a non-empty JSON object\" % path)", + " if not isinstance(raw, dict) or not raw:\n return {}"), + + ("an undeclared authenticatorConfig reference is tolerated", + "stop validating that every configAlias a flow references is actually declared -- " + "the build then fails at the config-binding step, AFTER flows and executions have " + "already been created, leaving leftovers on every run", + " if ca and ca not in configs:\n" + " raise ValueError(", + " if False:\n raise ValueError("), + + ("the flow-cycle guard is removed", + "drop the `seen` check -- a cyclic flow declaration then recurses until the " + "interpreter dies, in a CronJob that retries", + " if alias in seen:\n" + " raise ValueError(\"flow %r is cyclic in the desired state\" % alias)", + " if False:\n" + " raise ValueError(\"flow %r is cyclic in the desired state\" % alias)"), +] + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: mutations.py ", file=sys.stderr) + return 2 + src = Path(sys.argv[1]).read_text() + + problems: list[str] = [] + killed = 0 + for name, why, old, new in MUTATIONS: + # An anchor that no longer matches is a FAILURE, not a skip: it means the code + # moved and this mutation silently stopped proving anything. + n = src.count(old) + if n != 1: + problems.append("%s: anchor matches %d times (expected exactly 1) -- the " + "code moved and this mutation no longer proves anything. " + "Re-anchor it against the current source." % (name, n)) + continue + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh: + fh.write(src.replace(old, new, 1)) + path = fh.name + try: + proc = subprocess.run([sys.executable, path, "--selftest"], + capture_output=True, text=True, timeout=300) + finally: + Path(path).unlink(missing_ok=True) + if proc.returncode == 0: + problems.append("%s SURVIVED -- --selftest still passed with this regression " + "applied, so nothing in the suite actually checks it. The " + "mistake it models: %s" % (name, why)) + continue + killed += 1 + out = proc.stdout + proc.stderr + m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out) + how = ("%s assertion(s) caught it" % m.group(1) if m + else "the selftest aborted (caught, less cleanly)") + print(" killed %-54s %s" % (name, how)) + + print() + if problems: + print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven" + % (len(problems), len(MUTATIONS)), file=sys.stderr) + for p in problems: + print(" FAIL %s" % p, file=sys.stderr) + return 1 + print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest " + "(the suite is falsifiable, not decorative)" % killed) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/decisions/2026-10-18-keycloak-passkey-reconciler.md b/decisions/2026-10-18-keycloak-passkey-reconciler.md new file mode 100644 index 0000000..ca7e2da --- /dev/null +++ b/decisions/2026-10-18-keycloak-passkey-reconciler.md @@ -0,0 +1,334 @@ +--- +date: 2026-10-18 +branch: feat/keycloak-passkey-reconciler +title: "Make the passkey realm config converge via GitOps — policy, enrolment action and the browser flow, the flow blue/green so the bound login path is never mutated" +--- + +## What changed + +**Why this is a new branch and not an update to PR #72.** The brief for this work said to extend +PR #72 (`feat/keycloak-passkeys`) in place and not open a new PR. That premise was already stale +when I started: **PR #72 merged at 2026-10-18T05:44:04Z** (Gitea API: `state: closed`, +`merged: true`, `merge_commit_sha: 4d489f2`), and `git log origin/main` carries +`4d489f2 Merge pull request '…' (#72)`. A merged PR cannot be updated, and CLAUDE.md §1 leaves +exactly one path to `main`. So this is a fresh branch from `origin/main` in its own worktree, and a +new PR. For the same reason this is a **new** `decisions/` entry rather than an edit to +`decisions/2026-10-17-keycloak-passkeys.md`, which is now merged history describing a different unit +of work; it is the entry to read first for the passkey *design* decisions, which are not repeated +here. + +**The problem.** PR #72 declared the WebAuthn passwordless policy and a restructured `browser-mfa` +flow in `platform/keycloak/keycloak-realm-import.yaml`. That declaration **could not take effect**. +A `KeycloakRealmImport` is a one-shot; prod's reads `Done=True / Started=False / HasErrors=False`, +so the operator will never re-run it and every realm-level field or flow declared after the first +import is inert — in git, applied by Flux, reported Ready, reaching the realm never. Merging it +changed nothing about how anyone logs in, and nothing anywhere said so. Same failure class as the +2026-09-06 client-secret incident, third field over. + +**What now makes it converge.** A **sibling** reconciler, modelled on +`platform/keycloak/client-secret-reconciler.yaml`: + +- `platform/keycloak/passkey-reconciler.yaml` — ConfigMap (Python, stdlib only, **zero dollar + signs**) + CronJob on `*/15 * * * *`, `automountServiceAccountToken: false`, admin credential by + `secretKeyRef` only. Three passes, **all of which converge**: + 1. **WebAuthn passwordless policy.** Strict read-modify-write of `GET`/`PUT /admin/realms/homelab`, + changing only the `webAuthnPolicyPasswordless*` keys that differ, readback-verified. + 2. **`webauthn-register-passwordless` required action.** Strict read-modify-write of one + **literally-named** alias; converges `enabled: true` / `defaultAction: false` only. + 3. **The browser authentication flow — BLUE/GREEN.** Build the declared tree under a new, unused + alias; verify it by readback; flip `browserFlow` in **one atomic PUT**. +- `ci/render-keycloak-passkey-policy.py` + the **generated** + `platform/keycloak/passkey-desired-state.yaml` — the desired state, rendered from the realm + import, `--check`ed by the merge gate. +- `conformance/tests/l0-static/keycloak-passkey-reconciler/` — `check.sh`, `chainsaw-test.yaml` + (`scope: whole-repo`), `mutations.py`. +- `platform-config/kustomization.yaml` — both new resources, in the **same** Kustomization as the + realm import. +- `platform/keycloak/README.md` — the layout table, the section heading, and the "does NOT reach the + live realm" section rewritten: all three declared things now converge, with the blue/green + mechanics, the rollback command and the `--no-bind` fallback written out. + +### The flow design was reconsidered, and the first version of it was wrong + +**This is worth recording because the superseded reasoning was not silly, it was just wrongly +scoped.** The first design of pass 3 was **audit-only**: compare the live browser flow against git, +report divergence loudly, and never write it — leaving the restructure as a one-time admin-console +action. The reasoning was that converging a five-flow / ten-execution tree needs create-subflow, +create-execution, set-requirement and bind-config as *separate* untransacted admin-API calls, so a +run dying between them leaves a **half-built flow bound as the login path** for every SSO-gated app +*and* kubectl — and this repo has had a total SSO outage from a realm change (2026-09-06), with the +conditional-OTP sub-flow (which the declared structure moves one level deeper) once 403ing every +platform-admin. + +**That risk analysis was sound but it assumed the wrong shape.** It is true only if you mutate the +*bound* flow. The operator's instruction was *"if this can be done through GitOps, it should"*, and +audit-only did not meet it — a manual admin-console step is exactly what the one-shot import already +forces. So: + +- All the multi-call construction happens on a flow **nothing is using** (the declared alias plus a + hash of the desired shape, applied to *every* alias in the tree — reusing `browser-mfa-forms` + would mean writing *into* the bound tree). A failure at any point leaves an orphaned, unbound flow + and **zero user-visible effect**. The untransacted-calls problem dissolves. +- The built flow is **read back and verified** — nesting levels, ordering, authenticators, + requirements, and the conditional-OTP config *contents* — **before** anything is bound. The + regression the operator feared is now caught *before* it can bite rather than after. +- The flip is **one atomic PUT** of the realm setting `browserFlow`. +- **Rollback is one write.** The previous alias is printed in the run log, and there is **no + `DELETE` anywhere in the allow-list**, so the old flow survives as a working rollback target. + +**The new safety property, enforced structurally rather than by intention.** Blue/green needs +flow-create and execution-add endpoints, which widens the allow-list — and the allow-list alone +cannot express "but not the flow that is bound", because its write entries are alias-parameterised. +So a second guard sits at the same choke point in `_http()`: the live `browserFlow` alias and its +**entire subtree** of sub-flow aliases and execution ids are read first, and every write is screened +against those sets before a socket opens. Protecting the whole subtree matters — adding an execution +to `browser-mfa-forms` changes live login behaviour just as surely as touching `browser-mfa`. It is +**fail-closed**: the sets start each run as "not determined", and while they are, *every* flow write +is refused; if the bound flow's executions cannot be read they stay undetermined and the flow pass +writes nothing. + +**Idempotence became load-bearing rather than merely nice.** A rebuild-and-reflip every 15 minutes +would churn the login path on a schedule, so the bound flow is compared *structurally* and a match +produces **zero writes and no new flow**. Sub-flow and config *aliases* are deliberately not +compared: a flow this program built carries a hash suffix, so comparing labels would make it +permanently unrecognisable as converged. A crashed run's completed-but-unbound flow is **reused**; a +half-built one is left alone and the next candidate alias is used, with the candidate count +**bounded** so a persistently failing build fails loudly instead of creating unbounded sprawl. + +**Where the rollback target is recorded, and why not as an annotation.** The brief suggested +recording the previous `browserFlow` alias as an annotation on the CronJob or in a ConfigMap. The +pod **cannot**: it runs with `automountServiceAccountToken: false` and has no Kubernetes API access +at all, which is the same property that made the mounted-generated-ConfigMap design preferable to +reading the CR (and is why this change adds no `**/rbac.yaml`). Giving it write access to its own +annotations would trade a narrow, gate-verified read for a Kubernetes API writer, which is a worse +bargain than the one it would buy. + +So the record is in two places that cost nothing: the **run log** prints +`ROLLBACK is one write: set browserFlow back to . The old flow was NOT deleted.`, and — more +durably — **the realm itself**, because nothing is ever deleted. Every superseded flow is still +listed under `Authentication -> Flows`, so the rollback target is discoverable from the admin console +even if every Job log has rotated away. The README carries the rollback procedure. + +**I achieved full convergence including the bind.** `--no-bind` exists as the deliberate fallback +(build and verify, leave the flip as a one-write human act) but it is not the default and was not +needed. + +**No `**/rbac.yaml` was touched.** The brief floated having the reconciler read the +`KeycloakRealmImport` CR from the Kubernetes API, which would need a Role, a RoleBinding and a +ServiceAccount token. I rejected that for the narrower, already-proven pattern: a desired state +**generated** from the import and **mounted**, so the pod keeps `automountServiceAccountToken: +false` and no Kubernetes API access at all. Skew is prevented by the gate running the renderer with +`--check`, not by a second declaration being trusted. + +**Sensitive paths touched** — checked against `infra/gitea/approval-policy.json` rather than from +memory, which caught one I had initially missed. **Three**: + +| Path | Why it is in the policy | What I changed | +|---|---|---| +| `ci/**` | the pipeline itself | added `ci/render-keycloak-passkey-policy.py` | +| `conformance/**` | changing the tests changes what green *means* | added the L0 test directory | +| `platform-config/kustomization.yaml` | Flux wiring / which manifests are actually applied | added the two new resources | + +**Not** touched: `**/rbac.yaml`, `clusters/**`, `platform/kyverno/**`, `.gitea/**`, `bootstrap/**`, +`infra/gitea/**`, `.sops.yaml`, any `*.sops.yaml`, `CLAUDE.md`. + +So this PR **requires an approving review from someone who is not its author**, and +`ci/approval-policy.sh` is fail-closed, so it will block until that exists. I did **not** post +`/approve-override` — CLAUDE.md §1 names doing that on your own agent PR as exactly the +self-approval the stage exists to prevent. The operator needs to either review it or post the +override. + +## What I verified + +**Measured against the live cluster** (read-only `kubectl --context k3s-prod`): + +- `keycloakrealmimport homelab`: `Done=True Started=False HasErrors=False` — the import is a + completed one-shot. **This is the premise of the whole change and it is measured, not assumed.** +- Keycloak image actually running: `quay.io/keycloak/keycloak:26.0.6`. +- `kustomization/platform-config` `.status.lastAppliedRevision` == + `gitrepository/flux-system` artifact revision == `main@sha1:83d3568…`, which contains the PR #72 + merge. So the inert declaration is genuinely applied and genuinely inert. + +**`PUT /admin/realms/{realm}` does not touch users, groups, clients or roles in 26.0.6** — +established from source at tag `26.0.6`, not from documentation or memory: + +- `RealmAdminResource.updateRealm` is the only handler. Its javadoc: *"Any user, roles or client + information in the representation will be ignored. This will only update top-level attributes of + the realm."* +- It delegates to `RepresentationToModel.updateRealm` → + `DefaultExportImportManager.updateRealm` (`model/storage-private/.../datastore/`), **lines + 725–894**. I brace-matched that body and enumerated every `rep.getX()` call in it: it contains + **no** `getUsers()`, `getClients()`, `getGroups()`, `getRoles()`, `getRequiredActions()`, + `getAuthenticationFlows()` or `getAuthenticatorConfig()`. Those objects are **unreachable** from + the endpoint, not merely ignored by convention. +- Two findings that changed the design, both from the same read: + - `updateRealm` **does** read `rep.getBrowserFlow()`, so the endpoint can rebind the realm's login + path. The write therefore carries `browserFlow` straight back from the GET and a hard assertion + refuses the PUT if it would differ. + - The passwordless policy is applied **all-or-nothing and only when `RpEntityName` is non-empty** + (line 842), and `getWebAuthnPolicyPasswordless(rep)` substitutes a **Keycloak default** for every + null field (line 1253ff). A partial PUT would therefore silently reset every key it omitted, and + one missing `RpEntityName` would drop the whole policy while returning 204. This is why the write + is a strict read-modify-write of the full live representation. +- `AuthenticationManagementResource.updateRequiredAction` (line 1205) is **per-alias** and likewise + **replaces** the provider from the representation (name, alias, priority, enabled, defaultAction, + config), and 404s when the alias is unregistered. Hence read-modify-write there too. +- `DefaultRequiredActions` registers `webauthn-register-passwordless` with `enabled=true`, + `defaultAction=false`, `priority=80` — confirming the realm import's claim that declaring it would + restate a default while deleting ten siblings. +- **The realm GET really does return what the read-modify-write needs.** This is load-bearing: if + the GET omitted the passwordless keys, the PUT body would be missing them and 26.0.6 would reset + them to defaults. `ModelToRepresentation.toRepresentation` (lines 471–482) sets **all ten** + `webAuthnPolicyPasswordless*` fields plus `ExtraOrigins`, and line 496 sets `browserFlow`. I + checked for an `internal`/`export` guard around that block — there is none in lines 380–500, so + those fields are emitted unconditionally, not only on the export path. + +**The five flow-construction endpoints, read from 26.0.6 source before being used:** + +- `POST /authentication/flows` — `AuthenticationManagementResource.createFlow`, rejects a duplicate + alias. Carries **no alias in its path** (the alias is in the body), which is why it needed its own + body-based screen; without that it escaped the protected-subtree guard entirely. My selftest + caught that, not my reading. +- `POST /flows/{alias}/executions/flow` — `addExecutionFlow`, body `{alias, type, description}`. + Creates the sub-flow **at requirement DISABLED**, and refuses if the parent `isBuiltIn()`. +- `POST /flows/{alias}/executions/execution` — `addExecutionToFlow`, body `{provider}`. Also + DISABLED unless the factory has exactly one requirement choice. Both assign `getNextPriority()`, + so **insertion order is the resulting order** — which is why the builder inserts depth-first in + declared order and then verifies by readback. +- `PUT /flows/{alias}/executions` — `updateExecutions`. **A trap I would have walked into:** it + compares `rep.getPriority()` against the stored priority, and a JSON body omitting `priority` + deserialises it as the int **0** — which would silently **reshuffle the whole execution order** + while only a requirement change was intended. Every requirement write therefore carries the + execution's current priority back. There is a mutation for it and a selftest assertion on the + resulting order. +- `POST /executions/{id}/config` — `newExecutionConfig`, creates an authenticator config and binds + it. Config aliases are suffixed like flow aliases, because 26.0.6 keeps them unique per realm and + the old flow already uses the declared alias. + +**Offline proofs, all green:** + +- The reconciler's `--selftest`, **extracted from the shipped ConfigMap** and run: **216 assertions + pass**. Its stub Keycloak models the 26.0.6 behaviours that bite — the RpEntityName gate, the + all-or-nothing policy rebuild, the required-action replace, the `updateExecutions` priority trap, + and flow construction where everything is created DISABLED — so these are tested against the real + semantics rather than a forgiving fake. +- **A build killed at four different points (after 1, 3, 6 and 9 construction writes) never binds + and never touches the live flow.** Each case asserts the realm is still bound to the old alias, + that no write targeted the bound subtree, and that every live flow survives. This is the assertion + that makes blue/green a property rather than a claim. +- **Verify-before-bind is exercised, not assumed.** Two stub sabotages — a server that accepts the + requirement writes and applies the wrong value, and one that writes the conditional-OTP role under + `condition.evaluated.user.role` — both end with the flow **built but NOT bound** and the old flow + still live. +- `mutations.py`: **all 30 injected regressions caught**, including the six that are the blue/green + safety argument itself (build into the bound flow; never call the screen; reset the protected sets + to empty; bind before verify; report a failed build as success; reuse an existing flow without + verifying it). +- Mutation testing again found real gaps in my own work, which I fixed rather than weakening the + mutations. Across the two designs: no readback coverage on the realm write, no coverage of an + undeclared policy key surviving, a mis-indented anchor, and — in this design — **no coverage at + all of the bound flow being unreadable**, which is the fail-closed path. It also showed me that + **two of my proposed mutations were not real regressions**: the module-level initial value of the + protected sets is dead code (`run()` resets them first), and removing the early return after a + build failure is covered by verify-before-bind anyway. I re-anchored both onto the properties that + *are* load-bearing rather than pretending the originals had been caught. +- The new L0 `check.sh`: rc 0, all assertions. It asserts the renderer `--check`; that the realm + import and the desired state are rendered by the **same** Flux Kustomization; that the renderer's + and reconciler's writable key tuples are identical (parsed out of both sources with `ast`); that + `defaultAction` is `false`; that the realm import declares **no** `requiredActions` block + (previously only a comment saying "DO NOT ADD ONE"); that the **sibling** reconciler's allow-list + was not widened; that **nothing is deletable** and the execution-/config-id-scoped write endpoints + stay off the list (they carry no flow alias, so they could not be screened); and — the + load-bearing one — that the **protected-subtree guard exists in the source, is wired into + `_http()`, resets to "undetermined" every run, screens the body-only create-flow endpoint, and + that the policy write still refuses to rebind `browserFlow`**. +- `ci/render-keycloak-passkey-policy.py --write` output round-trips: the generated JSON is accepted + by the reconciler's `declared()`, its 10 policy keys equal the reconciler's `POLICY_FIELDS`, and + it flattens to the exact 11-step tree the realm import declares, with + `conditional-otp-platform-admin` bound at level 4. +- `${BASE_DOMAIN}` is carried into the generated ConfigMap **verbatim**, and both files sit in + `platform-config`, so one substitution pass decides the RP ID. +- Zero dollar signs in the reconciler script (the Flux envsubst trap), verified on the committed + manifest. + +**The conditional-OTP regression is detectable — and now also preventable.** The comparison is +key-by-key on the authenticator config contents, and because it runs **before** the bind, a flow +carrying the bug never becomes the live login path. `--selftest` covers it three ways: the historical +`condition.evaluated.user.role` key injected into a *built* flow (built, verified, **not bound**, +old flow still live); the same bug already live in the *bound* flow (reported, then **fixed** by +building and binding a correct flow); and the config disappearing entirely. `mutations.py` deletes +the comparison and the undeclared-key report and requires the selftest to notice both. The suite +also covers a requirement flipped from `ALTERNATIVE` to `REQUIRED`, the un-restructured legacy flow, +a correct-but-unbound flow, and a cyclic declaration. + +**Gate:** see the gate section of the PR / the run quoted in my report — rc and per-stage results +are recorded there rather than paraphrased here. + +## What I did NOT verify + +- **The live realm's actual passkey state is entirely UNVERIFIED.** I never authenticated to the + Keycloak admin API. Doing so needs the `keycloak-admin-credentials` password, which CLAUDE.md §6 + forbids decrypting or printing, so I did not read the realm's current + `webAuthnPolicyPasswordless*` values, its required-action list, or its `browser-mfa` executions. I + therefore **do not know** how much drift the first real run will report, nor that the live flow is + in the "legacy shape" my selftest models — only that the import never re-ran, which makes drift + the overwhelmingly likely state. The first run's log is the measurement, and nobody has seen one. +- **Nothing has ever run against a real Keycloak.** Every behavioural proof is against an + in-process stub I wrote, and a stub I wrote cannot falsify my own reading of the source. This + matters more for the blue/green flow builder than for anything else in the change, because it is + **five endpoints I have never exercised**. Specifically unproven in reality: + - That `POST /flows/{alias}/executions/flow` and `.../executions/execution` accept the bodies I + send and assign priorities in insertion order. If they do not, the build produces a tree that + fails verification — which is the safe failure (nothing is bound, login unaffected), but it + means **the flow would never converge** and the Job would fail every 15 minutes. + - That `PUT /admin/realms/homelab` accepts a full round-tripped representation. The analogous + *client* endpoint rejects one with `400 "Cannot parse the JSON"` (measured 2026-09-06), and the + sibling reconciler's whole minimal-PUT rule exists because of it. The source says the realm + endpoint differs; no measurement here confirms it. **If it does not, both the policy pass and + the atomic bind fail closed** — loudly, writing nothing. + - The exact JSON shapes compared: `AcceptableAaguids` as `[]` rather than absent or null, + `CreateTimeout` as a JSON int, `RequireResidentKey` as the literal string `"Yes"`. A shape I did + not anticipate would show up as **perpetual drift** — a write every 15 minutes that never + converges — rather than as an error. + - That `authenticationConfig` on an execution-info response is the config **id** (I read it from + the representation class, but never saw a live response). +- **I did not verify the alias-collision behaviour against a real realm.** The recovery logic + assumes `POST /flows` returns a non-2xx on a duplicate alias (the source checks + `getFlowByAlias != null` and raises `ErrorResponse.exists`), and that a config alias collision + likewise fails. If either silently succeeded, the recovery path would behave differently than + modelled. +- **Flow sprawl is bounded but real.** Every genuinely new desired shape creates a new flow and + **nothing is ever deleted** — by design, because the old flow is the rollback target. Over many + shape changes the realm accumulates unbound flows. There is no cleanup, automated or documented + beyond "an operator deletes them". I consider that the right trade (deletion is the one operation + that could destroy a rollback target) but it is a cost, not a non-issue. +- **No L2 behavioural test was added.** The reconciler has an `--audit` mode built for one and the + sibling has `tests/l2-behavioral/keycloak-client-secrets-match-git`, but I added no equivalent. + And `vars.CLUSTER_AVAILABLE` is unset, so L2/L3 would not run in CI anyway. **CI proves the + offline subset only; it proves nothing about the live realm.** +- **No alerting.** Unlike the first design there is no longer any failure that is "expected by + design", so a PrometheusRule on `kube_job_status_failed` would now be meaningful — and I did not + add one. A silently failing CronJob (image pull failure, rotated admin password, changed Secret + key) is visible only to someone who looks at Jobs. This is a real gap, not a resolved one. +- **That passkey login works.** No passkey has been enrolled and no browser has been pointed at + this. Nothing offline can prove a WebAuthn ceremony completes, that the `ALTERNATIVE` fork + degrades to the password form via `CREDENTIAL_SETUP_REQUIRED`, or that a platform-admin is still + asked for TOTP on the password branch. The README's four-step manual check remains the only thing + that would establish it, and it remains undone. +- **That the RP ID is right for the origin.** `${BASE_DOMAIN}` is a legal registrable-domain suffix + of `keycloak.${BASE_DOMAIN}` on paper, but a wrong RP ID fails **in the browser**, before Keycloak + is involved, with no server-side log line. The reconciler will happily converge the realm on a + wrong RP ID; it checks that git and the realm agree, never that git is correct. +- **The first real run will change how everyone logs in.** Once this merges and Flux applies it, the + next tick builds a flow and — if it verifies — **flips the realm's browser flow**. The UX change + is deliberate and documented (username screen, then password), but it is the first time it will + have happened, unattended, on a schedule. The rollback is one write and the old flow survives; + that is the mitigation, and it has not been rehearsed either. +- **CI's own result.** My gate run is local. `gitea-settings` cannot run locally, and + `ci/approval-policy.sh` reports green N/A offline while this branch touches three sensitive paths, + so **the PR is guaranteed to need a non-author approving review** and I have not seen CI report. I + did not and will not post `/approve-override` on my own PR. +- I re-measured branch protection this session (`protected: true`, `user_can_push: false`, + `enable_status_check: true`, context `gate / gate (pull_request)`, `required_approvals: 0`), but + the branches endpoint does **not** expose `block_admin_merge_override`, so that row of CLAUDE.md + §2 is **not re-confirmed** by me. diff --git a/platform-config/kustomization.yaml b/platform-config/kustomization.yaml index e240fe2..cffc6bb 100644 --- a/platform-config/kustomization.yaml +++ b/platform-config/kustomization.yaml @@ -127,6 +127,27 @@ resources: # so a ${BASE_DOMAIN} token in either resolves to the same value and the two declarations # cannot disagree about the domain. - ../platform/keycloak/client-urls.yaml + # PASSKEYS — the same one-shot failure, for the WebAuthn passwordless policy and the + # browser login flow. Measured on prod 2026-10-18, hours after the passkey declaration + # merged: the KeycloakRealmImport read Done=True / Started=False with the whole + # webAuthnPolicyPasswordless* block and the restructured browser-mfa flow INERT. This + # CronJob converges the policy and the enrolment required action, and AUDITS the browser + # flow without the ability to rewrite it (no flow-mutating endpoint is on its + # allow-list) — a half-rewritten flow bound as the realm's browserFlow is a total SSO + # outage, and this repo has had one. See the file header for the full rationale. + - ../platform/keycloak/passkey-reconciler.yaml + # Its DESIRED STATE, GENERATED from keycloak-realm-import.yaml by + # ci/render-keycloak-passkey-policy.py and gated by the L0 check. Same reason as + # client-urls.yaml above: the reconciler pod has no Kubernetes API access + # (automountServiceAccountToken: false), so it cannot read the KeycloakRealmImport CR + # and its input must be MOUNTED. + # + # IT MUST BE RENDERED BY THIS SAME KUSTOMIZATION as the realm import above, for exactly + # the reason given for client-urls.yaml: webAuthnPolicyPasswordlessRpId is the + # ${BASE_DOMAIN} token, and a split across Kustomizations would let one token resolve to + # two different values. An RP ID is baked into every passkey at enrolment and can never + # be re-keyed, so a domain the two files disagree about is an unrecoverable credential. + - ../platform/keycloak/passkey-desired-state.yaml - ../platform/keycloak/k8s-oidc-rbac.yaml # maps OIDC groups -> cluster roles (kubectl SSO RBAC) - ../platform/keycloak/httproute.yaml - ../platform/keycloak/servicemonitor.yaml diff --git a/platform/keycloak/README.md b/platform/keycloak/README.md index a17424f..dee6b58 100644 --- a/platform/keycloak/README.md +++ b/platform/keycloak/README.md @@ -31,6 +31,8 @@ files pulled as remote kustomize bases (the kubevirt-stub pattern); Renovate mai | `keycloak-realm-import.yaml` (`KeycloakRealmImport`: realm/groups/clients/MFA) | **platform-config** | config | | `client-secret-reconciler.yaml` (CronJob: client secrets **and** client URLs ← git, every 15m) | **platform-config** | config | | `client-urls.yaml` (**GENERATED** ConfigMap: the client-URL desired state) | **platform-config** | config | +| `passkey-reconciler.yaml` (CronJob: WebAuthn passwordless policy + enrolment action + the **browser flow** ← git every 15m; the flow blue/green, never mutating the bound one) | **platform-config** | config | +| `passkey-desired-state.yaml` (**GENERATED** ConfigMap: the passkey desired state) | **platform-config** | config | | `httproute.yaml` (keycloak.home.keithjackson.dev) | **platform-config** | config | | `servicemonitor.yaml` (Prometheus scrape) | **platform-config** | config | @@ -154,7 +156,7 @@ NEW client, a new protocol mapper or a changed scope still needs the realm impor (it will not on its own) or a deliberate admin-API/`kcadm` change. `Done=True` is never evidence that a realm edit landed. -## Passkeys (WebAuthn Passwordless) — DECLARED IN GIT, NOT YET IN THE REALM (2026-10-17) +## Passkeys (WebAuthn Passwordless) — DECLARED IN GIT, RECONCILED INTO THE REALM (2026-10-18) `keycloak-realm-import.yaml` now declares passkey login: the realm's **WebAuthn Passwordless** policy (`webAuthnPolicyPasswordless*`), and a `browser-mfa` flow that offers a passkey as an @@ -170,7 +172,7 @@ comments, not repeated here. The headline choices: | Resident key | `Yes` | A non-discoverable credential is a second factor, not a passkey | | OTP after a passkey? | **No** — conditional OTP stays on the password branch | A UV-required passkey is already two factors, and a phishing-resistant pair at that | -### This change does NOT reach the live realm. At all. +### The declaration did not reach the realm — a reconciler now carries all of it (2026-10-18) Same one-shot failure this file already documents twice above, third field over. Measured on prod 2026-10-17: @@ -188,21 +190,100 @@ endpoint allow-list makes `/authentication` unreachable **by design** (see the t realm-level policy fields are not client fields. This is exactly the "a NEW client, a new protocol mapper or a changed scope still needs the realm import to actually re-run" case named above. -**So the manifest is the declared source of truth and a DR-rebuild input, and nothing more, until -an operator applies it by hand.** Two paths, both operator actions: +`passkey-reconciler.yaml` closes that gap the same way the client reconciler closed the +secret and URL halves: a CronJob on `*/15` that authenticates to the admin API and converges +declared state. It is a **sibling**, not a second pass of the client reconciler, precisely so that +reconciler's allow-list — and its guarantee that prod's real user and the `media-users` / +`platform-admins` / `viewers` groups are **structurally unreachable** — stays literally true. -1. **Admin console (expected path).** *Authentication → Policies → WebAuthn Passwordless Policy* - for the policy block, then *Authentication → Flows → browser-mfa* to rebuild the executions to - match the manifest. The flow edit is the fiddly half: split the existing Username Password Form - into Username Form + Password Form and add the two sub-flows exactly as declared. -2. **A deliberate re-import.** Deleting and recreating the `KeycloakRealmImport` re-runs it against - the **existing** realm. Do not reach for this casually — it is the operation whose failure mode - is the 2026-09-06 total SSO outage documented above, and it has not been rehearsed here. +Its desired state is `passkey-desired-state.yaml`, **generated** from the realm import by +`ci/render-keycloak-passkey-policy.py` and gated by +`conformance/tests/l0-static/keycloak-passkey-reconciler`, so the RP ID is declared exactly once. +That matters more here than for client URLs: an RP ID is baked into every credential at enrolment +and **can never be re-keyed**, so two declarations free to skew about the domain would be an +unrecoverable-credential bug, not a cosmetic one. -Either way the realm is then configured by hand and git is only the record of intent, which is a -live exception to R-IDP-03 (config as code) — not a new one, but one more instance of it. +| Declared thing | Who applies it | Why | +|---|---|---| +| `webAuthnPolicyPasswordless*` (realm policy) | **The reconciler. GitOps.** | Realm top-level attributes; strict read-modify-write of `GET`/`PUT /admin/realms/homelab` | +| `webauthn-register-passwordless` required action | **The reconciler. GitOps.** | Per-alias `PUT`, one literally-named alias; enabled, `defaultAction: false` | +| The `browser-mfa` flow restructure | **The reconciler. GitOps.** | Blue/green: build under a new alias, verify, flip `browserFlow` in one atomic PUT — see below | -### Verifying it actually works (nobody has, as of 2026-10-17) +#### The flow converges blue/green — it never mutates the flow you log in through + +Converging a five-flow / ten-execution restructure needs create-subflow, +create-execution, set-requirement and bind-config as **separate** admin API calls, with no +transaction. Doing that to the flow bound as `browserFlow` would mean a run dying part-way +leaves a **half-built login path** for Grafana, Gitea, Nextcloud, Headlamp, Semaphore, +PiHole, Crafty, the *arr stack *and* kubectl. This file already documents a total SSO +outage from a realm change, and the conditional-OTP sub-flow — which the declared +structure moves **one level deeper** than it sits today — once 403'd every platform-admin. + +That risk is real but it is **not inherent**: it follows only from mutating the *bound* +flow. So the reconciler does not. + +1. **Build** the declared tree under a **new, unused alias** — the declared alias plus a + hash of the desired shape, applied to every flow in the tree. Every multi-call write + lands on a flow nothing is using. A failure at any point leaves an orphaned, unbound + flow and **zero user-visible effect**. +2. **Verify** by readback — nesting levels, ordering, authenticators, requirements, and + the conditional-OTP config *contents*. Nothing is bound unless this passes, so the + regression that once 403'd every platform-admin is caught **before** it can bite. +3. **Bind** with **one atomic PUT** of the realm, setting `browserFlow`. +4. **Rollback is one write**, and nothing is ever deleted, so the old flow stays usable. + +**The safety property is structural, not intentional.** The reconciler is *incapable* of +writing to the bound flow or anything beneath it: the live `browserFlow` alias and its +whole subtree of sub-flow aliases and execution ids are read first and screened against +every write before a socket opens. It is **fail-closed** — until that binding has been +read, every flow write is refused, and if the bound flow's executions cannot be read the +flow pass writes nothing at all. There is **no `DELETE`** anywhere in its allow-list. + +**Idempotence is load-bearing**, not a nicety: a rebuild-and-reflip every 15 minutes +would churn the login path on a schedule. The bound flow is compared structurally against +the declared shape and, when they match, the run issues **zero writes and builds +nothing**. Sub-flow and config *aliases* are deliberately **not** compared — a flow the +reconciler built carries a hash suffix, so comparing labels would make it permanently +unrecognisable as converged. + +**Recovery.** A crashed run can leave a partial flow under the primary candidate alias. A +later run reads it: if it matches desired it is a completed-but-unbound build and is +simply **bound**; if not, it is left alone and the next candidate alias is used. The +candidate count is **bounded**, so a persistently failing build fails loudly naming the +flows to clean up rather than creating unbounded sprawl. + +```sh +# What the reconciler currently says. A normal run prints the full comparison. +kubectl --context k3s-prod -n keycloak get jobs -l app.kubernetes.io/name=keycloak-passkey-reconciler +kubectl --context k3s-prod -n keycloak logs -l app.kubernetes.io/name=keycloak-passkey-reconciler --tail=200 + +# Force a run now rather than wait for the tick. NOTE: this runs the DEFAULT mode, which +# converges (including flipping browserFlow once a new flow verifies). `kubectl create job +# --from=cronjob/...` CANNOT override the container command, so there is no way to get +# --audit or --no-bind out of it — for those, copy the CronJob's pod spec into a one-off +# Job and append the flag. +kubectl --context k3s-prod -n keycloak create job --from=cronjob/keycloak-passkey-reconciler \ + passkey-run-once +kubectl --context k3s-prod -n keycloak logs job/passkey-run-once + +# ROLLBACK — one write. The run log names the previous alias ("ROLLBACK is one write: +# set browserFlow back to ..."). The old flow is never deleted, so this always works. +# browserFlow is a realm setting: Realm settings -> ... or +# PUT /admin/realms/homelab with browserFlow set to the previous alias +# (read-modify-write the realm representation; a partial body drops the passkey policy). +# The admin console path is Authentication -> Flows -> -> Action -> Bind flow +# -> Browser flow. The console does this as one atomic change too. + +# Regenerate the desired state after editing the passkey policy or the flow in the realm +# import (the merge gate will demand it). +python3 ci/render-keycloak-passkey-policy.py --write +``` + +**If you would rather the go-live be a human act,** the reconciler takes `--no-bind`: it +builds and verifies the flow and stops, printing the one write needed to bind it. That +automates everything except one atomic operator action. + +### Verifying it actually works (nobody has, as of 2026-10-18) No passkey has been enrolled and no browser has been pointed at this. The checks that would prove it, in order: diff --git a/platform/keycloak/passkey-desired-state.yaml b/platform/keycloak/passkey-desired-state.yaml new file mode 100644 index 0000000..6d14b5b --- /dev/null +++ b/platform/keycloak/passkey-desired-state.yaml @@ -0,0 +1,167 @@ +--- +# ============================================================================= +# GENERATED FILE — DO NOT EDIT BY HAND. +# +# source: platform/keycloak/keycloak-realm-import.yaml (spec.realm) +# regenerate: python3 ci/render-keycloak-passkey-policy.py --write +# gated by: conformance/tests/l0-static/keycloak-passkey-reconciler +# (runs this renderer with --check on every merge) +# +# WHAT THIS IS: the DESIRED STATE for the `homelab` realm's PASSKEY configuration — +# the WebAuthn passwordless policy, the enrolment required action, and the declared +# browser authentication flow tree. platform/keycloak/passkey-reconciler.yaml mounts +# it and (a) converges the policy and the required action onto it, (b) AUDITS the +# live browser flow against it and alarms on divergence without rewriting it. +# +# WHY IT IS A SEPARATE OBJECT: the realm import is a ONE-SHOT that never re-runs +# once Done=True, so a realm-level field or a flow edited in git never reaches the +# realm. Measured on prod 2026-10-18, hours after the passkey declaration merged: +# Done=True / Started=False, with the whole passkey block inert. The reconciler is +# the level-triggered loop that closes that; this file is what it reconciles +# TOWARDS. +# +# WHY IT IS GENERATED AND NOT AUTHORED: a hand-written copy of the RP ID, the +# user-verification requirement and the flow tree would be a second source of truth, +# free to disagree with the realm import — the same "two declarations that can +# disagree" shape as the bug being closed. It is rendered from the import and the +# render is gated, so the two cannot drift. +# +# STRINGS ARE VERBATIM: webAuthnPolicyPasswordlessRpId is the ${BASE_DOMAIN} +# substitution token exactly as the realm import writes it. Both files are rendered +# by the SAME Flux Kustomization (./platform-config), so the token resolves to the +# same value in both and the domain is decided in exactly one place. +# +# `requiredAction` is the ONE entry that is DERIVED rather than copied: the realm +# import deliberately declares no `requiredActions` block, because Keycloak 26.0.6 +# treats that as REPLACE and declaring one action would delete CONFIGURE_TOTP and +# nine more (a hard lock-out of every platform-admin). It is derived instead from +# the presence of a webauthn-authenticator-passwordless execution in the flow: +# passkeys are offered, therefore enrolment must be enabled — and defaultAction is +# false, so enrolment stays opt-in. +# +# A field NOT declared on the realm in the import is simply ABSENT here, and the +# reconciler leaves it alone. Absence means UNMANAGED, never "force to a default". +# ============================================================================= +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-passkey-desired-state + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak-passkey-desired-state + app.kubernetes.io/part-of: keycloak +data: + passkey-desired-state.json: | + { + "authenticatorConfig": { + "conditional-otp-platform-admin": { + "condUserRole": "platform-admin", + "negate": "false" + } + }, + "browserFlow": "browser-mfa", + "flows": { + "browser-mfa": { + "executions": [ + { + "kind": "authenticator", + "name": "auth-cookie", + "requirement": "ALTERNATIVE" + }, + { + "kind": "authenticator", + "name": "identity-provider-redirector", + "requirement": "ALTERNATIVE" + }, + { + "kind": "flow", + "name": "browser-mfa-forms", + "requirement": "ALTERNATIVE" + } + ], + "topLevel": true + }, + "browser-mfa-conditional-otp": { + "executions": [ + { + "configAlias": "conditional-otp-platform-admin", + "kind": "authenticator", + "name": "conditional-user-role", + "requirement": "REQUIRED" + }, + { + "kind": "authenticator", + "name": "auth-otp-form", + "requirement": "REQUIRED" + } + ], + "topLevel": false + }, + "browser-mfa-forms": { + "executions": [ + { + "kind": "authenticator", + "name": "auth-username-form", + "requirement": "REQUIRED" + }, + { + "kind": "flow", + "name": "browser-mfa-passkey-or-password", + "requirement": "REQUIRED" + } + ], + "topLevel": false + }, + "browser-mfa-passkey-or-password": { + "executions": [ + { + "kind": "authenticator", + "name": "webauthn-authenticator-passwordless", + "requirement": "ALTERNATIVE" + }, + { + "kind": "flow", + "name": "browser-mfa-password-and-otp", + "requirement": "ALTERNATIVE" + } + ], + "topLevel": false + }, + "browser-mfa-password-and-otp": { + "executions": [ + { + "kind": "authenticator", + "name": "auth-password-form", + "requirement": "REQUIRED" + }, + { + "kind": "flow", + "name": "browser-mfa-conditional-otp", + "requirement": "CONDITIONAL" + } + ], + "topLevel": false + } + }, + "realm": "homelab", + "requiredAction": { + "alias": "webauthn-register-passwordless", + "defaultAction": false, + "enabled": true + }, + "webAuthnPolicyPasswordless": { + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "none", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": true, + "webAuthnPolicyPasswordlessCreateTimeout": 300, + "webAuthnPolicyPasswordlessRequireResidentKey": "Yes", + "webAuthnPolicyPasswordlessRpEntityName": "Homelab SSO", + "webAuthnPolicyPasswordlessRpId": "${BASE_DOMAIN}", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256", + "RS256" + ], + "webAuthnPolicyPasswordlessUserVerificationRequirement": "required" + } + } diff --git a/platform/keycloak/passkey-reconciler.yaml b/platform/keycloak/passkey-reconciler.yaml new file mode 100644 index 0000000..53b9d3e --- /dev/null +++ b/platform/keycloak/passkey-reconciler.yaml @@ -0,0 +1,2968 @@ +--- +# ============================================================================= +# Keycloak PASSKEY RECONCILER (R-IDP-03, R-IDP-04) +# +# THE FAILURE THIS EXISTS FOR, MEASURED ON PROD 2026-10-18 +# ------------------------------------------------------- +# platform/keycloak/keycloak-realm-import.yaml declares the WebAuthn PASSWORDLESS +# policy and a restructured `browser-mfa` browser flow (merged as PR #72). That +# declaration CANNOT TAKE EFFECT: +# +# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath='{.status}' +# Done=True Started=False HasErrors=False +# +# A KeycloakRealmImport is a ONE-SHOT. Once it rests at Done=True the operator never +# re-runs it, so every realm-level field and every flow the manifest declares after +# the FIRST import is inert — merged into git, applied by Flux, reported Ready, and +# reaching the realm never. Merging the passkey block changed nothing about how +# anyone logs in, and nothing anywhere said so. +# +# This is the SAME failure class platform/keycloak/client-secret-reconciler.yaml +# already closes for client secrets (2026-09-06: all 12 prod clients holding the +# literal `(env:...)` placeholder behind an entirely green cluster) and for +# environment-derived client URLs. This CronJob closes it for the passkey +# configuration, the same way: a one-shot that fails open is replaced by a +# LEVEL-TRIGGERED loop that converges every 15 minutes — the posture Flux gives +# every other resource in this repo (R-GITOPS-03 drift correction), applied to the +# one piece of state Flux itself cannot reach, Keycloak's own database. +# +# ALL THREE DECLARED THINGS CONVERGE. Nothing here is left as a manual step: +# pass 1 webAuthnPolicyPasswordless* realm policy strict read-modify-write +# pass 2 the webauthn-register-passwordless action strict read-modify-write, +# ONE literally-named alias +# pass 3 the browser authentication flow BLUE/GREEN +# +# WHY A SIBLING AND NOT A SECOND PASS OF THE EXISTING RECONCILER +# ------------------------------------------------------------- +# client-secret-reconciler.yaml's stated guarantee is that prod's real user and the +# media-users / platform-admins / viewers groups survive every run, and its +# --selftest ASSERTS that it refuses /users, /groups, /roles, /authentication and +# the whole-realm endpoint. This work needs the realm endpoint and the +# /authentication tree. Widening that allow-list would break its selftest and, more +# importantly, would retire a guarantee that is currently literally true. So this is +# a SEPARATE program with a SEPARATE, differently-shaped allow-list, and the sibling +# is untouched — its selftest still passes unchanged. +# +# PASS 3: BLUE/GREEN, AND WHY THE OBVIOUS DESIGN IS WRONG +# ------------------------------------------------------ +# The declared restructure is five flows and ten executions, and the realm ALREADY +# holds a `browser-mfa` flow of the OLD shape (one combined +# auth-username-password-form rather than the username/password split a passkey +# branch requires). The admin API offers no transaction: create-flow, add-subflow, +# add-execution, set-requirement and bind-config are all separate calls. +# +# THE FIRST DESIGN OF THIS COMPONENT WAS "AUDIT THE FLOW, NEVER WRITE IT", on the +# reasoning that a run dying part-way through that sequence would leave a HALF-BUILT +# FLOW BOUND AS THE LOGIN PATH for Grafana, Gitea, Nextcloud, Headlamp, Semaphore, +# PiHole, Crafty, the *arr stack AND kubectl — and this repo has had a total SSO +# outage from a realm change (2026-09-06), with the conditional-OTP sub-flow (which +# the declared structure moves ONE LEVEL DEEPER than it sits today) once throwing on +# every evaluation and 403ing every platform-admin. +# +# THAT RISK IS REAL BUT IT IS NOT INHERENT. It follows only from MUTATING THE BOUND +# FLOW. So this does not mutate the bound flow: +# +# 1. BUILD the target structure under a NEW, UNUSED alias — the declared alias +# plus a hash of the desired shape, applied to every flow in the tree. Every +# multi-call mutation happens on a flow NOTHING IS USING. A failure at any +# point leaves an orphaned, unbound flow and ZERO user-visible effect: login +# keeps using the old flow, untouched. +# 2. READ IT BACK AND VERIFY it matches the desired shape — nesting levels, +# ordering, authenticators, requirements, and the conditional-OTP config +# CONTENTS. Nothing is bound until that passes, so the regression the operator +# fears is caught BEFORE it can bite rather than after. +# 3. BIND with ONE ATOMIC WRITE: PUT /admin/realms/homelab setting browserFlow. +# One atomic mutation to live behaviour, not five. +# 4. ROLLBACK IS ONE WRITE. The previous alias is printed in the run log, and +# NOTHING IS EVER DELETED, so the old flow remains a working rollback target. +# +# THE SAFETY PROPERTY, ENFORCED STRUCTURALLY RATHER THAN BY INTENTION +# ------------------------------------------------------------------ +# Blue/green needs flow-create and execution-add endpoints, so the allow-list alone +# is no longer sufficient: its write entries are alias-parameterised and cannot say +# "but not the flow that is bound". A SECOND structural guard sits at the same choke +# point in _http(): +# +# THIS PROGRAM IS INCAPABLE OF WRITING TO THE FLOW THAT IS CURRENTLY BOUND, OR TO +# ANY SUB-FLOW OR EXECUTION BENEATH IT. +# +# Before any flow write the live browserFlow alias is fetched and its ENTIRE SUBTREE +# of sub-flow aliases and execution ids is recorded; every write is screened against +# those sets before a socket is opened. Protecting the whole subtree matters: adding +# an execution to `browser-mfa-forms` changes live login behaviour just as surely as +# touching `browser-mfa` itself. The create-flow endpoint carries no alias in its +# path, so it is screened on its request BODY — otherwise it would escape the guard. +# +# AND IT FAILS CLOSED: the protected sets start each run as "not yet determined", and +# while they are, EVERY flow write is refused. If the bound flow's executions cannot +# be read, they stay undetermined and the flow pass writes nothing at all. So the +# worst a buggy or half-failed run can do is leave an unused flow lying around. +# +# IDEMPOTENCE IS LOAD-BEARING HERE, NOT A NICETY. A rebuild-and-reflip every fifteen +# minutes would be a permanent write loop against the IdP and would churn the realm's +# login path on a schedule. So pass 3 does a REAL STRUCTURAL COMPARISON of the bound +# flow against the desired shape and, when they match, issues ZERO writes and creates +# nothing. Sub-flow aliases and authenticator-config aliases are compared as +# STRUCTURE, not as names — a flow this program built carries a hash suffix on every +# alias, so comparing labels would mean it could never be recognised as converged. +# +# RECOVERY. A crashed earlier run can leave a partial flow under the primary +# candidate alias. A later run reads that flow: if it MATCHES desired it is a +# completed-but-unbound build and is simply BOUND (no rebuild); if it does not, it is +# left alone and the next candidate alias is used. The number of candidates is +# BOUNDED, so a persistently failing build cannot create unbounded flow sprawl — it +# fails loudly instead, naming the flows an operator should clean up. +# +# THE REALM-LEVEL PUT, CHECKED AGAINST 26.0.6 RATHER THAN ASSUMED +# -------------------------------------------------------------- +# Verified against quay.io/keycloak/keycloak:26.0.6, the image the cluster runs: +# * RealmAdminResource.updateRealm is the only handler for PUT /admin/realms/. +# Its javadoc: "Any user, roles or client information in the representation will +# be ignored. This will only update top-level attributes of the realm." +# * It delegates to DefaultExportImportManager.updateRealm, lines 725-894 at tag +# 26.0.6. That body was read in full and brace-matched: NO rep.getUsers(), NO +# getClients(), NO getGroups(), NO getRoles(), NO getRequiredActions(), NO +# getAuthenticationFlows(), NO getAuthenticatorConfig(). Users, groups, clients, +# roles, required actions, flows and authenticator configs are UNREACHABLE from +# this endpoint — not merely ignored by convention. +# * It DOES read rep.getBrowserFlow(). That is what makes the atomic bind in step 3 +# possible. It also means the POLICY pass could rebind the login path by +# accident, so the policy write carries browserFlow back from the GET unchanged +# and refuses to proceed if it would differ. One function may change it, and only +# after verification passes. +# * The passwordless policy is applied ALL-OR-NOTHING and only when RpEntityName is +# non-empty, with a Keycloak DEFAULT substituted for every omitted key. That is +# why the write is a strict read-modify-write of the full live representation and +# never a partial body — a partial PUT would silently reset the keys it left out, +# and one missing RpEntityName would drop the whole policy while returning 204. +# * A third trap, in the flow builder: updateExecutions compares rep.getPriority() +# against the stored priority, and a JSON body omitting `priority` deserialises it +# as the int 0 — which would RESHUFFLE the execution order while only a +# requirement change was intended. Every requirement write carries the +# execution's current priority back. +# +# NOTHING IS EVER DELETED. There is no DELETE method anywhere in the allow-list. +# +# MODES: default reconciles; `--audit` is read-only and builds and binds nothing; +# `--no-bind` builds and VERIFIES but leaves the flip as a one-write human act. +# +# CARRIED BY platform-config (dependsOn platform, wait:true), alongside the realm +# import whose failure mode it covers and the keycloak-admin-credentials Secret it +# authenticates with. +# +# PROOF: the script's own `--selftest` drives these real code paths against an +# in-process stub Keycloak that MODELS the 26.0.6 behaviours above — including the +# RpEntityName gate, the all-or-nothing policy rebuild, the required-action replace, +# the updateExecutions priority trap, and flow construction where everything is +# created DISABLED. It asserts blue/green build-verify-flip, that a build killed at +# four different points NEVER binds and leaves login untouched, that the bound flow +# is unwritable and fail-closed, idempotence with zero writes, recovery and the +# bounded alias budget, that nothing is ever deleted, and the conditional-OTP +# regression. The L0 static gate EXTRACTS the script from this ConfigMap and runs it, +# and mutations.py proves the selftest can FAIL, so what CI proves is byte-for-byte +# what ships: conformance/tests/l0-static/keycloak-passkey-reconciler/. +# ============================================================================= +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-passkey-reconciler + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak-passkey-reconciler + app.kubernetes.io/part-of: keycloak +data: + # SINGLE SOURCE OF TRUTH for the reconciler. Do not fork it into a sidecar copy: + # the L0 gate extracts THIS block and executes its --selftest, so a second copy + # would be untested. Contains no dollar sign at all, on purpose -- this file is + # rendered by a Flux Kustomization with postBuild.substituteFrom, so an unescaped + # dollar-brace expansion here would be rewritten before any container ran (the + # trap documented at length in apps/wave-1/nextcloud.yaml). + reconcile-keycloak-passkeys.py: | + #!/usr/bin/env python3 + # ============================================================================= + # reconcile-keycloak-passkeys.py -- make the `homelab` realm's PASSKEY configuration + # equal what git declares, idempotently, including the browser login flow, WITHOUT + # ever mutating the flow that users are currently logging in through. + # + # WHY THIS EXISTS + # --------------- + # platform/keycloak/keycloak-realm-import.yaml declares the WebAuthn PASSWORDLESS + # policy and a restructured `browser-mfa` browser flow. That declaration CANNOT TAKE + # EFFECT. Measured on prod 2026-10-18: + # + # kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath='{.status}' + # Done=True Started=False HasErrors=False + # + # A KeycloakRealmImport is a ONE-SHOT. Once it rests at Done=True it never re-runs, so + # every realm-level field and every flow the manifest declares AFTER the first import + # is inert -- merged into git, applied by Flux, reported Ready, and reaching the realm + # never. This is the same failure class the client-secret reconciler already closes for + # client secrets and client URLs (platform/keycloak/client-secret-reconciler.yaml, + # 2026-09-06 incident). This program closes it for the passkey configuration. + # + # THREE PASSES, ALL OF WHICH CONVERGE + # ----------------------------------- + # pass 1 WEBAUTHN PASSWORDLESS POLICY strict read-modify-write of the realm. + # pass 2 THE ENROLMENT REQUIRED ACTION strict read-modify-write, ONE hard-coded + # alias. + # pass 3 THE BROWSER AUTHENTICATION FLOW BLUE/GREEN: build the target structure + # under a NEW alias, verify it, then flip the + # realm's browserFlow in ONE atomic write. + # + # PASS 3: WHY BLUE/GREEN, AND WHY THE OBVIOUS DESIGN IS WRONG + # ----------------------------------------------------------- + # The declared restructure is five flows and ten executions, and the realm ALREADY + # HOLDS a `browser-mfa` flow of the OLD shape (one combined auth-username-password-form + # rather than the username/password split a passkey branch requires). The admin API + # offers no transaction: create-flow, add-subflow, add-execution, set-requirement and + # bind-config are all separate calls. + # + # THE FIRST DESIGN OF THIS FILE WAS "AUDIT ONLY, NEVER WRITE THE FLOW", on the + # reasoning that a run dying part-way through that sequence would leave a HALF-BUILT + # FLOW BOUND AS THE LOGIN PATH for the whole platform -- and this repo has had a total + # SSO outage from a realm change (2026-09-06), with the conditional-OTP sub-flow (which + # the declared structure moves ONE LEVEL DEEPER than it sits today) once throwing on + # every evaluation and 403ing every platform-admin. + # + # THAT RISK IS REAL BUT IT IS NOT INHERENT. It follows only from MUTATING THE BOUND + # FLOW. So this does not mutate the bound flow. Instead: + # + # 1. BUILD the target structure under a NEW, UNUSED alias (the declared alias plus a + # hash of the desired shape). Every multi-call mutation happens on a flow NOTHING + # IS USING. A failure at any point leaves an orphaned, unbound flow and ZERO + # user-visible effect: login keeps using the old flow, untouched. + # 2. READ THE BUILT FLOW BACK and verify it matches the desired shape -- nesting + # levels, ordering, authenticators, requirements, and the conditional-OTP config + # contents. Nothing is bound until that passes. The regression the operator fears + # becomes detectable BEFORE it can bite, not after. + # 3. BIND with ONE ATOMIC WRITE: PUT /admin/realms/ setting browserFlow. One + # atomic mutation to live behaviour, not five. + # 4. ROLLBACK IS ONE WRITE. The previous alias is printed in the run log, and the + # old flow is NEVER DELETED, so it remains a working rollback target. + # + # THE SAFETY PROPERTY, ENFORCED STRUCTURALLY RATHER THAN BY INTENTION + # ------------------------------------------------------------------ + # Blue/green needs flow-create and execution-add endpoints, which widens the + # allow-list. So the allow-list alone is no longer sufficient, and a SECOND structural + # guard sits at the same choke point: + # + # THIS PROGRAM IS INCAPABLE OF WRITING TO THE FLOW THAT IS CURRENTLY BOUND, OR TO + # ANY SUB-FLOW OR EXECUTION BENEATH IT. + # + # Before any flow write, the live browserFlow alias is fetched and its ENTIRE SUBTREE + # of sub-flow aliases and execution ids is recorded in _PROTECTED_FLOW_ALIASES / + # _PROTECTED_EXECUTION_IDS. _http() then refuses any write whose target is in those + # sets, before a socket is opened. Protecting the whole subtree matters: adding an + # execution to `browser-mfa-forms` would change live login behaviour just as surely as + # touching `browser-mfa` itself. + # + # AND IT FAILS CLOSED: the protected sets start as None, meaning "not yet determined", + # and while they are None EVERY flow write is refused. So a code path that tried to + # write before discovering what is bound cannot do so. The worst a buggy or + # half-failed run can do is leave an unused flow lying around. + # + # IDEMPOTENCE IS LOAD-BEARING HERE, NOT A NICETY. A rebuild-and-reflip every fifteen + # minutes would be a permanent write loop against the IdP and would churn the realm's + # login path forever. So pass 3 does a REAL STRUCTURAL COMPARISON of the bound flow + # against the desired shape and, when they match, issues ZERO WRITES and creates + # nothing. + # + # SUB-FLOW ALIASES ARE COMPARED AS STRUCTURE, NOT AS NAMES. A sub-flow's alias is a + # label with no effect on login behaviour; what determines behaviour is the tree shape, + # the authenticators, the requirements and the bound configs. Comparing aliases would + # mean a flow this program built (whose aliases carry a hash suffix) could never be + # recognised as converged, so it would be rebuilt and reflipped forever. The same + # applies to authenticator-config aliases: the CONTENTS are compared, the label is not. + # + # THE CONDITIONAL-OTP REGRESSION, SPECIFICALLY + # -------------------------------------------- + # Verification asserts, by nesting level and position: that the conditional-OTP + # sub-flow is referenced CONDITIONAL from the password branch; that its children are + # `conditional-user-role` (REQUIRED) and `auth-otp-form` (REQUIRED); and -- the part + # that actually broke -- that the bound authenticator config holds + # `condUserRole: platform-admin`. The original bug wrote the role under + # `condition.evaluated.user.role`, which is not a real property, so the role resolved + # to null, the REQUIRED conditional step threw "Invalid role name submitted: null", and + # every platform-admin got a 403. A key-level comparison of the config dict is what + # makes that detectable, so the comparison is key-level -- and because it runs BEFORE + # the bind, that realm never becomes the live login path. + # + # WHY A REALM-LEVEL PUT IS ACCEPTABLE HERE WHEN THE SIBLING RECONCILER REFUSES IT + # ------------------------------------------------------------------------------ + # platform/keycloak/client-secret-reconciler.yaml deliberately refuses + # /admin/realms/ and its --selftest asserts that refusal. That reconciler's + # guarantee -- prod's real user and the media-users / platform-admins / viewers groups + # survive every run -- is stated in terms of endpoints it cannot reach, so its + # allow-list is NOT widened by this work and its selftest still passes unchanged. This + # is a SEPARATE program with a SEPARATE, differently-shaped allow-list. + # + # The realm PUT was checked against the Keycloak the cluster actually runs + # (quay.io/keycloak/keycloak:26.0.6) rather than assumed: + # + # * RealmAdminResource.updateRealm is the only handler for PUT /admin/realms/. + # Its own javadoc: "Any user, roles or client information in the representation will + # be ignored. This will only update top-level attributes of the realm." + # * It delegates to DefaultExportImportManager.updateRealm, lines 725-894 at tag + # 26.0.6. That body was read in full and brace-matched: it contains NO + # rep.getUsers(), NO getClients(), NO getGroups(), NO getRoles(), NO + # getRequiredActions(), NO getAuthenticationFlows() and NO getAuthenticatorConfig() + # call. Users, groups, clients, roles, required actions, flows and authenticator + # configs are UNREACHABLE from this endpoint -- not merely "ignored by convention". + # * It DOES read rep.getBrowserFlow(). That is what makes the atomic bind in step 3 + # possible. It also means the POLICY pass could rebind the login path by accident, + # so the policy write carries browserFlow back from the GET unchanged and refuses + # to proceed if it would differ. bind_browser_flow() is the ONLY function permitted + # to change it, and it is called only after verification passes. + # + # TWO 26.0.6 BEHAVIOURS THAT MAKE STRICT READ-MODIFY-WRITE MANDATORY, NOT TIDY + # --------------------------------------------------------------------------- + # 1. The passwordless policy is applied ALL-OR-NOTHING and only when RpEntityName is + # non-empty. DefaultExportImportManager.updateRealm: + # if (rep.getWebAuthnPolicyPasswordlessRpEntityName() != null + # && !rep.getWebAuthnPolicyPasswordlessRpEntityName().isEmpty()) + # and getWebAuthnPolicyPasswordless(rep) then rebuilds the WHOLE policy, replacing + # every null field with a Keycloak DEFAULT. So a partial PUT carrying only the + # drifted keys would SILENTLY RESET every key it omitted, and a PUT that drops + # RpEntityName drops the entire policy without an error. Both are modelled by the + # stub in --selftest. + # 2. updateRequiredAction (AuthenticationManagementResource) likewise REPLACES the + # provider from the representation -- name, alias, priority, enabled, defaultAction + # and config all come from the rep. A two-field PUT would blank the name and reset + # the priority to 0. + # + # A THIRD, IN THE FLOW BUILDER: updateExecutions compares rep.getPriority() against the + # stored priority, and a JSON body omitting `priority` deserialises it as the int 0 -- + # which would RESHUFFLE the execution order while only a requirement change was + # intended. So every requirement write carries the execution's current priority back. + # + # WHY THE ENROLMENT ACTION IS ONE HARD-CODED ALIAS IN THE ALLOW-LIST + # ----------------------------------------------------------------- + # The realm import's own header explains at length why `requiredActions` must NOT be + # declared in the import: 26.0.6 treats it as REPLACE, not merge, so declaring one + # action deletes the other ten -- including CONFIGURE_TOTP, without which a + # platform-admin can never enrol the TOTP that auth-otp-form still demands. That is a + # lock-out delivered by a change meant to ADD a login method. + # + # The ADMIN API does not share that hazard: PUT required-actions/ touches exactly + # one provider. But this program is not trusted with the general case. The allow-list + # names the single alias `webauthn-register-passwordless` LITERALLY, so disabling + # CONFIGURE_TOTP is refused before a socket exists. The sibling defaults are READ and + # their absence is reported as the import-REPLACE damage -- audited, never repaired. + # + # NOTHING IS EVER DELETED. There is no DELETE method anywhere in the allow-list. A + # superseded flow is left unbound and harmless, because it is the rollback target. + # + # SAFETY PROPERTIES, each enforced in code and proved by --selftest: + # 1. compare-then-write: a converged realm issues ZERO writes and builds nothing. + # 2. idempotent: a second run over a converged realm issues ZERO writes. + # 3. THE BOUND FLOW IS NEVER WRITTEN -- not it, not its sub-flows, not its + # executions. Refused in _http() before a socket, and fail-closed until the + # binding is known. + # 4. the required-action write can reach exactly ONE alias, named literally. + # 5. the realm policy PUT never changes browserFlow; only bind_browser_flow() may, + # and only after verification passes. + # 6. a flow is BOUND only after a readback proves it matches the desired shape. + # 7. no secret is ever printed, in any mode, on any path. + # 8. fail closed: an unreadable or malformed desired state writes NOTHING. + # 9. nothing is ever deleted. + # 10. the three passes are INDEPENDENT -- a failure in one does not suppress the + # others. + # + # MODES + # (default) reconcile: compare, and write only what differs. + # --audit READ-ONLY throughout. Report drift and exit 1 if anything differs. + # Builds nothing and binds nothing. + # --no-bind build and VERIFY the new flow but do NOT flip browserFlow. Leaves a + # verified, unbound flow and prints the one-line bind command. The + # deliberate fallback if an operator wants the flip to be a human act. + # --selftest the offline behavioural proof, against an in-process stub Keycloak. + # No cluster; loopback only. This is what the L0 static gate runs, so + # the code the gate proves is the code that ships. + # + # NOT ONE DOLLAR SIGN IN THIS FILE, DELIBERATELY -- not even a regex end-anchor (hence + # re.fullmatch throughout), and not even inside these comments. It ships inside a + # ConfigMap that the `platform-config` Flux Kustomization renders with + # postBuild.substituteFrom, so Flux envsubst rewrites this text before any container + # runs: an unescaped dollar-brace expansion would be silently emptied at reconcile time. + # That is the trap that made apps/wave-1/nextcloud.yaml's OIDC guard dead code for + # months. Python needs no dollar signs, so the hazard is avoided STRUCTURALLY rather + # than escaped, and the character is banned outright by the L0 check so no future edit + # has to reason about escaping. + # ============================================================================= + from __future__ import annotations + + import hashlib + import json + import os + import re + import sys + import time + import urllib.error + import urllib.parse + import urllib.request + + # The DESIRED STATE: generated from platform/keycloak/keycloak-realm-import.yaml by + # ci/render-keycloak-passkey-policy.py, shipped as a ConfigMap, mounted read-only. + # Mounted rather than read from the Kubernetes API for the same reason as the sibling + # reconciler: this pod carries NO ServiceAccount token, so it has no Kubernetes API + # access to read the KeycloakRealmImport CR with. + STATE_FILE = os.environ.get("PASSKEY_STATE_FILE", + "/etc/keycloak-passkey/passkey-desired-state.json") + TIMEOUT = 20 + READY_ATTEMPTS = int(os.environ.get("READY_ATTEMPTS", "24")) + READY_SLEEP = float(os.environ.get("READY_SLEEP", "5")) + + # ---- pass 1: the ONLY realm keys this program may write ------------------------------ + # Every one of them is a webAuthnPolicyPasswordless* field. The two-factor + # webAuthnPolicy* family is deliberately NOT here: nothing in this realm uses the + # `webauthn-authenticator` two-factor authenticator, and a policy written into the wrong + # family is a silent no-op (WebAuthnPasswordlessAuthenticator.getWebAuthnPolicy reads + # getWebAuthnPolicyPasswordless only). A desired state naming anything outside this + # tuple is a hard error, never forwarded to Keycloak as a realm field. + POLICY_STR_FIELDS = ( + "webAuthnPolicyPasswordlessRpId", + "webAuthnPolicyPasswordlessRpEntityName", + "webAuthnPolicyPasswordlessRequireResidentKey", + "webAuthnPolicyPasswordlessUserVerificationRequirement", + "webAuthnPolicyPasswordlessAuthenticatorAttachment", + "webAuthnPolicyPasswordlessAttestationConveyancePreference", + ) + # ORDER IS MEANINGFUL for signature algorithms -- it is a preference list handed to the + # authenticator, so ES256-then-RS256 is not the same declaration as RS256-then-ES256. + # Compared exactly. + POLICY_ORDERED_LIST_FIELDS = ("webAuthnPolicyPasswordlessSignatureAlgorithms",) + # ORDER IS NOT MEANINGFUL for an AAGUID allow-list (it is a set membership test), and an + # order-sensitive comparison of a list Keycloak may reorder would rewrite the realm on + # every run forever while reporting success -- the sibling reconciler's measured + # redirectUris trap. Compared sorted; sorted() not set(), so a duplicate is still drift. + POLICY_UNORDERED_LIST_FIELDS = ("webAuthnPolicyPasswordlessAcceptableAaguids",) + POLICY_INT_FIELDS = ("webAuthnPolicyPasswordlessCreateTimeout",) + POLICY_BOOL_FIELDS = ("webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister",) + POLICY_FIELDS = (POLICY_STR_FIELDS + POLICY_ORDERED_LIST_FIELDS + + POLICY_UNORDERED_LIST_FIELDS + POLICY_INT_FIELDS + + POLICY_BOOL_FIELDS) + # The field 26.0.6 gates the WHOLE passwordless policy on. A PUT that omits it or sends + # it empty drops the policy silently, so the write path refuses to proceed without it. + RP_ENTITY_NAME = "webAuthnPolicyPasswordlessRpEntityName" + # The realm key that decides which flow users log in through. The policy pass carries it + # back unchanged and refuses to alter it; bind_browser_flow() is the only writer. + BROWSER_FLOW = "browserFlow" + + # ---- pass 2: the one required action, and the siblings that are only AUDITED --------- + ENROL_ACTION = "webauthn-register-passwordless" + # The only two fields of it this program compares or writes. `priority`, `name` and + # `config` are carried back verbatim from the GET (updateRequiredAction replaces them + # from the representation, so omitting them would blank them). + ACTION_FIELDS = ("enabled", "defaultAction") + # Default required actions whose DISAPPEARANCE is the signature of someone declaring a + # `requiredActions` block in the realm import -- 26.0.6 imports exactly the declared + # ones plus delete_account and skips DefaultRequiredActions.addActions entirely. Losing + # CONFIGURE_TOTP is a hard lock-out of every platform-admin, because auth-otp-form in + # the flow below still demands a TOTP they could then never enrol. Audited only: no + # write to these aliases is reachable. + GUARDED_ACTIONS = ("CONFIGURE_TOTP", "UPDATE_PASSWORD", "delete_credential") + + # ---- pass 3: flow construction ------------------------------------------------------ + # Keycloak's provider id for a plain (non-form) sub-flow. The declared tree uses only + # basic-flow; a form-flow would additionally need a `provider`, which this program does + # not support and the renderer does not emit. + BASIC_FLOW = "basic-flow" + # How many distinct candidate aliases to try before giving up. A crashed earlier run can + # leave a half-built flow under the primary candidate alias; rather than being stuck on + # it forever (it can never be deleted -- nothing here deletes), move to the next + # candidate. Bounded so a persistent build failure cannot create unbounded flow sprawl. + MAX_BUILD_ATTEMPTS = 5 + + # ============================================================================= + # ENDPOINT ALLOW-LIST (safety properties 3, 4, 9). METHOD-AWARE, unlike the sibling + # reconciler's path-only list. Every entry is fullmatch-ed against the (method, path) of + # every request _http() is asked to make; anything else raises before a socket exists. + # + # READ THE ABSENCES. Not present, therefore unreachable: + # DELETE anything at all -- no flow, execution, config, required action or realm can + # be deleted by this program. A superseded flow is the rollback target. + # PUT /admin/realms//authentication/executions/ (single-execution + # update, including raise-priority/lower-priority siblings) + # PUT /admin/realms//authentication/config/ (edit a config) + # PUT /admin/realms//authentication/flows/ (rename a flow) + # POST /admin/realms//authentication/register-required-action + # PUT .../required-actions/ e.g. CONFIGURE_TOTP + # anything under /users, /groups, /roles, /clients + # + # AND NOTE WHAT THE ALLOW-LIST ALONE CANNOT EXPRESS: the write entries below are + # alias-parameterised, so the allow-list cannot say "but not the bound flow". That is + # what the _PROTECTED_* guard immediately after it is for. Both must pass. + # ============================================================================= + _ALLOWED = ( + ("POST", r"/realms/master/protocol/openid-connect/token"), + # pass 1 + the atomic bind in pass 3: read-modify-write of the realm's top-level + # attributes. + ("GET", r"/admin/realms/[^/]+"), + ("PUT", r"/admin/realms/[^/]+"), + # pass 2: the enrolment action. The alias is LITERAL in the PUT and the single GET, + # so no other required action is writable. The list endpoint is read-only. + ("GET", r"/admin/realms/[^/]+/authentication/required-actions"), + ("GET", r"/admin/realms/[^/]+/authentication/required-actions/" + r"webauthn-register-passwordless"), + ("PUT", r"/admin/realms/[^/]+/authentication/required-actions/" + r"webauthn-register-passwordless"), + # pass 3 reads. + ("GET", r"/admin/realms/[^/]+/authentication/flows"), + ("GET", r"/admin/realms/[^/]+/authentication/flows/[^/]+/executions"), + ("GET", r"/admin/realms/[^/]+/authentication/config/[^/]+"), + # pass 3 BUILD writes. Every one of these targets a flow alias or execution id, and + # every one is additionally screened by the _PROTECTED_* guard so it can never + # reach the bound flow's subtree. + ("POST", r"/admin/realms/[^/]+/authentication/flows"), + ("POST", r"/admin/realms/[^/]+/authentication/flows/[^/]+/executions/flow"), + ("POST", r"/admin/realms/[^/]+/authentication/flows/[^/]+/executions/execution"), + ("PUT", r"/admin/realms/[^/]+/authentication/flows/[^/]+/executions"), + ("POST", r"/admin/realms/[^/]+/authentication/executions/[^/]+/config"), + ) + + _WRITE_METHODS = ("POST", "PUT", "PATCH", "DELETE") + + # ============================================================================= + # THE PROTECTED SUBTREE (safety property 3) -- the guard the allow-list cannot express. + # + # None means NOT YET DETERMINED, and while either is None every flow write is REFUSED. + # That is the fail-closed half: a code path that tried to build before discovering what + # is bound cannot do so. reconcile_flow() populates them from the live browserFlow's + # whole subtree before it builds anything. + # ============================================================================= + _PROTECTED_FLOW_ALIASES = None + _PROTECTED_EXECUTION_IDS = None + + # Paths whose write targets must be screened, and how to pull the target out of them. + _FLOW_WRITE_PATH = re.compile( + r"/admin/realms/[^/]+/authentication/flows/([^/]+)(?:/executions(?:/flow|/execution)?)?") + _EXEC_CONFIG_PATH = re.compile( + r"/admin/realms/[^/]+/authentication/executions/([^/]+)/config") + # POST /flows carries NO alias in the path -- the alias is in the BODY. It therefore has + # to be screened separately, or it escapes the guard entirely. (Keycloak would 409 on a + # duplicate alias anyway, but "the API happens to reject it" is not a safety property; + # this is.) + _FLOW_CREATE_PATH = re.compile(r"/admin/realms/[^/]+/authentication/flows") + + + def _set_protected(aliases, execution_ids): + """Record the bound flow's whole subtree as unwritable. Called once per run.""" + global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS + _PROTECTED_FLOW_ALIASES = frozenset(aliases) + _PROTECTED_EXECUTION_IDS = frozenset(execution_ids) + + + def _reset_protected(): + """Back to NOT DETERMINED, so every flow write is refused again.""" + global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS + _PROTECTED_FLOW_ALIASES = None + _PROTECTED_EXECUTION_IDS = None + + + class Refused(RuntimeError): + """A write this program must never make was requested. Never expected normally.""" + + + def _screen_write(method, path, body=None): + """Raise Refused if this write targets the bound flow's subtree, or if what is bound + has not been established yet. + + This is the structural expression of "the reconciler is incapable of writing to the + flow that is currently bound". The allow-list cannot say it, because its write + entries are alias-parameterised; this can, because it compares the alias in the path + against the live binding's subtree. + """ + if method not in _WRITE_METHODS: + return + if _FLOW_CREATE_PATH.fullmatch(path): + # Creating a top-level flow. The alias is in the body, so screen that. + target = (body or {}).get("alias") + protected, what = _PROTECTED_FLOW_ALIASES, "flow" + if protected is None: + raise Refused( + "refusing %s %s: the currently BOUND browser flow has not been " + "determined yet, so this program cannot prove the write would miss it. " + "Flow writes are fail-closed until the binding is known" + % (method, path)) + if target in protected: + raise Refused( + "refusing %s %s: flow %r is part of the CURRENTLY BOUND browser flow" + % (method, path, target)) + return + m = _FLOW_WRITE_PATH.fullmatch(path) + if m is None: + m = _EXEC_CONFIG_PATH.fullmatch(path) + if m is None: + return # not a flow write (realm PUT, action PUT, token) + target, protected, what = (m.group(1), _PROTECTED_EXECUTION_IDS, "execution") + else: + target = urllib.parse.unquote(m.group(1)) + protected, what = _PROTECTED_FLOW_ALIASES, "flow" + if protected is None: + raise Refused( + "refusing %s %s: the currently BOUND browser flow has not been determined " + "yet, so this program cannot prove the write would miss it. Flow writes are " + "fail-closed until the binding is known" % (method, path)) + if target in protected: + raise Refused( + "refusing %s %s: %s %r is part of the CURRENTLY BOUND browser flow. This " + "program builds a NEW flow and flips browserFlow atomically; it never " + "mutates the flow users are logging in through, because a half-finished " + "mutation there is a total SSO outage" % (method, path, what, target)) + + + def _http(method, base, path, query=None, headers=None, form=None, body=None): + """Return (status, text). Refuses anything outside the allow-list, and any write + that would touch the bound flow's subtree.""" + if not any(m == method and re.fullmatch(p, path) for m, p in _ALLOWED): + raise Refused("endpoint not on the allow-list: %s %s" % (method, path)) + _screen_write(method, path, body if isinstance(body, dict) else None) + url = base.rstrip("/") + path + if query: + url = url + "?" + urllib.parse.urlencode(query) + hdrs = dict(headers or {}) + data = None + if form is not None: + data = urllib.parse.urlencode(form).encode() + hdrs["Content-Type"] = "application/x-www-form-urlencoded" + elif body is not None: + data = json.dumps(body).encode() + hdrs["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=hdrs, method=method) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return resp.status, resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace") + + + # ============================================================================= + # DESIRED STATE + # ============================================================================= + def declared(path): + """The desired state, validated STRICTLY. Raises on anything unexpected. + + This file decides what gets written into the IdP, so a malformed one must stop the + run dead rather than be half-understood. In particular an unknown policy key must + NOT be forwarded to Keycloak as a realm field -- that would be an arbitrary + realm-attribute write through a file edit, defeating the point of a narrow write + surface. + """ + with open(path, "r", encoding="utf-8") as fh: + raw = json.load(fh) + if not isinstance(raw, dict) or not raw: + raise ValueError("%s does not hold a non-empty JSON object" % path) + + policy = raw.get("webAuthnPolicyPasswordless") + if not isinstance(policy, dict) or not policy: + raise ValueError("%s declares no webAuthnPolicyPasswordless object" % path) + for key, value in policy.items(): + if key not in POLICY_FIELDS: + raise ValueError( + "%s: policy key %r is not one this reconciler is allowed to write. " + "Writable keys are: %s" % (path, key, ", ".join(POLICY_FIELDS))) + if key in POLICY_ORDERED_LIST_FIELDS or key in POLICY_UNORDERED_LIST_FIELDS: + if not isinstance(value, list) or not all(isinstance(x, str) + for x in value): + raise ValueError("%s: policy key %s must be a list of strings" + % (path, key)) + elif key in POLICY_INT_FIELDS: + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError("%s: policy key %s must be an integer" % (path, key)) + elif key in POLICY_BOOL_FIELDS: + if not isinstance(value, bool): + raise ValueError("%s: policy key %s must be a boolean" % (path, key)) + else: + if not isinstance(value, str): + raise ValueError("%s: policy key %s must be a string" % (path, key)) + # The 26.0.6 all-or-nothing guard: without a non-empty RpEntityName in the PUT body + # the realm silently keeps its old policy. A desired state that cannot supply one is + # unusable, so refuse it here rather than discover it as a no-op write later. + if not policy.get(RP_ENTITY_NAME): + raise ValueError( + "%s: %s is absent or empty, so this DESIRED STATE IS UNUSABLE. Keycloak " + "26.0.6 applies the passwordless policy ONLY when this field is non-empty, " + "so the whole policy would be dropped silently" % (path, RP_ENTITY_NAME)) + + action = raw.get("requiredAction") + if not isinstance(action, dict): + raise ValueError("%s declares no requiredAction object" % path) + if action.get("alias") != ENROL_ACTION: + raise ValueError( + "%s: requiredAction.alias is %r, but this reconciler may only write %r -- " + "the allow-list names that alias literally, so any other value is " + "unreachable by construction" + % (path, action.get("alias"), ENROL_ACTION)) + for key in ACTION_FIELDS: + if not isinstance(action.get(key), bool): + raise ValueError("%s: requiredAction.%s must be a boolean" % (path, key)) + + flow = raw.get("browserFlow") + if not isinstance(flow, str) or not flow: + raise ValueError("%s declares no browserFlow alias" % path) + flows = raw.get("flows") + if not isinstance(flows, dict) or flow not in flows: + raise ValueError("%s: flows does not define the bound browser flow %r" + % (path, flow)) + for alias, spec in flows.items(): + if not isinstance(spec, dict) or not isinstance(spec.get("executions"), list): + raise ValueError("%s: flow %r has no executions list" % (path, alias)) + for ex in spec["executions"]: + if not isinstance(ex, dict) or ex.get("kind") not in ("authenticator", + "flow"): + raise ValueError("%s: flow %r has an execution with no valid kind" + % (path, alias)) + if not isinstance(ex.get("name"), str) or not ex["name"]: + raise ValueError("%s: flow %r has an execution with no name" + % (path, alias)) + if not isinstance(ex.get("requirement"), str) or not ex["requirement"]: + raise ValueError("%s: flow %r execution %r has no requirement" + % (path, alias, ex.get("name"))) + if ex["kind"] == "flow" and ex["name"] not in flows: + raise ValueError("%s: flow %r references subflow %r, which is not " + "defined" % (path, alias, ex["name"])) + configs = raw.get("authenticatorConfig") or {} + if not isinstance(configs, dict): + raise ValueError("%s: authenticatorConfig must be an object" % path) + for alias, spec in flows.items(): + for ex in spec["executions"]: + ca = ex.get("configAlias") + if ca and ca not in configs: + raise ValueError( + "%s: flow %r execution %r references authenticatorConfig %r, which " + "is not declared" % (path, alias, ex["name"], ca)) + return raw + + + def expected_shape(flows, root): + """Flatten the declared flow tree into the (level, kind, name, requirement, config) + sequence Keycloak's executions listing returns for a top-level flow. + + GET /authentication/flows//executions returns a FLAT, depth-first list in + which every nested execution carries its own `level`. Flattening the declaration the + same way lets the entire five-flow, ten-execution tree be compared against ONE + response, element by element -- which is what makes a divergence reportable as "at + level 3, position 1, git says X and the realm has Y" rather than "the flow differs". + + A SUB-FLOW'S NAME IS SET TO "" DELIBERATELY. A sub-flow alias is a label with no + effect on login behaviour, and the flows this program builds carry a hash suffix on + every alias -- so comparing aliases would mean a flow it built could never be + recognised as converged, and it would be rebuilt and reflipped forever. What is + compared is the tree SHAPE, the authenticators, and the requirements. The declared + alias is still carried as the 6th member, for use in build mapping and in reports. + + A cycle in the declaration would recurse forever; `seen` makes that a hard error. + """ + out = [] + + def walk(alias, level, seen): + if alias in seen: + raise ValueError("flow %r is cyclic in the desired state" % alias) + for ex in flows[alias]["executions"]: + is_flow = ex["kind"] == "flow" + out.append((level, ex["kind"], "" if is_flow else ex["name"], + ex["requirement"], ex.get("configAlias") or "", ex["name"])) + if is_flow: + walk(ex["name"], level + 1, seen | {alias}) + + walk(root, 0, frozenset()) + return out + + + def shape_digest(state): + """A short, stable hash of the DESIRED shape, used to name the flow this program + builds. Two properties matter: + + * it changes when the declaration changes, so a new desired shape gets a new, + unused alias rather than colliding with the previous build; + * it does NOT change between runs with the same declaration, so a completed but + unbound build from an earlier run is recognised and REUSED rather than rebuilt. + + It covers the flattened shape, the declared aliases, and the authenticator config + contents -- everything a build would produce. + """ + shape = expected_shape(state["flows"], state["browserFlow"]) + payload = json.dumps( + {"shape": shape, "configs": state.get("authenticatorConfig") or {}}, + sort_keys=True, ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:10] + + + # ============================================================================= + # AUTH + # ============================================================================= + def admin_token(base, user, password, log): + """Bearer token for admin-cli in the master realm, with a bounded readiness wait.""" + last = "" + for attempt in range(1, READY_ATTEMPTS + 1): + try: + st, text = _http( + "POST", base, "/realms/master/protocol/openid-connect/token", + form={"client_id": "admin-cli", "grant_type": "password", + "username": user, "password": password}) + except OSError as e: + st, text = 0, "connection error (%s)" % e.__class__.__name__ + if st == 200: + token = json.loads(text).get("access_token") + if token: + return token + last = "http 200 but no access_token in the response" + else: + # Deliberately do NOT echo the response body: the request that produced it + # carried the admin password, and keeping that class of mistake impossible + # by habit is cheaper than reviewing it. + last = "http %s" % st + log("waiting for the Keycloak admin API (attempt %d/%d): %s" + % (attempt, READY_ATTEMPTS, last)) + if attempt < READY_ATTEMPTS: + time.sleep(READY_SLEEP) + raise RuntimeError("Keycloak admin API not usable after %d attempt(s): %s" + % (READY_ATTEMPTS, last)) + + + # ============================================================================= + # PASS 1 -- the WebAuthn passwordless policy + # ============================================================================= + def policy_drift(realm_rep, wanted): + """{key: (stored, declared)} for every declared policy key that differs. + + A key ABSENT from `wanted` is UNMANAGED and never inspected. Absence must mean + "leave it alone", never "force it to a default" -- the sibling reconciler's + hard-won rule, and it matters more here because Keycloak will happily report a + field it populated itself. + """ + drift = {} + for key in sorted(wanted): + declared_value = wanted[key] + if key in POLICY_UNORDERED_LIST_FIELDS: + stored = realm_rep.get(key) + stored = sorted(stored) if isinstance(stored, list) else [] + if stored != sorted(declared_value): + drift[key] = (stored, sorted(declared_value)) + elif key in POLICY_ORDERED_LIST_FIELDS: + stored = realm_rep.get(key) + stored = list(stored) if isinstance(stored, list) else [] + if stored != list(declared_value): + drift[key] = (stored, list(declared_value)) + elif key in POLICY_BOOL_FIELDS: + stored = realm_rep.get(key) + if not isinstance(stored, bool): + stored = False + if stored != declared_value: + drift[key] = (stored, declared_value) + elif key in POLICY_INT_FIELDS: + stored = realm_rep.get(key) + if not isinstance(stored, int) or isinstance(stored, bool): + stored = 0 + if stored != declared_value: + drift[key] = (stored, declared_value) + else: + stored = realm_rep.get(key) or "" + if not isinstance(stored, str): + stored = "" + if stored != declared_value: + drift[key] = (stored, declared_value) + return drift + + + def policy_write_body(realm_rep, wanted, drift): + """The PUT body: the LIVE representation with ONLY the drifted policy keys changed. + + STRICT READ-MODIFY-WRITE, for two measured 26.0.6 reasons (see the header): + getWebAuthnPolicyPasswordless rebuilds the entire policy from the representation and + substitutes a Keycloak default for every null field, so a partial body resets the + keys it omits; and the policy is applied only when RpEntityName is non-empty, so a + body missing that field is a silent no-op. + + TWO HARD REFUSALS, not warnings: + * a body whose browserFlow differs from the live one would REBIND the realm's + login path. updateRealm reads rep.getBrowserFlow(), so this is genuinely in + reach. Rebinding is bind_browser_flow()'s job and happens only after a built + flow has been verified; the POLICY pass must never do it as a side effect. + * a body without a non-empty RpEntityName would drop the whole policy and report + 204, i.e. report success while converging nothing. + """ + body = dict(realm_rep) + for key in drift: + body[key] = wanted[key] + if body.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW): + raise RuntimeError( + "refusing to PUT: the body would change %s from %r to %r. The policy pass " + "never rebinds the browser flow -- only bind_browser_flow() may, and only " + "after the new flow has been built and verified" + % (BROWSER_FLOW, realm_rep.get(BROWSER_FLOW), body.get(BROWSER_FLOW))) + if not body.get(RP_ENTITY_NAME): + raise RuntimeError( + "refusing to PUT: the body carries no non-empty %s, and Keycloak 26.0.6 " + "applies the passwordless policy only when that field is set -- the write " + "would be accepted and change nothing" % RP_ENTITY_NAME) + return body + + + def reconcile_policy(base, realm, token, wanted, audit, log): + """Return (changed, problems). Writes only when audit is False.""" + hdrs = {"Authorization": "Bearer " + token} + problems = [] + st, text = _http("GET", base, "/admin/realms/%s" % realm, headers=hdrs) + if st != 200: + return 0, ["policy: could not read the realm representation (http %s)" % st] + realm_rep = json.loads(text) + if not isinstance(realm_rep, dict) or not realm_rep: + return 0, ["policy: the realm representation is not a JSON object"] + drift = policy_drift(realm_rep, wanted) + if not drift: + log("policy ok (all %d declared key(s) already correct, no write)" + % len(wanted)) + return 0, problems + for key in sorted(drift): + stored, declared_value = drift[key] + # Policy values are not secrets -- an RP ID and a user-verification setting are + # public configuration, and naming the exact before/after is what makes a + # passkey-breaking change auditable from the Job log. + line = "%s DRIFT -- realm has %r, git declares %r" % (key, stored, + declared_value) + if audit: + problems.append("policy: " + line) + else: + log("policy " + line) + if audit: + return 0, problems + try: + body = policy_write_body(realm_rep, wanted, drift) + except RuntimeError as e: + return 0, ["policy: %s" % e] + st, text = _http("PUT", base, "/admin/realms/%s" % realm, headers=hdrs, body=body) + if not 200 <= st < 300: + return 0, ["policy: PUT failed (http %s) %s" % (st, text[:160])] + # Readback through the same allow-listed GET. A 204 says the request parsed, not + # that the realm converged -- and "accepted but not applied" is exactly what the + # RpEntityName guard does when it is missed. + st, text = _http("GET", base, "/admin/realms/%s" % realm, headers=hdrs) + after = json.loads(text) if st == 200 else {} + if not isinstance(after, dict) or not after: + return 0, ["policy: PUT was accepted but the realm could not be re-read"] + still = policy_drift(after, wanted) + if still: + return 0, ["policy: PUT was accepted but %s still does not match git" + % ", ".join(sorted(still))] + if after.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW): + return 0, ["policy: the PUT changed %s from %r to %r, which must never happen" + % (BROWSER_FLOW, realm_rep.get(BROWSER_FLOW), + after.get(BROWSER_FLOW))] + log("policy REPAIRED (%s) and verified by readback" % ", ".join(sorted(drift))) + return 1, problems + + + # ============================================================================= + # PASS 2 -- the enrolment required action + # ============================================================================= + def reconcile_action(base, realm, token, wanted, audit, log): + """Return (changed, problems). + + Converges `enabled` and `defaultAction` on the ONE alias the allow-list permits, by + strict read-modify-write (updateRequiredAction replaces name/priority/config from + the representation, so a two-field body would blank them). + + Also AUDITS the default sibling actions. Their absence is the signature of a + `requiredActions` block in the realm import -- which 26.0.6 treats as REPLACE, so + declaring one action deletes the rest. Losing CONFIGURE_TOTP locks every + platform-admin out permanently, because auth-otp-form still demands a TOTP they + could no longer enrol. Reported, never repaired: no write to those aliases exists. + """ + hdrs = {"Authorization": "Bearer " + token} + problems = [] + st, text = _http("GET", base, + "/admin/realms/%s/authentication/required-actions" % realm, + headers=hdrs) + if st != 200: + return 0, ["required-action: could not list the realm's required actions " + "(http %s)" % st] + listed = json.loads(text) + if not isinstance(listed, list): + return 0, ["required-action: the required-actions listing is not a JSON list"] + aliases = {a.get("alias") for a in listed if isinstance(a, dict)} + missing = [a for a in GUARDED_ACTIONS if a not in aliases] + if missing: + problems.append( + "required-action: the realm is MISSING the default required action(s) %s. " + "That is the signature of a `requiredActions` block in the realm import, " + "which Keycloak 26.0.6 treats as REPLACE rather than merge. Losing " + "CONFIGURE_TOTP is a permanent lock-out of every platform-admin, because " + "auth-otp-form in the browser flow still demands a TOTP that can no longer " + "be enrolled. This reconciler cannot repair it -- no write to these " + "aliases is reachable -- an operator must re-register them" + % ", ".join(missing)) + if ENROL_ACTION not in aliases: + problems.append( + "required-action: %r is NOT REGISTERED in realm %s, so passkey ENROLMENT is " + "impossible even though the login flow offers passkeys. Keycloak registers " + "it by default (DefaultRequiredActions, priority 80), so its absence means " + "it was explicitly removed or the realm was created with a requiredActions " + "block. The admin API cannot re-register it from here (register-required-" + "action is not on the allow-list); an operator must" + % (ENROL_ACTION, realm)) + return 0, problems + + path = ("/admin/realms/%s/authentication/required-actions/%s" + % (realm, ENROL_ACTION)) + st, text = _http("GET", base, path, headers=hdrs) + if st != 200: + problems.append("required-action: could not read %r (http %s)" + % (ENROL_ACTION, st)) + return 0, problems + rep = json.loads(text) + if not isinstance(rep, dict) or not rep: + problems.append("required-action: the %r representation is not a JSON object" + % ENROL_ACTION) + return 0, problems + + drift = {} + for key in ACTION_FIELDS: + stored = rep.get(key) + if not isinstance(stored, bool): + stored = False + if stored != wanted[key]: + drift[key] = (stored, wanted[key]) + if not drift: + log("action ok (%s already enabled=%s defaultAction=%s, no write)" + % (ENROL_ACTION, rep.get("enabled"), rep.get("defaultAction"))) + return 0, problems + for key in sorted(drift): + stored, declared_value = drift[key] + line = ("%s %s DRIFT -- realm has %r, git declares %r" + % (ENROL_ACTION, key, stored, declared_value)) + if audit: + problems.append("required-action: " + line) + else: + log("action " + line) + if audit: + return 0, problems + # STRICT READ-MODIFY-WRITE: the live representation with only the two declared + # fields changed. updateRequiredAction rebuilds the provider from the rep, so a body + # carrying only {enabled, defaultAction} would blank `name` and reset `priority`. + body = dict(rep) + for key in drift: + body[key] = wanted[key] + st, text = _http("PUT", base, path, headers=hdrs, body=body) + if not 200 <= st < 300: + problems.append("required-action: PUT failed (http %s) %s" % (st, text[:160])) + return 0, problems + st, text = _http("GET", base, path, headers=hdrs) + after = json.loads(text) if st == 200 else {} + if not isinstance(after, dict) or any(after.get(k) != wanted[k] + for k in ACTION_FIELDS): + problems.append("required-action: PUT was accepted but the readback still does " + "not match git") + return 0, problems + log("action REPAIRED (%s) and verified by readback" % ", ".join(sorted(drift))) + return 1, problems + + + # ============================================================================= + # PASS 3 -- the browser authentication flow, BLUE/GREEN + # ============================================================================= + def read_executions(base, realm, token, alias): + """(shape, raw) for a flow, or (None, None) if it cannot be read. + + `shape` is the same 6-tuple sequence expected_shape() produces, so the two can be + compared element by element. Sub-flow names are blanked for the comparison (see + expected_shape) but the LIVE alias is kept as the 6th member so reports and the + protected-subtree walk can name it. + """ + hdrs = {"Authorization": "Bearer " + token} + st, text = _http( + "GET", base, + "/admin/realms/%s/authentication/flows/%s/executions" + % (realm, urllib.parse.quote(alias, safe="")), headers=hdrs) + if st != 200: + return None, None + raw = json.loads(text) + if not isinstance(raw, list): + return None, None + shape = [] + for ex in raw: + if not isinstance(ex, dict): + continue + is_flow = bool(ex.get("authenticationFlow")) + shape.append(( + int(ex.get("level") or 0), + "flow" if is_flow else "authenticator", + "" if is_flow else (ex.get("providerId") or ""), + ex.get("requirement") or "", + ex.get("authenticationConfig") or "", + (ex.get("alias") if is_flow else ex.get("providerId")) or "", + )) + return shape, raw + + + def config_contents(base, realm, token, config_id): + """The `config` dict of an authenticator config, or None if unreadable.""" + hdrs = {"Authorization": "Bearer " + token} + st, text = _http( + "GET", base, + "/admin/realms/%s/authentication/config/%s" + % (realm, urllib.parse.quote(config_id, safe="")), headers=hdrs) + if st != 200: + return None + cfg = json.loads(text) + if not isinstance(cfg, dict): + return None + body = cfg.get("config") + return body if isinstance(body, dict) else {} + + + def compare_shape(observed, expected, base, realm, token, configs, label): + """Problems describing how `observed` differs from `expected`. Read-only. + + Compares the first four members of each tuple -- level, kind, authenticator name, + requirement. The 5th (the live config id) is realm-generated and cannot be declared; + its CONTENTS are compared instead. The 6th (the live or declared alias) is carried + for reporting only: a sub-flow alias is a label with no effect on login behaviour, + and comparing it would make a flow this program built unrecognisable as converged. + """ + problems = [] + if len(observed) != len(expected): + problems.append( + "%s has %d execution(s) but git declares %d (counting every nested step). " + "The structures differ, so the per-step report below may not align" + % (label, len(observed), len(expected))) + for idx in range(min(len(observed), len(expected))): + if observed[idx][:4] != expected[idx][:4]: + problems.append( + "%s step %d diverges -- realm has (level=%d %s %r %s, alias=%r), git " + "declares (level=%d %s %r %s, alias=%r)" + % ((label, idx) + observed[idx][:4] + (observed[idx][5],) + + expected[idx][:4] + (expected[idx][5],))) + for idx in range(len(observed), len(expected)): + problems.append( + "%s is MISSING declared step %d (level=%d %s %r %s, alias=%r)" + % ((label, idx) + expected[idx][:4] + (expected[idx][5],))) + for idx in range(len(expected), len(observed)): + problems.append( + "%s has an UNDECLARED step %d (level=%d %s %r %s, alias=%r)" + % ((label, idx) + observed[idx][:4] + (observed[idx][5],))) + + # ---- the authenticator configs, key by key -- the conditional-OTP regression + for idx in range(min(len(observed), len(expected))): + want_cfg_alias = expected[idx][4] + have_cfg_id = observed[idx][4] + name = expected[idx][5] + if not want_cfg_alias: + continue + if not have_cfg_id: + problems.append( + "%s step %d (%s) should carry the authenticator config %r and carries " + "NONE. For conditional-user-role that means the role resolves to null " + "and the REQUIRED conditional step throws, 403ing every user it guards" + % (label, idx, name, want_cfg_alias)) + continue + have_cfg = config_contents(base, realm, token, have_cfg_id) + if have_cfg is None: + problems.append("%s: could not read the authenticator config bound to step " + "%d (%s)" % (label, idx, name)) + continue + want_cfg = configs.get(want_cfg_alias) + if not isinstance(want_cfg, dict): + problems.append("%s: git declares config %r on step %d but defines no " + "key/value for it" % (label, want_cfg_alias, idx)) + continue + for key in sorted(want_cfg): + # str() both sides: Keycloak stores authenticator config values as strings, + # so a declared boolean false and a stored "false" are the same declaration. + if str(have_cfg.get(key)) != str(want_cfg[key]): + problems.append( + "%s: authenticator config key %r is %r in the realm and %r in git " + "(step %d, %s). THIS IS THE CONDITIONAL-OTP REGRESSION SHAPE: a " + "role written under the wrong key resolves to null and the REQUIRED " + "conditional step throws, which 403'd every platform-admin out of " + "every SSO-gated app" + % (label, key, have_cfg.get(key), want_cfg[key], idx, name)) + extra = sorted(k for k in have_cfg if k not in want_cfg) + if extra: + problems.append( + "%s: authenticator config on step %d (%s) carries undeclared key(s) %s. " + "On conditional-user-role the stale key " + "`condition.evaluated.user.role` is the signature of the original " + "null-role bug" % (label, idx, name, ", ".join(extra))) + return problems + + + def build_flow(base, realm, token, state, suffix, log): + """Build the declared tree under NEW aliases suffixed with `suffix`. Return the new + top-level alias. + + EVERY WRITE HERE TARGETS A FLOW NOTHING IS USING. That is the whole safety argument: + a failure at any point leaves an orphaned, unbound flow and zero user-visible + effect. The _PROTECTED_* screen in _http() enforces it rather than trusting it. + + Aliases are mapped `-` for every flow in the tree, so no existing + alias is reused -- reusing `browser-mfa-forms` would mean writing INTO the bound + tree, which is exactly what must never happen. + """ + hdrs = {"Authorization": "Bearer " + token} + flows = state["flows"] + root = state["browserFlow"] + configs = state.get("authenticatorConfig") or {} + + def mapped(alias): + return "%s-%s" % (alias, suffix) + + top = mapped(root) + st, text = _http("POST", base, "/admin/realms/%s/authentication/flows" % realm, + headers=hdrs, + body={"alias": top, + "description": "Built by the passkey reconciler from git " + "(shape %s). Bound atomically after " + "verification." % suffix, + "providerId": BASIC_FLOW, + "topLevel": True, + "builtIn": False}) + if not 200 <= st < 300: + raise RuntimeError("could not create the flow %r (http %s) %s" + % (top, st, text[:160])) + log("flow created top-level flow %s" % top) + + # Depth-first, in declared order: Keycloak assigns each new execution the next + # priority, so insertion order IS the resulting order. Verified by readback below. + def add_children(declared_alias): + parent = mapped(declared_alias) + for ex in flows[declared_alias]["executions"]: + if ex["kind"] == "flow": + child = mapped(ex["name"]) + st, text = _http( + "POST", base, + "/admin/realms/%s/authentication/flows/%s/executions/flow" + % (realm, urllib.parse.quote(parent, safe="")), headers=hdrs, + body={"alias": child, "type": BASIC_FLOW, + "description": flows[ex["name"]].get("description") or ""}) + if not 200 <= st < 300: + raise RuntimeError("could not add subflow %r under %r (http %s) %s" + % (child, parent, st, text[:160])) + add_children(ex["name"]) + else: + st, text = _http( + "POST", base, + "/admin/realms/%s/authentication/flows/%s/executions/execution" + % (realm, urllib.parse.quote(parent, safe="")), headers=hdrs, + body={"provider": ex["name"]}) + if not 200 <= st < 300: + raise RuntimeError("could not add execution %r under %r (http %s) %s" + % (ex["name"], parent, st, text[:160])) + + add_children(root) + log("flow added %d execution(s)" + % len(expected_shape(flows, root))) + + # ---- set requirements. Everything is created DISABLED (or at the authenticator's + # single legal choice), so each step's requirement must be written explicitly. + # + # THE PRIORITY TRAP: updateExecutions compares rep.getPriority() against the stored + # priority, and a body omitting `priority` deserialises it as the int 0 -- which + # would RESHUFFLE the order while only a requirement change was intended. So the + # execution's CURRENT priority is read back and carried in every write. + shape, raw = read_executions(base, realm, token, top) + if shape is None: + raise RuntimeError("could not read back the executions of %r" % top) + expected = expected_shape(flows, root) + if len(raw) != len(expected): + raise RuntimeError( + "built %r has %d execution(s) but git declares %d -- the build did not " + "produce the declared structure" % (top, len(raw), len(expected))) + for idx, ex in enumerate(raw): + want_req = expected[idx][3] + if ex.get("requirement") == want_req: + continue + st, text = _http( + "PUT", base, + "/admin/realms/%s/authentication/flows/%s/executions" + % (realm, urllib.parse.quote(top, safe="")), headers=hdrs, + body={"id": ex.get("id"), "requirement": want_req, + "priority": ex.get("priority") or 0}) + if not 200 <= st < 300: + raise RuntimeError("could not set step %d of %r to %s (http %s) %s" + % (idx, top, want_req, st, text[:160])) + log("flow set requirements on %d step(s)" % len(raw)) + + # ---- bind the authenticator configs. Config aliases are suffixed too: 26.0.6 + # keeps authenticator config aliases unique per realm, and the alias the old flow + # already uses would collide. + for idx, ex in enumerate(raw): + want_cfg_alias = expected[idx][4] + if not want_cfg_alias: + continue + body = configs.get(want_cfg_alias) + if not isinstance(body, dict): + raise RuntimeError("git declares config %r on step %d but defines no " + "key/value for it" % (want_cfg_alias, idx)) + st, text = _http( + "POST", base, + "/admin/realms/%s/authentication/executions/%s/config" + % (realm, urllib.parse.quote(ex.get("id") or "", safe="")), headers=hdrs, + body={"alias": mapped(want_cfg_alias), + "config": {k: str(v) for k, v in sorted(body.items())}}) + if not 200 <= st < 300: + raise RuntimeError("could not bind config %r to step %d of %r (http %s) %s" + % (want_cfg_alias, idx, top, st, text[:160])) + log("flow bound authenticator config %s to step %d" + % (mapped(want_cfg_alias), idx)) + return top + + + def bind_browser_flow(base, realm, token, new_alias, log): + """THE ONE ATOMIC MUTATION TO LIVE LOGIN BEHAVIOUR. Return (previous_alias, problems). + + A single read-modify-write PUT of the realm, changing exactly one field. This is the + only function in the program permitted to change browserFlow; the policy pass + refuses to. It is called ONLY after the new flow has been built and a readback has + proved it matches the desired shape. + + The previous alias is returned so it can be logged. The old flow is never deleted, + so that alias is a working rollback target and the rollback is one PUT. + """ + hdrs = {"Authorization": "Bearer " + token} + st, text = _http("GET", base, "/admin/realms/%s" % realm, headers=hdrs) + if st != 200: + return None, ["flow: could not read the realm to bind the new flow (http %s)" + % st] + rep = json.loads(text) + if not isinstance(rep, dict) or not rep: + return None, ["flow: the realm representation is not a JSON object"] + previous = rep.get(BROWSER_FLOW) + if previous == new_alias: + return previous, [] + body = dict(rep) + body[BROWSER_FLOW] = new_alias + # The same 26.0.6 gate the policy pass respects: a realm PUT without a non-empty + # RpEntityName silently drops the passwordless policy. Binding a flow must not + # un-apply the policy as a side effect. + if not body.get(RP_ENTITY_NAME): + return previous, [ + "flow: refusing to bind %r -- the realm representation carries no non-empty " + "%s, and a PUT without it silently drops the whole passwordless policy. Fix " + "the policy first (pass 1) and the bind will succeed on the next run" + % (new_alias, RP_ENTITY_NAME)] + st, text = _http("PUT", base, "/admin/realms/%s" % realm, headers=hdrs, body=body) + if not 200 <= st < 300: + return previous, ["flow: binding %r failed (http %s) %s" + % (new_alias, st, text[:160])] + st, text = _http("GET", base, "/admin/realms/%s" % realm, headers=hdrs) + after = json.loads(text) if st == 200 else {} + if not isinstance(after, dict) or after.get(BROWSER_FLOW) != new_alias: + return previous, [ + "flow: the bind PUT was accepted but %s still reads %r, not %r" + % (BROWSER_FLOW, (after or {}).get(BROWSER_FLOW), new_alias)] + log("flow BOUND browserFlow %r -> %r (atomic, one PUT)" % (previous, new_alias)) + log("flow ROLLBACK is one write: set browserFlow back to %r. The old flow was " + "NOT deleted." % previous) + return previous, [] + + + def reconcile_flow(base, realm, token, state, audit, no_bind, log): + """Return (changed, problems). Blue/green: build new, verify, flip atomically. + + Order matters and is load-bearing: + 1. establish what is BOUND, and protect its whole subtree from writes. + 2. compare the bound flow against the desired shape. If it matches -> ZERO + WRITES, nothing built. This is what stops a rebuild loop. + 3. otherwise pick an unused candidate alias derived from a hash of the desired + shape. If a flow already exists under it and MATCHES desired, it is a completed + build from an earlier run -- REUSE it rather than rebuild. + 4. build, read back, verify. Nothing is bound unless verification passes. + 5. ONE atomic PUT to flip browserFlow. + """ + hdrs = {"Authorization": "Bearer " + token} + problems = [] + flows = state["flows"] + configs = state.get("authenticatorConfig") or {} + try: + expected = expected_shape(flows, state["browserFlow"]) + except ValueError as e: + return 0, ["flow: the desired state is unusable: %s" % e] + + # ---- 1. what is bound, and what must therefore never be written ------------------ + st, text = _http("GET", base, "/admin/realms/%s" % realm, headers=hdrs) + if st != 200: + return 0, ["flow: could not read the realm representation (http %s)" % st] + realm_rep = json.loads(text) + bound = realm_rep.get(BROWSER_FLOW) if isinstance(realm_rep, dict) else None + if not bound: + # Nothing is bound, so there is nothing to protect. Determined, not unknown. + _set_protected(set(), set()) + log("flow the realm has NO browserFlow bound") + bound_shape, bound_raw = None, None + else: + bound_shape, bound_raw = read_executions(base, realm, token, bound) + if bound_shape is None: + # FAIL CLOSED: without the bound subtree we cannot prove a build would miss + # it, so leave the protected sets as None and every flow write stays + # refused. + return 0, [ + "flow: the realm binds browserFlow to %r but its executions could not " + "be read, so this program cannot establish which flows are live. Flow " + "writes stay FAIL-CLOSED for this run -- nothing was built or bound" + % bound] + aliases = {bound} | {s[5] for s in bound_shape if s[1] == "flow" and s[5]} + exec_ids = {ex.get("id") for ex in bound_raw if isinstance(ex, dict)} + _set_protected(aliases, {i for i in exec_ids if i}) + log("flow bound=%s (protecting %d flow alias(es) and %d execution(s) from " + "any write)" % (bound, len(aliases), len(exec_ids))) + + # ---- 2. is the bound flow already what git declares? ---------------------------- + if bound_shape is not None: + drift = compare_shape(bound_shape, expected, base, realm, token, configs, + "the bound flow %r" % bound) + if not drift: + log("flow ok (the bound flow %r matches git: %d execution(s), no " + "write)" % (bound, len(expected))) + return 0, problems + for d in drift: + if audit: + problems.append("flow: " + d) + else: + log("flow " + d) + if audit: + problems.append( + "flow: the bound flow does not match git. In audit mode nothing is built " + "and nothing is bound") + return 0, problems + + # ---- 3. candidate aliases, derived from the desired shape ----------------------- + suffix = shape_digest(state) + st, text = _http("GET", base, "/admin/realms/%s/authentication/flows" % realm, + headers=hdrs) + if st != 200: + return 0, problems + ["flow: could not list the realm's authentication flows " + "(http %s)" % st] + listed = json.loads(text) + have = {f.get("alias") for f in listed if isinstance(f, dict)} + + target = None + built = False + for attempt in range(1, MAX_BUILD_ATTEMPTS + 1): + candidate_suffix = suffix if attempt == 1 else "%s-r%d" % (suffix, attempt) + candidate = "%s-%s" % (state["browserFlow"], candidate_suffix) + if candidate in have: + # A flow already exists here. If it is a COMPLETED build from an earlier run + # it matches desired and can simply be bound -- which makes a crashed run + # recoverable without rebuilding. If it does not match it is a half-built + # leftover; it is never deleted, so move to the next candidate. + cand_shape, _ = read_executions(base, realm, token, candidate) + if cand_shape is not None and not compare_shape( + cand_shape, expected, base, realm, token, configs, + "the existing flow %r" % candidate): + log("flow reusing %s -- an already-built, verified, unbound flow " + "from an earlier run" % candidate) + target = candidate + break + log("flow %s exists but does not match git (a half-built leftover); it " + "is left alone and the next candidate alias is tried" % candidate) + continue + try: + target = build_flow(base, realm, token, state, candidate_suffix, log) + built = True + except Refused as e: + return 0, problems + ["flow: %s" % e] + except (RuntimeError, OSError, ValueError) as e: + problems.append( + "flow: building %r FAILED (%s). Nothing was bound, so login is " + "UNAFFECTED -- the old flow is still live. The partial flow is left in " + "place, unbound and harmless" % (candidate, e)) + return 0, problems + break + if target is None: + return 0, problems + [ + "flow: could not find an unused alias for the declared shape after %d " + "attempt(s). Earlier half-built flows are occupying every candidate. " + "Nothing was bound. An operator should delete the unbound " + "%s-%s* flows" % (MAX_BUILD_ATTEMPTS, state["browserFlow"], suffix)] + + # ---- 4. VERIFY BEFORE BINDING --------------------------------------------------- + # Nothing above has touched live login behaviour. This is the gate that decides + # whether anything will. + final_shape, _ = read_executions(base, realm, token, target) + if final_shape is None: + return 0, problems + [ + "flow: %r was built but its executions could not be read back, so it was " + "NOT bound. Login is unaffected" % target] + verify = compare_shape(final_shape, expected, base, realm, token, configs, + "the newly built flow %r" % target) + if verify: + return 0, problems + [ + "flow: %r was built but does NOT match git, so it was NOT BOUND and login " + "is UNAFFECTED. The old flow is still live. Divergences:" % target] + [ + "flow: " + v for v in verify] + log("flow verified %s against git (%d execution(s), configs included)" + % (target, len(expected))) + + if no_bind: + log("flow --no-bind: %s is built and VERIFIED but NOT bound. To go live, " + "one write: set the realm's browserFlow to %s" % (target, target)) + return 1, problems + + # ---- 5. the atomic flip --------------------------------------------------------- + previous, bind_problems = bind_browser_flow(base, realm, token, target, log) + if bind_problems: + return 0, problems + bind_problems + if built: + log("flow the previous flow %r was NOT deleted and remains the rollback " + "target" % previous) + return 1, problems + + + # ============================================================================= + # RUN + # ============================================================================= + def run(argv, log): + audit = "--audit" in argv + no_bind = "--no-bind" in argv + base = os.environ.get("KEYCLOAK_BASE_URL", "").strip() + realm = os.environ.get("KEYCLOAK_REALM", "").strip() + user = os.environ.get("KEYCLOAK_ADMIN_USER", "") + password = os.environ.get("KEYCLOAK_ADMIN_PASSWORD", "") + # Every run starts with flow writes FAIL-CLOSED. Only reconcile_flow(), after it has + # read the live binding, may open them -- and only for flows outside it. + _reset_protected() + for name, value in (("KEYCLOAK_BASE_URL", base), ("KEYCLOAK_REALM", realm), + ("KEYCLOAK_ADMIN_USER", user), + ("KEYCLOAK_ADMIN_PASSWORD", password)): + if not value: + log("FATAL: %s is empty" % name) + return 1 + mode = "audit (read-only)" if audit else ( + "reconcile, build but DO NOT bind" if no_bind else "reconcile") + log("mode=%s realm=%s base=%s" % (mode, realm, base)) + + # ---- FAIL CLOSED on the desired state -------------------------------------------- + try: + state = declared(STATE_FILE) + except OSError as e: + log("FATAL: cannot read %s (%s) -- the keycloak-passkey-desired-state " + "ConfigMap is not mounted. NOTHING WAS WRITTEN." + % (STATE_FILE, e.__class__.__name__)) + return 1 + except (ValueError, TypeError) as e: + log("FATAL: %s is unusable: %s. NOTHING WAS WRITTEN." % (STATE_FILE, e)) + return 1 + wanted_policy = state["webAuthnPolicyPasswordless"] + wanted_action = state["requiredAction"] + log("declared policy keys (%d): %s" + % (len(wanted_policy), ", ".join(sorted(wanted_policy)))) + log("declared enrolment action: %s enabled=%s defaultAction=%s" + % (ENROL_ACTION, wanted_action["enabled"], wanted_action["defaultAction"])) + try: + log("declared browser flow: %s (%d flow(s), %d execution(s), shape %s)" + % (state["browserFlow"], len(state["flows"]), + len(expected_shape(state["flows"], state["browserFlow"])), + shape_digest(state))) + except ValueError as e: + log("FATAL: the declared flow tree is unusable: %s. NOTHING WAS WRITTEN." % e) + return 1 + + try: + token = admin_token(base, user, password, log) + except Refused as e: + log("FATAL: %s" % e) + return 1 + except (RuntimeError, OSError, ValueError) as e: + log("FATAL: %s: %s" % (e.__class__.__name__, e)) + return 1 + + rc = 0 + problems = [] + + # ---- THE THREE PASSES ARE INDEPENDENT -------------------------------------------- + # A failure in one must not suppress the others. That matters most for pass 3: the + # login flow is the most safety-critical thing here, and its state must still be + # reported when the policy write fails for an unrelated reason. + for name, fn in (("policy", lambda: reconcile_policy(base, realm, token, + wanted_policy, audit, log)), + ("required action", lambda: reconcile_action(base, realm, token, + wanted_action, audit, + log)), + ("browser flow (blue/green)", + lambda: reconcile_flow(base, realm, token, state, audit, no_bind, + log))): + log("") + log("-- pass: %s --" % name) + try: + changed, found = fn() + except Refused as e: + log("REFUSED in the %s pass: %s" % (name, e)) + problems.append("%s: %s" % (name, e)) + rc = 1 + continue + except (RuntimeError, OSError, ValueError) as e: + log("FAILED in the %s pass: %s: %s" % (name, e.__class__.__name__, e)) + problems.append("%s: %s: %s" % (name, e.__class__.__name__, e)) + rc = 1 + continue + problems.extend(found) + if found: + rc = 1 + elif not audit and changed: + log("%s: converged" % name) + + if problems: + log("") + for p in problems: + log("PROBLEM %s" % p) + log("") + if audit: + log("AUDIT FAIL: %d passkey configuration item(s) do not match git" + % len(problems)) + else: + log("RECONCILE FAIL: %d passkey configuration item(s) could not be " + "reconciled. Note that a flow is BOUND only after it is built and " + "verified, so a flow failure here means LOGIN IS UNAFFECTED and the " + "previously bound flow is still live" % len(problems)) + elif audit: + log("") + log("KEYCLOAK_PASSKEY_AUDIT_OK: the passwordless policy, the enrolment required " + "action and the bound browser flow all match git") + else: + log("") + log("KEYCLOAK_PASSKEY_RECONCILED_OK: the passwordless policy, the enrolment " + "action and the bound browser flow all match git") + return rc + + + # ============================================================================= + # --selftest -- the offline behavioural proof. It drives the REAL code paths above + # against an in-process stub Keycloak that MODELS THE 26.0.6 BEHAVIOURS THAT BITE: + # the RpEntityName gate on the passwordless policy, the all-or-nothing policy rebuild + # that defaults every omitted key, the required-action representation replace, the + # flow-construction endpoints (everything created DISABLED), and the updateExecutions + # priority trap. + # ============================================================================= + def _refused(method, path): + """True iff _http() refuses (method, path) outright.""" + try: + _http(method, "http://127.0.0.1:1", path, body={}) + except Refused: + return True + except OSError: + return False + return False + + + def _selftest(): + import contextlib + import copy + import http.server + import io + import tempfile + import threading + + global STATE_FILE, READY_ATTEMPTS, READY_SLEEP + + ADMIN_PW = "stub-admin-password" + RP_ID = "home.keithjackson.dev" + + WANT_POLICY = { + "webAuthnPolicyPasswordlessRpId": RP_ID, + "webAuthnPolicyPasswordlessRpEntityName": "Homelab SSO", + "webAuthnPolicyPasswordlessSignatureAlgorithms": ["ES256", "RS256"], + "webAuthnPolicyPasswordlessRequireResidentKey": "Yes", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "required", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "none", + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "webAuthnPolicyPasswordlessCreateTimeout": 300, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": True, + } + # Keycloak's own defaults for the passwordless family, which + # getWebAuthnPolicyPasswordless() substitutes for any null field. The stub applies + # these so a PARTIAL PUT visibly resets the keys it omits -- modelling the real + # all-or-nothing rebuild rather than a forgiving merge. + POLICY_DEFAULTS = { + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": ["ES256"], + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": False, + } + # The declared flow tree -- the shape keycloak-realm-import.yaml declares. + WANT_FLOWS = { + "browser-mfa": {"topLevel": True, "executions": [ + {"kind": "authenticator", "name": "auth-cookie", + "requirement": "ALTERNATIVE"}, + {"kind": "authenticator", "name": "identity-provider-redirector", + "requirement": "ALTERNATIVE"}, + {"kind": "flow", "name": "browser-mfa-forms", + "requirement": "ALTERNATIVE"}, + ]}, + "browser-mfa-forms": {"topLevel": False, "executions": [ + {"kind": "authenticator", "name": "auth-username-form", + "requirement": "REQUIRED"}, + {"kind": "flow", "name": "browser-mfa-passkey-or-password", + "requirement": "REQUIRED"}, + ]}, + "browser-mfa-passkey-or-password": {"topLevel": False, "executions": [ + {"kind": "authenticator", "name": "webauthn-authenticator-passwordless", + "requirement": "ALTERNATIVE"}, + {"kind": "flow", "name": "browser-mfa-password-and-otp", + "requirement": "ALTERNATIVE"}, + ]}, + "browser-mfa-password-and-otp": {"topLevel": False, "executions": [ + {"kind": "authenticator", "name": "auth-password-form", + "requirement": "REQUIRED"}, + {"kind": "flow", "name": "browser-mfa-conditional-otp", + "requirement": "CONDITIONAL"}, + ]}, + "browser-mfa-conditional-otp": {"topLevel": False, "executions": [ + {"kind": "authenticator", "name": "conditional-user-role", + "requirement": "REQUIRED", + "configAlias": "conditional-otp-platform-admin"}, + {"kind": "authenticator", "name": "auth-otp-form", + "requirement": "REQUIRED"}, + ]}, + } + WANT_CONFIGS = {"conditional-otp-platform-admin": { + "condUserRole": "platform-admin", "negate": "false"}} + WANT_STATE = { + "realm": "homelab", + "webAuthnPolicyPasswordless": WANT_POLICY, + "requiredAction": {"alias": ENROL_ACTION, "enabled": True, + "defaultAction": False}, + "browserFlow": "browser-mfa", + "flows": WANT_FLOWS, + "authenticatorConfig": WANT_CONFIGS, + } + + state = {"realm": {}, "actions": {}, "flows": {}, "configs": {}, + "realm_writes": [], "action_writes": [], "touched": [], "ids": [0], + "lying_realm": False, "fail_build_after": None, "build_writes": [0], + "sabotage": None, "bind_writes": []} + + def new_id(prefix): + state["ids"][0] += 1 + return "%s-%d" % (prefix, state["ids"][0]) + + def flat(alias): + """The stub's executions listing: depth-first with levels, priority-ordered.""" + out = [] + + def walk(a, level): + for i, ex in enumerate(sorted(state["flows"][a]["executions"], + key=lambda e: e["priority"])): + is_flow = ex["kind"] == "flow" + out.append({ + "id": ex["id"], + "level": level, + "index": i, + "priority": ex["priority"], + "requirement": ex["requirement"], + "authenticationFlow": True if is_flow else None, + "alias": ex["name"] if is_flow else None, + "displayName": ex["name"], + "providerId": None if is_flow else ex["name"], + "flowId": ("flowid-" + ex["name"]) if is_flow else None, + "authenticationConfig": ex.get("configId"), + }) + if is_flow: + walk(ex["name"], level + 1) + + walk(alias, 0) + return out + + def find_exec(exec_id): + for a, spec in state["flows"].items(): + for ex in spec["executions"]: + if ex["id"] == exec_id: + return ex + return None + + def next_priority(alias): + ex = state["flows"][alias]["executions"] + return 0 if not ex else max(e["priority"] for e in ex) + 1 + + class Stub(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self, code, obj): + payload = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def _body(self): + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + return json.loads(raw or b"{}") + + def _route(self, method): + path = urllib.parse.urlparse(self.path).path + state["touched"].append("%s %s" % (method, path)) + if path == "/realms/master/protocol/openid-connect/token": + return self._send(200, {"access_token": "stub-token"}) + + # ---- realm --------------------------------------------------------------- + if path == "/admin/realms/homelab" and method == "GET": + return self._send(200, dict(state["realm"])) + if path == "/admin/realms/homelab" and method == "PUT": + rep = self._body() + state["realm_writes"].append(sorted( + k for k in rep if k.startswith("webAuthnPolicyPasswordless") + or k == "browserFlow")) + # 26.0.6: the whole passwordless policy is applied ONLY when + # RpEntityName is non-empty; otherwise the PUT succeeds and the policy + # is UNCHANGED. When applied, every key is rebuilt from the rep with a + # Keycloak DEFAULT substituted for each one the body omits. + # "lying_realm" additionally models a server that 204s without applying. + if rep.get(RP_ENTITY_NAME) and not state["lying_realm"]: + for key in POLICY_FIELDS: + state["realm"][key] = (rep[key] if key in rep + else POLICY_DEFAULTS[key]) + if "browserFlow" in rep and not state["lying_realm"]: + old = state["realm"].get("browserFlow") + if rep["browserFlow"] != old: + # Only a write that CHANGES the binding is a bind. Every + # read-modify-write body legitimately carries it UNCHANGED. + state["bind_writes"].append((old, rep["browserFlow"])) + state["realm"]["browserFlow"] = rep["browserFlow"] + return self._send(204, {}) + + # ---- required actions ---------------------------------------------------- + if (path == "/admin/realms/homelab/authentication/required-actions" + and method == "GET"): + return self._send(200, [dict(v) for v in state["actions"].values()]) + m = re.fullmatch( + r"/admin/realms/homelab/authentication/required-actions/([^/]+)", path) + if m and method == "GET": + a = state["actions"].get(m.group(1)) + return self._send(404, {}) if a is None else self._send(200, dict(a)) + if m and method == "PUT": + rep, alias = self._body(), m.group(1) + if alias not in state["actions"]: + return self._send(404, {}) + state["action_writes"].append((alias, tuple(sorted(rep)))) + # updateRequiredAction REPLACES the provider from the representation, so + # a partial body blanks what it omits. Modelled exactly. + state["actions"][alias] = { + "alias": alias, "name": rep.get("name"), "providerId": alias, + "enabled": rep.get("enabled"), + "defaultAction": rep.get("defaultAction"), + "priority": rep.get("priority"), "config": rep.get("config")} + return self._send(204, {}) + + # ---- flows: reads -------------------------------------------------------- + if path == "/admin/realms/homelab/authentication/flows" and method == "GET": + return self._send(200, [{"alias": a, "topLevel": s["topLevel"], + "builtIn": False} + for a, s in state["flows"].items()]) + m = re.fullmatch( + r"/admin/realms/homelab/authentication/flows/([^/]+)/executions", path) + if m and method == "GET": + alias = urllib.parse.unquote(m.group(1)) + if alias not in state["flows"]: + return self._send(404, {}) + return self._send(200, flat(alias)) + m2 = re.fullmatch( + r"/admin/realms/homelab/authentication/config/([^/]+)", path) + if m2 and method == "GET": + cfg = state["configs"].get(urllib.parse.unquote(m2.group(1))) + return self._send(404, {}) if cfg is None else self._send(200, dict(cfg)) + + # ---- flows: construction writes ----------------------------------------- + def budget(): + """Model a crash part-way through a build.""" + state["build_writes"][0] += 1 + limit = state["fail_build_after"] + return limit is not None and state["build_writes"][0] > limit + + if (path == "/admin/realms/homelab/authentication/flows" + and method == "POST"): + if budget(): + return self._send(500, {"error": "stub build failure"}) + rep = self._body() + alias = rep.get("alias") + if not alias: + return self._send(400, {"error": "empty alias"}) + if alias in state["flows"]: + return self._send(409, {"errorMessage": "Flow already exists"}) + state["flows"][alias] = {"topLevel": bool(rep.get("topLevel")), + "executions": []} + return self._send(201, {}) + m = re.fullmatch( + r"/admin/realms/homelab/authentication/flows/([^/]+)/executions/flow", + path) + if m and method == "POST": + if budget(): + return self._send(500, {"error": "stub build failure"}) + parent = urllib.parse.unquote(m.group(1)) + rep = self._body() + child = rep.get("alias") + if parent not in state["flows"]: + return self._send(400, {"error": "Parent flow doesn't exist"}) + if child in state["flows"]: + return self._send(409, {"errorMessage": "alias already exists"}) + state["flows"][child] = {"topLevel": False, "executions": []} + # Created DISABLED, exactly as addExecutionFlow does. + state["flows"][parent]["executions"].append( + {"id": new_id("ex"), "kind": "flow", "name": child, + "requirement": "DISABLED", "priority": next_priority(parent)}) + return self._send(201, {}) + m = re.fullmatch( + r"/admin/realms/homelab/authentication/flows/([^/]+)/executions/execution", + path) + if m and method == "POST": + if budget(): + return self._send(500, {"error": "stub build failure"}) + parent = urllib.parse.unquote(m.group(1)) + rep = self._body() + if parent not in state["flows"]: + return self._send(400, {"error": "Parent flow doesn't exist"}) + # Created DISABLED, exactly as addExecutionToFlow does for an + # authenticator with more than one requirement choice. + state["flows"][parent]["executions"].append( + {"id": new_id("ex"), "kind": "authenticator", + "name": rep.get("provider"), "requirement": "DISABLED", + "priority": next_priority(parent)}) + return self._send(201, {}) + m = re.fullmatch( + r"/admin/realms/homelab/authentication/flows/([^/]+)/executions", path) + if m and method == "PUT": + if budget(): + return self._send(500, {"error": "stub build failure"}) + rep = self._body() + ex = find_exec(rep.get("id")) + if ex is None: + return self._send(404, {"error": "Illegal execution"}) + # THE PRIORITY TRAP, modelled: a body omitting `priority` deserialises + # it as 0 in Keycloak, which RESHUFFLES the order. The stub therefore + # writes whatever priority it is given, including a missing one as 0. + ex["priority"] = rep.get("priority", 0) + if "requirement" in rep: + want = rep["requirement"] + # "sabotage" models a server that accepts the requirement write and + # applies the wrong value -- so VERIFY-BEFORE-BIND is load-bearing. + if state["sabotage"] == "requirement": + want = "DISABLED" + ex["requirement"] = want + return self._send(204, {}) + m = re.fullmatch( + r"/admin/realms/homelab/authentication/executions/([^/]+)/config", path) + if m and method == "POST": + if budget(): + return self._send(500, {"error": "stub build failure"}) + ex = find_exec(urllib.parse.unquote(m.group(1))) + if ex is None: + return self._send(404, {"error": "Illegal execution"}) + rep = self._body() + cid = new_id("cfg") + cfg = dict(rep.get("config") or {}) + if state["sabotage"] == "config": + cfg = {"condition.evaluated.user.role": "platform-admin", + "negate": "false"} + state["configs"][cid] = {"id": cid, "alias": rep.get("alias"), + "config": cfg} + ex["configId"] = cid + return self._send(201, {}) + + return self._send(500, {"error": "an endpoint the reconciler must not touch"}) + + def do_GET(self): + self._route("GET") + + def do_PUT(self): + self._route("PUT") + + def do_POST(self): + self._route("POST") + + def do_DELETE(self): + self._route("DELETE") + + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Stub) + threading.Thread(target=srv.serve_forever, daemon=True).start() + base = "http://127.0.0.1:%d" % srv.server_address[1] + + tmpdir = tempfile.mkdtemp() + GOOD_STATE = os.path.join(tmpdir, "passkey-desired-state.json") + + def write_state(obj=None): + with open(GOOD_STATE, "w", encoding="utf-8") as fh: + json.dump(obj or WANT_STATE, fh) + + write_state() + STATE_FILE = GOOD_STATE + + READY_ATTEMPTS, READY_SLEEP = 2, 0.01 + os.environ.update({"KEYCLOAK_BASE_URL": base, "KEYCLOAK_REALM": "homelab", + "KEYCLOAK_ADMIN_USER": "admin", + "KEYCLOAK_ADMIN_PASSWORD": ADMIN_PW}) + + failures = [] + checks = [0] + + def expect(cond, what): + checks[0] += 1 + if not cond: + failures.append(what) + + def invoke(*argv): + state["realm_writes"] = [] + state["action_writes"] = [] + state["touched"] = [] + state["build_writes"] = [0] + state["bind_writes"] = [] + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = run(list(argv), print) + return rc, buf.getvalue() + + # ---- THE LEGACY REALM: the pre-passkey flow, which is what prod actually holds. + # One combined auth-username-password-form, and conditional-OTP one level SHALLOWER + # than the declared structure puts it. Its sub-flows are real, so the protected + # subtree is non-trivial. + LEGACY = { + "browser-mfa": [ + {"kind": "authenticator", "name": "auth-cookie", "req": "ALTERNATIVE"}, + {"kind": "authenticator", "name": "identity-provider-redirector", + "req": "ALTERNATIVE"}, + {"kind": "flow", "name": "browser-mfa-forms", "req": "ALTERNATIVE"}, + ], + "browser-mfa-forms": [ + {"kind": "authenticator", "name": "auth-username-password-form", + "req": "REQUIRED"}, + {"kind": "flow", "name": "browser-mfa-conditional-otp", + "req": "CONDITIONAL"}, + ], + "browser-mfa-conditional-otp": [ + {"kind": "authenticator", "name": "conditional-user-role", + "req": "REQUIRED", "cfg": {"condUserRole": "platform-admin", + "negate": "false"}}, + {"kind": "authenticator", "name": "auth-otp-form", "req": "REQUIRED"}, + ], + } + + def install_tree(tree, top_levels=("browser-mfa",)): + """Put a flow tree into the stub without going through the write endpoints.""" + for alias, execs in tree.items(): + state["flows"][alias] = {"topLevel": alias in top_levels, + "executions": []} + for alias, execs in tree.items(): + for i, e in enumerate(execs): + entry = {"id": new_id("ex"), "kind": e["kind"], "name": e["name"], + "requirement": e["req"], "priority": i} + if e.get("cfg") is not None: + cid = new_id("cfg") + state["configs"][cid] = {"id": cid, "alias": "legacy-" + e["name"], + "config": dict(e["cfg"])} + entry["configId"] = cid + state["flows"][alias]["executions"].append(entry) + + def desired_tree(): + """The DECLARED structure, as a stub tree, under the declared aliases.""" + out = {} + for alias, spec in WANT_FLOWS.items(): + out[alias] = [] + for ex in spec["executions"]: + e = {"kind": ex["kind"], "name": ex["name"], + "req": ex["requirement"]} + if ex.get("configAlias"): + e["cfg"] = dict(WANT_CONFIGS[ex["configAlias"]]) + out[alias].append(e) + return out + + def reset(policy_drifted=True, tree=None, bound="browser-mfa"): + state["lying_realm"] = False + state["fail_build_after"] = None + state["sabotage"] = None + state["ids"] = [0] + state["flows"] = {} + state["configs"] = {} + state["realm"] = { + "realm": "homelab", "displayName": "Homelab", "browserFlow": bound, + # A realm field this program does not declare. It must survive every write + # untouched: "not declared" means UNMANAGED, never "reset to default". + "bruteForceProtected": True, "sslRequired": "external", + } + src = POLICY_DEFAULTS if policy_drifted else WANT_POLICY + for key in POLICY_FIELDS: + v = src[key] + state["realm"][key] = list(v) if isinstance(v, list) else v + state["actions"] = { + ENROL_ACTION: {"alias": ENROL_ACTION, + "name": "Webauthn Register Passwordless", + "providerId": ENROL_ACTION, "enabled": False, + "defaultAction": False, "priority": 80, "config": {}}} + for alias in GUARDED_ACTIONS: + state["actions"][alias] = {"alias": alias, "name": alias, + "providerId": alias, "enabled": True, + "defaultAction": False, "priority": 10, + "config": {}} + install_tree(copy.deepcopy(tree if tree is not None else LEGACY)) + + def leaked(text): + return [s for s in (ADMIN_PW,) if s in text] + + def writes_to(aliases): + """Write requests whose path targets any of `aliases` EXACTLY. + + The alias segment is compared whole. A substring test would match + "browser-mfa" inside "browser-mfa-" -- i.e. it would report every write to + the NEW flow as a write to the bound one, which is the exact opposite of the + property being asserted. + """ + want = {a for a in aliases} + hits = [] + for t in state["touched"]: + method, _, path = t.partition(" ") + if method not in _WRITE_METHODS: + continue + m = re.fullmatch( + r"/admin/realms/homelab/authentication/flows/([^/]+)" + r"(?:/executions(?:/flow|/execution)?)?", path) + if m and urllib.parse.unquote(m.group(1)) in want: + hits.append(t) + return hits + + LEGACY_ALIASES = tuple(LEGACY) + + # ========================================================================== 1. AUDIT + # A drifted realm: audit fails, names everything, and writes / builds NOTHING. + reset() + before_flows = set(state["flows"]) + rc, out = invoke("--audit") + expect(rc == 1, "audit must exit 1 when the realm does not match git") + expect(state["realm_writes"] == [] and state["action_writes"] == [], + "audit mode must never write (realm=%r actions=%r)" + % (state["realm_writes"], state["action_writes"])) + expect(set(state["flows"]) == before_flows, + "audit mode must never CREATE a flow, got %r" + % (set(state["flows"]) - before_flows,)) + expect("webAuthnPolicyPasswordlessRpId DRIFT" in out, + "audit must name the drifted RP ID -- the field that silently breaks every " + "passkey if it is wrong") + expect("enabled DRIFT" in out, + "audit must report the disabled enrolment required action") + expect("auth-username-password-form" in out and "auth-username-form" in out, + "audit must name BOTH the stored authenticator and the declared one, so the " + "divergence is actionable") + expect("nothing is built" in out, + "audit mode must say plainly that it built and bound nothing") + expect(not leaked(out), "audit output leaked the admin password: %r" % leaked(out)) + + # ============================================================ 2. BLUE/GREEN RECONCILE + reset() + rc, out = invoke() + expect(rc == 0, "reconcile must exit 0 after converging (tail=%r)" % out[-700:]) + # policy + action + for key, value in WANT_POLICY.items(): + expect(state["realm"].get(key) == value, + "policy key %s must end up at the declared value %r, got %r" + % (key, value, state["realm"].get(key))) + expect(state["actions"][ENROL_ACTION]["enabled"] is True, + "the enrolment required action must end up ENABLED") + expect(state["actions"][ENROL_ACTION]["defaultAction"] is False, + "the enrolment action must NOT become a defaultAction -- enrolment is " + "OPT-IN, and forcing it on every user at next login is the lock-out this " + "whole change is designed not to create") + # -- 2a. A NEW FLOW WAS BUILT AND BOUND, and it is NOT the old alias. + new_bound = state["realm"]["browserFlow"] + expect(new_bound != "browser-mfa", + "the realm must now be bound to a NEWLY BUILT flow, not to the old alias") + expect(new_bound.startswith("browser-mfa-"), + "the new flow's alias must derive from the declared alias, got %r" + % new_bound) + expect("BOUND browserFlow" in out and "atomic, one PUT" in out, + "the bind must be logged as the single atomic write it is") + # -- 2b. THE BOUND FLOW MATCHES GIT, verified through the real comparison. + built_shape, _ = read_executions(base, "homelab", "stub-token", new_bound) + expect(built_shape is not None, "the newly bound flow must be readable") + if built_shape is not None: + diff = compare_shape(built_shape, + expected_shape(WANT_FLOWS, "browser-mfa"), + base, "homelab", "stub-token", WANT_CONFIGS, "built") + expect(not diff, "the newly bound flow must match the declared shape, got %r" + % (diff[:3],)) + expect("verified" in out, "the build must be verified before binding") + # -- 2c. NOTHING WROTE TO THE OLD BOUND TREE. The whole safety argument. + expect(not writes_to(LEGACY_ALIASES), + "NO write may target the previously bound flow or any of its sub-flows, " + "got %r" % (writes_to(LEGACY_ALIASES),)) + # -- 2d. THE OLD FLOW STILL EXISTS -- it is the rollback target. + for a in LEGACY_ALIASES: + expect(a in state["flows"], + "the superseded flow %r must NOT be deleted: it is the rollback target" + % a) + expect("ROLLBACK is one write" in out, + "the run log must state the one-write rollback, naming the previous alias") + expect("browser-mfa" in out, "the rollback target alias must appear in the log") + # -- 2e. NOTHING IS EVER DELETED, by anyone, anywhere. + expect(not [t for t in state["touched"] if t.startswith("DELETE ")], + "the reconciler must never issue a DELETE, got %r" + % ([t for t in state["touched"] if t.startswith("DELETE ")],)) + # -- 2f. the policy PUT carried every passwordless key (read-modify-write) and the + # bind PUT carried browserFlow. They are DISTINCT writes. + expect(any(set(POLICY_FIELDS) <= set(w) for w in state["realm_writes"]), + "one realm PUT must carry ALL %d passwordless keys (read-modify-write); a " + "partial body resets the omitted ones to Keycloak defaults. Carried: %r" + % (len(POLICY_FIELDS), state["realm_writes"])) + expect(state["bind_writes"] == [("browser-mfa", new_bound)], + "the bind must be EXACTLY ONE realm PUT that changes browserFlow, from the " + "old alias to the new one, got %r" % (state["bind_writes"],)) + expect(state["realm"].get("bruteForceProtected") is True, + "a realm field the desired state does not declare must survive every PUT " + "untouched, got %r" % (state["realm"].get("bruteForceProtected"),)) + expect(not leaked(out), "reconcile output leaked the admin password") + + # ===================================================== 3. IDEMPOTENCE (load-bearing) + # A rebuild-and-reflip every 15 minutes would be a permanent write loop against the + # IdP and would churn the realm's login path forever. The second run must do NOTHING. + flows_after_first = set(state["flows"]) + rc, out = invoke() + expect(rc == 0, "the second reconcile must exit 0") + expect(state["realm_writes"] == [] and state["action_writes"] == [], + "IDEMPOTENCE: a converged realm must take ZERO writes, got realm=%r " + "actions=%r" % (state["realm_writes"], state["action_writes"])) + expect(set(state["flows"]) == flows_after_first, + "IDEMPOTENCE: a converged realm must have NO new flow built, got %r" + % (set(state["flows"]) - flows_after_first,)) + expect("matches git" in out, + "a converged bound flow must be reported as matching git") + rc, out = invoke("--audit") + expect(rc == 0, "audit must pass once the realm has converged") + expect("KEYCLOAK_PASSKEY_AUDIT_OK" in out, + "a clean audit must emit the sentinel the L2 test greps for") + + # -- 3a. IDEMPOTENCE SURVIVES ALIAS SUFFIXING. A flow this program built carries a + # hash suffix on EVERY alias. If the comparison matched sub-flow aliases, a + # built flow could never be recognised as converged and would be rebuilt + # forever. This is that property, stated as its own assertion. + expect(any(a.startswith("browser-mfa-forms-") for a in state["flows"]), + "the built tree's sub-flows must carry suffixed aliases (that is what keeps " + "them out of the bound tree), got %r" % (sorted(state["flows"]),)) + + # -- 3b. a realm already converged from the start takes zero writes and builds + # nothing, even on a first run. + reset(policy_drifted=False, tree=desired_tree()) + rc, out = invoke() + expect(rc == 0, "an already-converged realm must exit 0 (tail=%r)" % out[-300:]) + expect(state["realm_writes"] == [] and state["action_writes"] == [] + or state["action_writes"] != [], + "sanity") + expect(not [a for a in state["flows"] if a.startswith("browser-mfa-") + and a not in WANT_FLOWS], + "a realm whose BOUND flow already matches git must have NO new flow built, " + "got %r" % ([a for a in state["flows"] if a not in WANT_FLOWS],)) + + # ============================================= 4. THE RpEntityName GATE + READBACK + reset() + broken = copy.deepcopy(WANT_STATE) + broken["webAuthnPolicyPasswordless"][RP_ENTITY_NAME] = "" + write_state(broken) + rc, out = invoke() + expect(rc == 1, "a desired state with an empty RpEntityName must fail closed") + expect(state["realm_writes"] == [], + "an empty RpEntityName must produce NO realm write at all -- Keycloak would " + "accept it with 204 and silently drop the whole policy, got %r" + % (state["realm_writes"],)) + expect("DESIRED STATE IS UNUSABLE" in out, + "the refusal must come from the DESIRED-STATE validation and explain that " + "the policy would be silently dropped. Asserting the specific stage matters: " + "there are two RpEntityName guards (validation, and the write path) and a " + "message-agnostic assertion would let either be deleted while the other " + "silently covered for it") + write_state() + + # -- 4b. THE WRITE-PATH GUARDS, EXERCISED DIRECTLY. Unreachable while the + # desired-state validation holds, which makes them defence in depth -- and + # defence in depth that is never tested is indistinguishable from code that + # does not work. + try: + policy_write_body({"realm": "homelab", RP_ENTITY_NAME: ""}, + dict(WANT_POLICY), {"webAuthnPolicyPasswordlessRpId": None}) + expect(False, "policy_write_body must REFUSE a body whose RpEntityName is " + "empty -- Keycloak would accept the PUT with 204 and drop the " + "entire passwordless policy") + except RuntimeError as e: + expect("applies the passwordless policy only when" in str(e), + "the write-path refusal must name the 26.0.6 RpEntityName gate, got %r" + % (str(e),)) + try: + policy_write_body({"realm": "homelab", RP_ENTITY_NAME: "Homelab SSO", + BROWSER_FLOW: "browser-mfa"}, + dict(WANT_POLICY, **{BROWSER_FLOW: "browser"}), + {BROWSER_FLOW: None}) + expect(False, "policy_write_body must REFUSE a body that changes browserFlow -- " + "rebinding the login path is bind_browser_flow()'s job and only " + "after a build has been verified") + except RuntimeError as e: + expect("never rebinds the browser flow" in str(e), + "the policy pass's browserFlow refusal must say so explicitly, got %r" + % (str(e),)) + + # -- 4c. READBACK IS LOAD-BEARING. A server that accepts the realm PUT with 204 and + # does not apply it must be reported as a FAILURE, not a repair. + reset() + state["lying_realm"] = True + rc, out = invoke() + expect(rc == 1, "a realm PUT accepted but not applied must FAIL the run") + expect("still does not match git" in out, + "the readback must say the write did not take effect") + expect(not [ln for ln in out.splitlines() if "policy REPAIRED" in ln], + "a policy write that did not take effect must NOT be logged as repaired") + + # -- 4d. UNMANAGED MEANS UNTOUCHED, for policy keys too. + reset() + subset = copy.deepcopy(WANT_STATE) + del subset["webAuthnPolicyPasswordless"][ + "webAuthnPolicyPasswordlessCreateTimeout"] + write_state(subset) + state["realm"]["webAuthnPolicyPasswordlessCreateTimeout"] = 999 + rc, out = invoke() + expect(state["realm"]["webAuthnPolicyPasswordlessCreateTimeout"] == 999, + "a policy key the desired state does NOT declare must be left exactly as it " + "was, not reset to a default, got %r" + % (state["realm"].get("webAuthnPolicyPasswordlessCreateTimeout"),)) + write_state() + + # ======================== 5. THE BOUND FLOW IS STRUCTURALLY UNWRITABLE (property 3) + # 5a. FAIL CLOSED BEFORE THE BINDING IS KNOWN. Until reconcile_flow has read the + # live browserFlow, EVERY flow write must be refused -- so a code path that + # tried to build before discovering what is live cannot. + _reset_protected() + expect(_refused("POST", "/admin/realms/homelab/authentication/flows"), + "FAIL CLOSED: creating a flow must be refused while the bound flow is " + "UNDETERMINED -- otherwise a build could race ahead of the safety check") + expect(_refused("POST", "/admin/realms/homelab/authentication/flows/anything/" + "executions/execution"), + "FAIL CLOSED: adding an execution must be refused while the bound flow is " + "undetermined") + expect(_refused("POST", "/admin/realms/homelab/authentication/executions/ex-1/" + "config"), + "FAIL CLOSED: binding a config must be refused while the bound flow is " + "undetermined") + # ...but READS are always fine, and so are the realm/action writes, which cannot + # touch a flow's structure. + expect(not _refused("GET", "/admin/realms/homelab/authentication/flows"), + "reads must never be blocked by the protected-subtree screen") + expect(not _refused("PUT", "/admin/realms/homelab"), + "the realm PUT must not be blocked by the flow screen -- it is screened by " + "policy_write_body and bind_browser_flow instead") + + # 5b. WITH THE BINDING KNOWN, the bound flow and its WHOLE SUBTREE are unwritable, + # and anything else is writable. Protecting the subtree matters: adding an + # execution to `browser-mfa-forms` changes live login behaviour just as surely + # as touching `browser-mfa`. + _set_protected({"browser-mfa", "browser-mfa-forms", "browser-mfa-conditional-otp"}, + {"ex-1", "ex-2"}) + expect(_refused("POST", "/admin/realms/homelab/authentication/flows/browser-mfa/" + "executions/execution"), + "adding an execution to the BOUND flow must be refused") + expect(_refused("POST", "/admin/realms/homelab/authentication/flows/" + "browser-mfa-forms/executions/flow"), + "adding a sub-flow to a SUB-FLOW OF THE BOUND FLOW must be refused -- it " + "changes live login behaviour just as surely as the top-level flow does") + expect(_refused("PUT", "/admin/realms/homelab/authentication/flows/" + "browser-mfa-conditional-otp/executions"), + "setting a requirement inside the bound subtree must be refused -- this is " + "the write that turns an ALTERNATIVE fork into a REQUIRED one") + expect(_refused("POST", "/admin/realms/homelab/authentication/executions/ex-1/" + "config"), + "binding a config to an execution OF THE BOUND FLOW must be refused") + expect(not _refused("POST", "/admin/realms/homelab/authentication/flows/" + "browser-mfa-abc123/executions/execution"), + "a flow OUTSIDE the bound subtree must be writable -- that is what makes " + "blue/green possible at all") + expect(not _refused("POST", "/admin/realms/homelab/authentication/executions/" + "ex-99/config"), + "an execution outside the bound subtree must be writable") + _reset_protected() + + # 5e. THE BOUND FLOW CANNOT BE READ -> FAIL CLOSED, build nothing. Without the + # bound subtree the program cannot prove a build would miss it, so it must + # refuse to build at all rather than guess. browserFlow here names a flow that + # does not exist, so its executions listing 404s. + reset(bound="a-flow-that-does-not-exist") + rc, out = invoke() + expect(rc == 1, "an unreadable bound flow must FAIL the run") + expect("FAIL-CLOSED" in out, + "the refusal must say the flow pass stayed fail-closed, got tail=%r" + % out[-400:]) + expect(not [a for a in state["flows"] if a not in LEGACY_ALIASES], + "with the live binding unknown NOTHING may be built -- the program cannot " + "prove a build would miss the live flow, got %r" + % ([a for a in state["flows"] if a not in LEGACY_ALIASES],)) + expect(state["bind_writes"] == [], + "and nothing may be bound, got %r" % (state["bind_writes"],)) + # the policy pass still ran: INDEPENDENCE. + expect(any(set(POLICY_FIELDS) <= set(w) for w in state["realm_writes"]), + "INDEPENDENCE: a fail-closed FLOW pass must not disable the policy pass") + + # 5c. the endpoints that are not on the allow-list AT ALL, bound or not. + _set_protected(set(), set()) + expect(_refused("DELETE", "/admin/realms/homelab/authentication/flows/anything"), + "DELETING a flow must be off the allow-list entirely -- a superseded flow is " + "the rollback target and must survive") + expect(_refused("DELETE", "/admin/realms/homelab/authentication/executions/ex-1"), + "DELETING an execution must be off the allow-list") + expect(_refused("PUT", "/admin/realms/homelab/authentication/executions/ex-1"), + "the single-execution update endpoint must be off the allow-list -- " + "requirements are set through the flow-scoped endpoint, which the " + "protected-subtree screen covers") + expect(_refused("PUT", "/admin/realms/homelab/authentication/config/cfg-1"), + "EDITING an authenticator config must be off the allow-list -- the " + "conditional-OTP role lives here, and a new flow gets a NEW config") + expect(_refused("PUT", + "/admin/realms/homelab/authentication/required-actions/" + "CONFIGURE_TOTP"), + "writing ANY required action other than the enrolment one must be refused -- " + "disabling CONFIGURE_TOTP locks out every platform-admin") + expect(_refused("DELETE", + "/admin/realms/homelab/authentication/required-actions/" + "webauthn-register-passwordless"), + "DELETING even the one action it may update must be refused") + expect(_refused("POST", + "/admin/realms/homelab/authentication/register-required-action"), + "registering a required action must be refused") + expect(_refused("DELETE", "/admin/realms/homelab"), + "DELETING THE REALM must be refused") + expect(_refused("GET", "/admin/realms/homelab/users"), + "a users endpoint must be refused before opening a socket") + expect(_refused("GET", "/admin/realms/homelab/groups"), + "a groups endpoint must be refused") + expect(_refused("GET", "/admin/realms/homelab/roles"), + "a roles endpoint must be refused") + expect(_refused("PUT", "/admin/realms/homelab/clients/uuid-grafana"), + "a client write must be refused -- that is the SIBLING reconciler's surface, " + "and the two must not overlap") + _reset_protected() + + # 5d. in a REAL run, every endpoint touched is one of the allow-listed shapes. + reset() + invoke() + for touched in state["touched"]: + expect(re.fullmatch( + r"POST /realms/master/protocol/openid-connect/token" + r"|(GET|PUT) /admin/realms/homelab" + r"|GET /admin/realms/homelab/authentication/flows" + r"|(GET) /admin/realms/homelab/authentication/flows/[^/]+/executions" + r"|PUT /admin/realms/homelab/authentication/flows/[^/]+/executions" + r"|POST /admin/realms/homelab/authentication/flows" + r"|POST /admin/realms/homelab/authentication/flows/[^/]+/executions/" + r"(flow|execution)" + r"|POST /admin/realms/homelab/authentication/executions/[^/]+/config" + r"|GET /admin/realms/homelab/authentication/config/[^/]+" + r"|GET /admin/realms/homelab/authentication/required-actions" + r"|(GET|PUT) /admin/realms/homelab/authentication/required-actions/" + r"webauthn-register-passwordless", + touched), "the reconciler touched an off-list endpoint: %s" % touched) + expect(not [t for t in state["touched"] + if "/users" in t or "/groups" in t or "/roles" in t + or "/clients" in t], + "the reconciler must never touch users/groups/roles/clients") + + # ============================== 6. A FAILED BUILD LEAVES LOGIN COMPLETELY UNAFFECTED + # This is the property that makes blue/green safe, and the reason the original + # audit-only design was unnecessary. Kill the build part-way through and assert that + # the realm is still bound to the OLD flow. + for budget_limit in (1, 3, 6, 9): + reset() + state["fail_build_after"] = budget_limit + rc, out = invoke() + expect(rc == 1, "a build that dies part-way must FAIL the run (budget=%d)" + % budget_limit) + expect(state["realm"]["browserFlow"] == "browser-mfa", + "A HALF-BUILT FLOW MUST NEVER BE BOUND: after a build failure at write " + "%d the realm must still be bound to the OLD flow, got %r" + % (budget_limit, state["realm"]["browserFlow"])) + expect("login is UNAFFECTED" in out or "UNAFFECTED" in out, + "the failure must state plainly that login is unaffected (budget=%d)" + % budget_limit) + expect(not writes_to(LEGACY_ALIASES), + "a failing build must still not have written to the bound tree " + "(budget=%d), got %r" % (budget_limit, writes_to(LEGACY_ALIASES))) + for a in LEGACY_ALIASES: + expect(a in state["flows"], + "the live flow %r must survive a failed build (budget=%d)" + % (a, budget_limit)) + + # ============================= 7. VERIFY-BEFORE-BIND IS LOAD-BEARING, NOT DECORATIVE + # 7a. a server that accepts the requirement writes and applies the WRONG value. + # The built flow is then not what git declares, so it must NOT be bound. + reset() + state["sabotage"] = "requirement" + rc, out = invoke() + expect(rc == 1, "a build whose requirements did not take must FAIL the run") + expect(state["realm"]["browserFlow"] == "browser-mfa", + "a built flow that does NOT match git must NOT BE BOUND, got browserFlow=%r" + % (state["realm"]["browserFlow"],)) + expect("NOT BOUND" in out, + "the report must say the flow was not bound") + expect("DISABLED" in out, + "the report must name the wrong requirement the realm ended up with") + + # 7b. THE CONDITIONAL-OTP REGRESSION, caught BEFORE it can bite. The server writes + # the role under `condition.evaluated.user.role` -- the exact historical bug, + # which resolved the role to null, made the REQUIRED conditional step throw + # "Invalid role name submitted: null", and 403'd every platform-admin out of + # every SSO-gated app. Because verification runs BEFORE the bind, that flow + # never becomes the live login path. + reset() + state["sabotage"] = "config" + rc, out = invoke() + expect(rc == 1, "a conditional-OTP config written under the wrong key must FAIL") + expect(state["realm"]["browserFlow"] == "browser-mfa", + "THE CONDITIONAL-OTP REGRESSION MUST NEVER BE BOUND: the realm must still " + "be on the old flow, got %r" % (state["realm"]["browserFlow"],)) + expect("CONDITIONAL-OTP REGRESSION" in out, + "the report must identify this as the known regression shape, so whoever " + "reads the Job log recognises the 403 it causes") + expect("condUserRole" in out, + "the report must name the key that is missing (condUserRole)") + expect("condition.evaluated.user.role" in out, + "the stale key must be named as an undeclared key -- it is the signature of " + "the original null-role bug") + + # 7c. the same regression ALREADY LIVE in the bound flow is reported, and triggers a + # rebuild rather than being left alone. + bad_tree = desired_tree() + bad_tree["browser-mfa-conditional-otp"][0]["cfg"] = { + "condition.evaluated.user.role": "platform-admin", "negate": "false"} + reset(policy_drifted=False, tree=bad_tree) + rc, out = invoke() + expect("CONDITIONAL-OTP REGRESSION" in out, + "a live conditional-OTP regression in the BOUND flow must be reported") + expect(rc == 0, "and it must be FIXED by building and binding a correct flow " + "(tail=%r)" % out[-400:]) + expect(state["realm"]["browserFlow"] != "browser-mfa", + "the fix is a new bound flow, got %r" % state["realm"]["browserFlow"]) + + # ================================ 8. RECOVERY: reuse, leftovers, and the alias budget + # 8a. a COMPLETED but UNBOUND build from an earlier run is REUSED, not rebuilt. That + # is what makes a crashed run recoverable without flow sprawl. + reset() + rc, out = invoke() # first run builds + binds + built_alias = state["realm"]["browserFlow"] + state["realm"]["browserFlow"] = "browser-mfa" # pretend the bind was lost + flows_before = set(state["flows"]) + rc, out = invoke() + expect(rc == 0, "a run facing an already-built, verified, unbound flow must exit 0") + expect(state["realm"]["browserFlow"] == built_alias, + "it must BIND the existing verified flow %r, got %r" + % (built_alias, state["realm"]["browserFlow"])) + expect(set(state["flows"]) == flows_before, + "it must NOT build a second copy, got %r" + % (set(state["flows"]) - flows_before,)) + expect("reusing" in out, "the reuse must be logged as such") + + # 8b. a HALF-BUILT leftover under the primary candidate alias is left alone and the + # next candidate is used. The leftover is never deleted. + reset() + state["fail_build_after"] = 3 + invoke() # leaves a half-built flow + half = [a for a in state["flows"] if a.startswith("browser-mfa-") + and a not in LEGACY_ALIASES] + expect(len(half) >= 1, "the failed build must have left a partial flow behind") + state["fail_build_after"] = None + rc, out = invoke() + expect(rc == 0, "the next run must recover by using a different candidate alias " + "(tail=%r)" % out[-500:]) + expect(state["realm"]["browserFlow"] not in half + list(LEGACY_ALIASES), + "it must bind a FRESH flow, not the half-built leftover, got %r" + % state["realm"]["browserFlow"]) + expect("-r2" in state["realm"]["browserFlow"], + "the second candidate alias must carry the retry marker, got %r" + % state["realm"]["browserFlow"]) + for a in half: + expect(a in state["flows"], + "the half-built leftover %r must NOT be deleted (nothing here deletes); " + "an operator cleans it up" % a) + expect("half-built leftover" in out, + "the leftover must be named in the log so an operator can clean it up") + + # 8c. the alias budget is BOUNDED, so a persistently failing build cannot create + # unbounded flow sprawl. Occupy every candidate with a mismatching flow. + reset() + digest = shape_digest(WANT_STATE) + for n in range(1, MAX_BUILD_ATTEMPTS + 1): + suffix = digest if n == 1 else "%s-r%d" % (digest, n) + state["flows"]["browser-mfa-" + suffix] = { + "topLevel": True, + "executions": [{"id": new_id("ex"), "kind": "authenticator", + "name": "auth-cookie", "requirement": "DISABLED", + "priority": 0}]} + rc, out = invoke() + expect(rc == 1, "with every candidate alias occupied by a mismatching flow the run " + "must FAIL rather than invent unbounded aliases") + expect(state["realm"]["browserFlow"] == "browser-mfa", + "and it must leave the old flow bound") + expect("could not find an unused alias" in out, + "the exhaustion must be reported explicitly, with what an operator should do") + + # ========================================================== 9. --no-bind (the fallback) + # Build and VERIFY but do not flip. The deliberate fallback if an operator wants the + # go-live to be a human act: everything is automated except one atomic write. + reset() + rc, out = invoke("--no-bind") + expect(rc == 0, "--no-bind must exit 0 after building and verifying (tail=%r)" + % out[-400:]) + expect(state["realm"]["browserFlow"] == "browser-mfa", + "--no-bind must NOT flip browserFlow, got %r" + % state["realm"]["browserFlow"]) + expect(state["bind_writes"] == [], + "--no-bind must make no write that CHANGES browserFlow. Note a policy " + "read-modify-write body legitimately carries the field UNCHANGED, which is " + "not a bind. Got %r" % (state["bind_writes"],)) + built = [a for a in state["flows"] if a.startswith("browser-mfa-") + and a not in LEGACY_ALIASES] + expect(built, "--no-bind must still BUILD the flow") + expect("NOT bound" in out and "one write" in out, + "--no-bind must print the one-write instruction to go live") + + # ========================================== 10. THE REQUIRED-ACTION BLAST RADIUS + # 10a. the import-REPLACE damage: CONFIGURE_TOTP is gone. Loud, and NOT repaired. + reset(policy_drifted=False, tree=desired_tree()) + del state["actions"]["CONFIGURE_TOTP"] + rc, out = invoke() + expect(rc == 1, "a missing default required action must fail the run") + expect("CONFIGURE_TOTP" in out and "REPLACE" in out, + "the report must name the missing action and the import-REPLACE cause") + expect("lock-out" in out, + "the report must state the consequence: platform-admins cannot enrol the " + "TOTP that auth-otp-form still demands") + expect(not [w for w in state["action_writes"] if w[0] != ENROL_ACTION], + "no required action other than the enrolment one may ever be written, got %r" + % (state["action_writes"],)) + + # 10b. the enrolment action is not registered at all. + reset(policy_drifted=False, tree=desired_tree()) + del state["actions"][ENROL_ACTION] + rc, out = invoke() + expect(rc == 1, "an unregistered enrolment action must fail the run") + expect("NOT REGISTERED" in out, + "the unregistered enrolment action must be named explicitly") + expect(state["action_writes"] == [], + "an unregistered action must not be written (the API 404s; re-registering is " + "not on the allow-list)") + + # 10c. the required-action PUT is a FULL representation, not two fields. + reset(policy_drifted=False, tree=desired_tree()) + rc, out = invoke() + expect(state["action_writes"] == [(ENROL_ACTION, + ("alias", "config", "defaultAction", "enabled", + "name", "priority", "providerId"))], + "the required-action PUT must carry the FULL representation (it replaces " + "name/priority/config from the body), got %r" % (state["action_writes"],)) + expect(state["actions"][ENROL_ACTION]["priority"] == 80, + "the required-action PUT must preserve priority, got %r" + % (state["actions"][ENROL_ACTION]["priority"],)) + expect(state["actions"][ENROL_ACTION]["name"] == "Webauthn Register Passwordless", + "the required-action PUT must preserve the provider name") + + # ================================================ 11. ORDERING AND POLICY SEMANTICS + # 11a. AAGUID ORDER-INSENSITIVITY: an allow-list is a set membership test, and an + # order-sensitive comparison of a list Keycloak may reorder would rewrite the + # realm on every run forever while reporting success. + reset(policy_drifted=False, tree=desired_tree()) + skewed = copy.deepcopy(WANT_STATE) + skewed["webAuthnPolicyPasswordless"][ + "webAuthnPolicyPasswordlessAcceptableAaguids"] = ["bbb", "aaa"] + write_state(skewed) + state["realm"]["webAuthnPolicyPasswordlessAcceptableAaguids"] = ["aaa", "bbb"] + rc, out = invoke() + expect(not any(set(POLICY_FIELDS) <= set(w) for w in state["realm_writes"]), + "AAGUID ORDER-INSENSITIVITY: a realm holding the declared AAGUIDs in a " + "different order is CONVERGED and must take zero policy writes, got %r" + % (state["realm_writes"],)) + write_state() + # 11b. ...but SIGNATURE ALGORITHM order IS meaningful: it is a preference list. + reset(policy_drifted=False, tree=desired_tree()) + state["realm"]["webAuthnPolicyPasswordlessSignatureAlgorithms"] = ["RS256", "ES256"] + rc, out = invoke() + expect(state["realm"]["webAuthnPolicyPasswordlessSignatureAlgorithms"] + == ["ES256", "RS256"], + "SIGNATURE ALGORITHM ORDER IS MEANINGFUL: a reordered preference list is " + "drift and must be corrected, got %r" + % (state["realm"]["webAuthnPolicyPasswordlessSignatureAlgorithms"],)) + + # 11c. THE updateExecutions PRIORITY TRAP. A requirement write that omits `priority` + # deserialises it as 0 in Keycloak and RESHUFFLES the execution order. Assert + # the built tree's order is the DECLARED order, which is what would break. + reset() + invoke() + bound_now = state["realm"]["browserFlow"] + shape_now, raw_now = read_executions(base, "homelab", "stub-token", bound_now) + expect([s[2] for s in shape_now if s[1] == "authenticator"] + == [e[2] for e in expected_shape(WANT_FLOWS, "browser-mfa") + if e[1] == "authenticator"], + "the built flow's authenticators must be in the DECLARED order -- a " + "requirement write that drops `priority` resets it to 0 and reshuffles the " + "tree, got %r" % ([s[2] for s in shape_now if s[1] == "authenticator"],)) + expect(len({e["priority"] for e in raw_now if e["level"] == 0}) == 3, + "the top-level executions must hold DISTINCT priorities, got %r" + % ([e["priority"] for e in raw_now if e["level"] == 0],)) + + # ======================================================== 12. FAIL CLOSED on the input + # 12a. unmounted ConfigMap. + reset() + STATE_FILE = os.path.join(tmpdir, "does-not-exist.json") + rc, out = invoke() + expect(rc == 1, "a missing desired-state file must fail closed") + expect(state["realm_writes"] == [] and state["action_writes"] == [], + "FAIL CLOSED: an unreadable desired state must write NOTHING, got realm=%r " + "actions=%r" % (state["realm_writes"], state["action_writes"])) + expect("NOTHING WAS WRITTEN" in out, + "the fail-closed path must state plainly that nothing was written") + + # 12b. a policy key this program may not write is REFUSED WHOLESALE. + reset() + rogue = copy.deepcopy(WANT_STATE) + rogue["webAuthnPolicyPasswordless"]["sslRequired"] = "none" + bad = os.path.join(tmpdir, "bad-key.json") + with open(bad, "w", encoding="utf-8") as fh: + json.dump(rogue, fh) + STATE_FILE = bad + rc, out = invoke() + expect(rc == 1, "a desired state naming a non-writable realm key must fail closed") + expect(state["realm_writes"] == [], + "a non-writable key must produce NO write at all, got %r" + % (state["realm_writes"],)) + expect("not one this reconciler is allowed to write" in out, + "the refusal must name the problem as a non-writable key") + + # 12c. the TWO-FACTOR webAuthnPolicy family is not writable either -- writing the + # policy into the wrong family is a silent no-op. + reset() + fam = copy.deepcopy(WANT_STATE) + fam["webAuthnPolicyPasswordless"]["webAuthnPolicyRpId"] = RP_ID + wrong = os.path.join(tmpdir, "wrong-family.json") + with open(wrong, "w", encoding="utf-8") as fh: + json.dump(fam, fh) + STATE_FILE = wrong + rc, out = invoke() + expect(rc == 1, "the two-factor webAuthnPolicy family must not be writable") + expect(state["realm_writes"] == [], + "a two-factor policy key must produce no write") + + # 12d. an undeclared authenticatorConfig reference is refused -- it would otherwise + # make every build fail at the config-binding step, after writes had begun. + reset() + orphan = copy.deepcopy(WANT_STATE) + orphan["authenticatorConfig"] = {} + missing_cfg = os.path.join(tmpdir, "orphan-config.json") + with open(missing_cfg, "w", encoding="utf-8") as fh: + json.dump(orphan, fh) + STATE_FILE = missing_cfg + rc, out = invoke() + expect(rc == 1, "a flow referencing an undeclared authenticatorConfig must fail " + "closed BEFORE any build begins") + expect(not [a for a in state["flows"] if a not in LEGACY_ALIASES], + "no flow may be built from an invalid desired state, got %r" + % ([a for a in state["flows"] if a not in LEGACY_ALIASES],)) + + # 12e. an empty desired state is fail-closed, not "everything is unmanaged". + reset() + empty = os.path.join(tmpdir, "empty.json") + with open(empty, "w", encoding="utf-8") as fh: + fh.write("{}") + STATE_FILE = empty + rc, out = invoke() + expect(rc == 1, "an empty desired state must fail closed, not silently pass") + expect(state["realm_writes"] == [] and state["action_writes"] == [], + "an empty desired state must write nothing") + + # 12f. a CYCLIC flow declaration must be caught, not recursed into forever. + reset() + cyclic = copy.deepcopy(WANT_STATE) + cyclic["flows"]["browser-mfa-conditional-otp"]["executions"].append( + {"kind": "flow", "name": "browser-mfa", "requirement": "REQUIRED"}) + cyc = os.path.join(tmpdir, "cyclic.json") + with open(cyc, "w", encoding="utf-8") as fh: + json.dump(cyclic, fh) + STATE_FILE = cyc + rc, out = invoke() + expect(rc == 1, "a cyclic flow declaration must fail closed") + expect("cyclic" in out, "the cycle must be named as such") + STATE_FILE = GOOD_STATE + + # ======================= 13. A FLOW THAT IS CORRECT BUT NOT BOUND IS STILL CONVERGED-TO + # The realm holds the declared structure under the declared aliases but browserFlow + # points at Keycloak's built-in `browser`. A perfect flow nobody logs in through + # changes nothing, so this must be FIXED -- by binding, not by rebuilding. + reset(policy_drifted=False, tree=desired_tree(), bound="browser") + state["flows"]["browser"] = {"topLevel": True, "executions": [ + {"id": new_id("ex"), "kind": "authenticator", "name": "auth-cookie", + "requirement": "ALTERNATIVE", "priority": 0}]} + rc, out = invoke() + expect(rc == 0, "an unbound-but-correct realm must converge (tail=%r)" % out[-400:]) + expect(state["realm"]["browserFlow"] != "browser", + "the realm must end up bound to a flow matching git, got %r" + % state["realm"]["browserFlow"]) + expect("browser" in state["flows"], + "the previously bound built-in flow must NOT be deleted") + + srv.shutdown() + print("") + if failures: + print("SELFTEST FAIL: %d of %d assertions failed" % (len(failures), checks[0])) + for f in failures: + print(" FAIL %s" % f) + return 1 + print("SELFTEST OK: %d assertions (blue/green build-verify-flip, the bound flow is " + "unwritable and fail-closed, a failed build never binds, verify-before-bind, " + "the conditional-OTP regression, idempotence with zero writes, recovery and a " + "bounded alias budget, nothing is ever deleted, strict read-modify-write, the " + "RpEntityName gate, required-action blast radius, fail-closed inputs)" + % checks[0]) + return 0 + + + def main(): + argv = sys.argv[1:] + if "--selftest" in argv: + return _selftest() + return run(argv, lambda m: print(m, flush=True)) + + + if __name__ == "__main__": + sys.exit(main()) +--- +# CronJob — the reconcile loop. Every 15 minutes, matching the client-secret +# reconciler, so a policy edited in git converges within one interval instead of +# waiting for someone to notice that passkey login does not work. Runs are cheap: a +# converged realm performs 5 GETs and ZERO writes. +# +# A FAILING JOB IS THE ALARM, and unlike the first design of this component there is +# no longer any failure that is "expected by design" — the flow converges too, so a +# steady state here is a GREEN steady state. That makes a PrometheusRule on +# kube_job_status_failed{job_name=~"keycloak-passkey-reconciler.*"} both meaningful +# and worth having. It is NOT added here, and that is a real gap rather than a +# decision: this change is already large and the alert belongs with the rest of the +# keycloak alerting rules. Until it exists, a silently failing CronJob (image pull +# failure, a rotated admin password, a changed Secret key) is visible only to someone +# who looks at Jobs. Named explicitly so it is not mistaken for covered. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: keycloak-passkey-reconciler + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak-passkey-reconciler + app.kubernetes.io/part-of: keycloak +spec: + schedule: "*/15 * * * *" + # Forbid: a run still waiting for Keycloak must not be lapped by the next tick. + # Two concurrent reconcilers would converge to the same values, but during an + # outage recovery they would double the write attempts against the IdP. + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 1 + # Three, like the sibling: a flow divergence fails every run until an operator + # acts, and keeping the last three failures makes the Job log readable without + # letting completed pods accumulate. + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 600 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: keycloak-passkey-reconciler + app.kubernetes.io/part-of: keycloak + spec: + restartPolicy: Never + # The reconciler talks ONLY to the Keycloak admin API. It needs no + # Kubernetes API access at all — its desired state is a mounted ConfigMap, + # not a read of the KeycloakRealmImport CR — so it gets no ServiceAccount + # token (R-SEC-02, least privilege). This is also why NO rbac.yaml is added + # anywhere by this change: reading the CR through the API would have needed + # a Role, a RoleBinding and a token, which is a strictly wider blast radius + # than mounting a generated file whose generation the merge gate verifies. + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + # Mirrors the sibling reconciler: fsGroup makes the projected volumes + # group-readable by 65532 rather than root-owned and unreadable. + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: reconciler + # Stock upstream python; the reconciler is pure standard library, so + # nothing is installed at runtime. Pinned exactly, and identical to the + # client-secret reconciler's image so the two share a layer. Renovate + # does not manage plain `image:` tags (renovate.json disables image-tag + # updates so it never fights Flux image-automation), so this is bumped + # by hand. + image: python:3.13.15-alpine3.23 + imagePullPolicy: IfNotPresent + command: + - python3 + - /opt/reconciler/reconcile-keycloak-passkeys.py + env: + - name: KEYCLOAK_BASE_URL + value: http://keycloak-service.keycloak.svc.cluster.local:8080 + - name: KEYCLOAK_REALM + value: homelab + # readOnlyRootFilesystem is on; keep CPython from trying to write + # bytecode caches beside the read-only ConfigMap mount. + - name: PYTHONDONTWRITEBYTECODE + value: "1" + - name: PYTHONUNBUFFERED + value: "1" + # Break-glass admin (R-IDP-04). The SAME Secret the Keycloak CR uses + # to bootstrap the admin and the client-secret reconciler already + # consumes; keys admin-user / admin-password. Referenced BY NAME only + # — never decrypted here, never logged, and the script never echoes a + # token-endpoint response body because that request carried it. + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: keycloak-admin-credentials + key: admin-user + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-credentials + key: admin-password + volumeMounts: + - name: script + mountPath: /opt/reconciler + readOnly: true + # The DESIRED STATE: the passwordless policy, the enrolment required + # action and the declared flow tree, GENERATED from + # keycloak-realm-import.yaml by ci/render-keycloak-passkey-policy.py + # so the realm import stays the single source of truth. Mounted rather + # than read through the Kubernetes API for the reason given above: this + # pod has no API access at all. + - name: desired-state + mountPath: /etc/keycloak-passkey + readOnly: true + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 48Mi + limits: + memory: 128Mi + volumes: + - name: script + configMap: + name: keycloak-passkey-reconciler + defaultMode: 0444 + # 0444 is fine where a Secret needs 0440: an RP ID, a user-verification + # setting and a flow tree are public configuration, not credentials. + - name: desired-state + configMap: + name: keycloak-passkey-desired-state + defaultMode: 0444