#!/usr/bin/env python3 # ============================================================================= # mutations.py -- prove the passkey reconciler's --selftest IS FALSIFIABLE. # # WHY THIS EXISTS # --------------- # A selftest that passes proves nothing on its own: a selftest asserting `True == # True` also passes. This reconciler exists because a one-shot realm import # REPORTED SUCCESS while the realm was never converged (Done=True / Started=False # with the whole passkey block inert, measured on prod 2026-10-18), and the same # trap applies one level up -- a green test suite that cannot fail is the same lie # in a different place. # # It matters more here than for the sibling client reconciler, because this program # CONSTRUCTS AUTHENTICATION FLOWS and then REBINDS THE REALM'S LOGIN PATH. An # untested safety property on that surface is not a documentation gap, it is an # outage waiting for a schedule tick. # # So this injects a series of PLAUSIBLE regressions into the shipped script, one at # a time, and requires that --selftest FAILS for every one. A mutation that # SURVIVES means the suite does not actually check that property, and the check # fails, naming it. # # THE FIRST SIX ARE THE BLUE/GREEN SAFETY ARGUMENT ITSELF. The design builds a new # flow under an unused alias, verifies it, and flips `browserFlow` in one atomic # write -- never mutating the flow users are logging in through. Each of those # properties is enforced in code; these mutations are the proof that the enforcement # is load-bearing rather than commentary. Several model the obvious "simplification" # someone will propose: # * "just build it under the declared alias, the suffix is ugly" # * "bind it first, then check -- the check is just belt and braces" # * "the protected-alias screen is redundant, the allow-list covers it" # Every one of those would be a total SSO outage waiting to happen. # # ANCHORS ARE AUTHORED AGAINST THE EXTRACTED SCRIPT, not against the YAML manifest. # The sibling mutations.py dedents by four because its anchors are copied out of the # ConfigMap block scalar; these are copied out of the script as `python3 -` sees it, # which is one less transformation to get wrong. check.sh passes the extracted file, # so what is proven falsifiable is byte-for-byte what runs on the cluster either way. # # python3 mutations.py # # Runtime is ~85s serial (one subprocess per mutation, each running the full selftest # against an in-process stub Keycloak; no network, no cluster). The selftests run in a # bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and # verdicts are independent of the pool size. # ============================================================================= from __future__ import annotations import os import re import subprocess import sys import tempfile from concurrent.futures import ThreadPoolExecutor from pathlib import Path # (name, the mistake being modelled, anchor, replacement) MUTATIONS: list[tuple[str, str, str, str]] = [ # ---------------------------------------------------------------- blue/green safety ("build INTO the currently bound flow", "'the suffix is ugly, just build under the declared alias' -- every " "create-execution and set-requirement call would then land on the flow users are " "logging in through, and a run dying part-way leaves a HALF-BUILT LOGIN PATH for " "every SSO app and kubectl. This is the single regression the whole design exists " "to make impossible", " def mapped(alias):\n" " return \"%s-%s\" % (alias, suffix)", " def mapped(alias):\n return alias"), ("the protected-subtree screen is never called", "'the allow-list already covers it' -- it does not: the allow-list's write entries " "are alias-parameterised, so only this screen can say 'but not the flow that is " "bound'. Without it the reconciler may write anywhere in the authentication tree", " _screen_write(method, path, body if isinstance(body, dict) else None)", " pass"), ("the per-run reset makes the protected sets EMPTY, not UNDETERMINED", "reset them to frozenset() so the code reads more simply -- it silently removes " "the fail-closed property, so a build that ran before the live binding had been " "read would be PERMITTED, and could write straight into the live flow. (The " "module-level initial value is deliberately NOT the anchor here: run() resets " "these at the start of every run, so the initialiser is unreachable and mutating " "it would prove nothing.)", " global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS, _BUILD_EXECUTION_IDS\n" " _PROTECTED_FLOW_ALIASES = None\n" " _PROTECTED_EXECUTION_IDS = None", " global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS, _BUILD_EXECUTION_IDS\n" " _PROTECTED_FLOW_ALIASES = frozenset()\n" " _PROTECTED_EXECUTION_IDS = frozenset()"), ("bind BEFORE verifying the build", "'bind it first, the readback is just belt and braces' -- it is not: the readback " "is what catches a build that was accepted but came out wrong, including the " "conditional-OTP config landing under the wrong key. Binding first makes that the " "live login path before anyone notices", " verify = compare_shape(final_shape, expected, base, realm, token, configs,\n" " \"the newly built flow %r\" % target)", " verify = []"), ("a failed build is reported as SUCCESS", "swallow the build error and return a converged result -- the realm is then left " "on the old flow (which blue/green makes safe) but the run says it converged, so " "nobody ever learns the passkey flow was never built. That is the one-shot " "import's fail-open, reintroduced. NOTE: simply removing the early return here " "is NOT a regression -- verify-before-bind catches a half-built flow anyway, which " "is defence in depth working as intended -- so the mutation targets the reporting", " problems.append(\n" " \"flow: building %r FAILED (%s). Nothing was bound, so login is \"\n" " \"UNAFFECTED -- the old flow is still live. The partial flow is left in \"\n" " \"place, unbound and harmless\" % (candidate, e))\n" " return 0, problems", " return 1, []"), ("an existing candidate flow is reused WITHOUT verifying it", "'if the alias exists we already built it' -- a half-built leftover from a crashed " "run lives under exactly that alias, so this binds a partially constructed login " "path", " if cand_shape is not None and not compare_shape(\n" " cand_shape, expected, base, realm, token, configs,\n" " \"the existing flow %r\" % candidate):", " if cand_shape is not None:"), # ------------------------------------- 2026-10-29 review findings, each made falsifiable ("sub-flows identified by `alias` again", "read `alias` off a sub-flow row, as the first version did -- Keycloak 26.0.6's " "recurseExecutions sets only displayName (and flowId) on a sub-flow row, so every " "sub-flow resolves to nothing and the protected subtree is the top-level flow alone: " "a write into browser-mfa-forms would NOT be refused", " name = ex.get(\"displayName\")\n" " return name if isinstance(name, str) and name else None", " name = ex.get(\"alias\")\n" " return name if isinstance(name, str) and name else None"), ("an unnameable sub-flow is tolerated", "carry on when a sub-flow row has no displayName -- the protected subtree is then " "SMALLER than the live tree, which is the fail-open the guard exists to prevent", " if name is None:\n" " return None, None", " if name is None:\n" " name = \"\""), ("the requirement PUT is screened on its path only", "trust the alias in PUT /flows//executions -- updateExecutions resolves " "body.id realm-wide and never checks it belongs to , so a candidate alias in " "the path can carry a BOUND execution's id in the body", " targets.append((\"execution\", (body or {}).get(\"id\")))", " pass"), ("execution-addressed writes are not held to the build tree", "'outside the bound subtree is enough' -- it is not: every OTHER flow in the realm is " "outside it too, so a stray id would edit an unrelated flow", " if what == \"execution\" and (_BUILD_EXECUTION_IDS is None\n" " or target not in _BUILD_EXECUTION_IDS):", " if False:"), ("the requirement PUT may carry any key", "let the body carry flowId / displayName -- the same handler then RENAMES whatever " "flow flowId names, bound or not", " if extra:\n" " raise Refused(", " if False:\n" " raise Refused("), ("git's bind switch is ignored", "bind whenever verification passes, as the first version did -- the first live use of " "an untested login flow becomes a schedule tick, and an operator's rollback is " "undone within 15 minutes, forever", " bind = git_bind and not no_bind", " bind = not no_bind"), ("bind=false still binds inside the flow pass", "keep the flag but forget to honour it where the bind happens", " if not bind:\n" " # THE BIND SWITCH IS OFF.", " if False:\n" " # THE BIND SWITCH IS OFF."), ("an absent bind switch means bind", "default to true when the key is missing -- a ConfigMap predating the switch, or a " "deleted line, would silently put an untested flow live", " except FileNotFoundError:\n" " return False", " except FileNotFoundError:\n" " return True"), ("an invalid bind switch is guessed at", "read anything that is not \"true\" as false -- a typo then passes silently instead of " "stopping the run, and the operator believes the flow is live", " if raw == \"false\":\n" " return False\n" " raise ValueError(", " return False\n" " raise ValueError("), ("bind=false rebuilds beside a hand-edited bound flow", "treat the hand-edited bound candidate as a 'half-built leftover' -- with the switch " "off this builds an -r2 next to the operator's hotfix on every tick", " if candidate in have and candidate == bound and not bind:", " if False:"), # ------------------------------------------------------------------- idempotence ("sub-flow aliases are compared as names", "compare the sub-flow alias like any other field -- the flows this program builds " "carry a hash suffix on every alias, so a built flow could NEVER be recognised as " "converged: it would be rebuilt and the login path reflipped every fifteen " "minutes, forever", " out.append((level, ex[\"kind\"], \"\" if is_flow else ex[\"name\"],\n" " ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))", " out.append((level, ex[\"kind\"], ex[\"name\"],\n" " ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))"), ("the converged short-circuit is removed", "drop the 'bound flow already matches git' early return -- every run rebuilds and " "reflips, which is a permanent write loop against the IdP and churns the realm's " "login path on a schedule", " if not drift:\n" " log(\"flow ok (the bound flow %r matches git: %d execution(s), no \"\n" " \"write)\" % (bound, len(expected)))\n" " return 0, problems", " if False:\n return 0, problems"), ("the execution priority is dropped from the requirement write", "omit `priority` from the body -- Keycloak's updateExecutions deserialises a " "missing int as 0 and compares it against the stored priority, so this RESHUFFLES " "the execution order while only a requirement change was intended. The passkey " "branch would end up in the wrong place in the tree", " body={\"id\": ex.get(\"id\"), \"requirement\": want_req,\n" " \"priority\": ex.get(\"priority\") or 0})", " body={\"id\": ex.get(\"id\"), \"requirement\": want_req})"), ("the alias budget is unbounded", "loop forever looking for a free alias -- a persistently failing build then creates " "unbounded flow sprawl in the realm, and nothing here can delete any of it", " for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):", " for attempt in range(1, 10000):"), ("fail-open when the bound flow cannot be read", "carry on when the live flow's executions are unreadable -- the program then cannot " "know which flows are live, so it cannot prove a build would miss them, and it " "would build with the protected sets still undetermined", " return 0, [\n" " \"flow: the realm binds browserFlow to %r but its executions could not \"\n" " \"be read, so this program cannot establish which flows are live. Flow \"\n" " \"writes stay FAIL-CLOSED for this run -- nothing was built or bound\"\n" " % bound]", " _set_protected(set(), set())\n bound_shape, bound_raw = None, None"), # ------------------------------------------------------- the conditional-OTP regression ("authenticator config contents no longer compared", "check only that a config is BOUND, not what it says -- this is THE conditional-OTP " "regression: the original bug wrote the role under " "`condition.evaluated.user.role` instead of `condUserRole`, so the role resolved " "to null, the REQUIRED conditional step threw 'Invalid role name submitted: null' " "and every platform-admin got a 403 from every SSO-gated app", " if str(have_cfg.get(key)) != str(want_cfg[key]):", " if False:"), ("undeclared config keys are ignored", "stop reporting keys the realm holds and git does not -- the stale " "`condition.evaluated.user.role` key is the signature of the original null-role " "bug, and a flow carrying both keys would look correct", " extra = sorted(k for k in have_cfg if k not in want_cfg)", " extra = []"), ("flow structure divergence detected but not reported", "leave the per-step comparison out -- the realm's login flow could be the old " "pre-passkey shape forever and every run would report the realm converged, which " "is the exact fail-open of the one-shot import this program replaces", " if observed[idx][:4] != expected[idx][:4]:", " if False:"), ("the browserFlow BINDING is no longer compared", "stop noticing that a correct flow is not the one bound -- a perfect, fully-built " "passkey flow that nothing logs in through changes nothing at all", " if bound_shape is not None:\n" " drift = compare_shape(bound_shape, expected, base, realm, token, configs,", " if False:\n" " drift = compare_shape(bound_shape, expected, base, realm, token, configs,"), # ------------------------------------------------------------- realm / policy writes ("partial policy PUT instead of read-modify-write", "send only the drifted policy keys -- Keycloak 26.0.6 rebuilds the WHOLE " "passwordless policy from the representation and substitutes its own DEFAULT for " "every key the body omits, so a partial PUT silently resets everything it did not " "mention while returning 204", " body = dict(realm_rep)\n" " for key in drift:\n" " body[key] = wanted[key]", " body = {RP_ENTITY_NAME: realm_rep.get(RP_ENTITY_NAME),\n" " BROWSER_FLOW: realm_rep.get(BROWSER_FLOW)}\n" " for key in drift:\n" " body[key] = wanted[key]"), ("the policy pass rebinds browserFlow as a side effect", "'while we are here, let the policy write fix the flow binding too' -- the realm " "PUT does read rep.getBrowserFlow(), so this is genuinely in reach, and it would " "rebind the login path to an UNVERIFIED flow from a pass whose stated surface is " "the passkey policy", " if body.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW):\n" " raise RuntimeError(", " body[BROWSER_FLOW] = \"browser\"\n" " if False:\n" " raise RuntimeError("), ("the RpEntityName gate removed from desired-state validation", "accept a desired state with an empty RpEntityName -- Keycloak 26.0.6 applies the " "passwordless policy ONLY when that field is non-empty, so every realm PUT would " "be accepted with 204 and change nothing: reported success, zero convergence", " if not policy.get(RP_ENTITY_NAME):\n" " raise ValueError(", " if False:\n raise ValueError("), ("the RpEntityName gate removed from the write path", "drop the second, defence-in-depth guard -- it is unreachable while the " "desired-state validation holds, which is exactly why deleting it would otherwise " "be an invisible regression", " if not body.get(RP_ENTITY_NAME):\n" " raise RuntimeError(", " if False:\n raise RuntimeError("), ("no readback verification on the policy write", "trust the 204 -- a server that accepts the PUT and does not apply it is reported " "as REPAIRED, which is 'reports success while the realm never converged' again, " "and it is precisely what the RpEntityName gate does when it is missed", " still = policy_drift(after, wanted)", " still = {}"), ("AAGUID comparison made order-sensitive", "compare the AAGUID allow-list in the order Keycloak returns it -- an allow-list " "is a set membership test, so an order-sensitive comparison rewrites a CONVERGED " "realm on every run forever while every run still reports success", " if stored != sorted(declared_value):\n" " drift[key] = (stored, sorted(declared_value))", " if stored != declared_value:\n" " drift[key] = (stored, declared_value)"), ("signature-algorithm order treated as insignificant", "sort the signature algorithms before comparing -- it is a PREFERENCE LIST handed " "to the authenticator, so ES256-then-RS256 is not the same declaration as " "RS256-then-ES256, and a reorder in the realm would never be corrected", " if stored != list(declared_value):", " if sorted(stored) != sorted(declared_value):"), ("unmanaged policy keys forced to a value", "iterate every writable key instead of only the declared ones -- a " "webAuthnPolicyPasswordless field the realm holds and git does not declare would " "be overwritten, turning 'not declared' into 'must equal my default'", " for key in sorted(wanted):\n declared_value = wanted[key]", " for key in POLICY_FIELDS:\n declared_value = wanted.get(key, \"\")"), # ------------------------------------------------------------------ required actions ("required-action PUT sends only the two drifted fields", "a minimal body looks safer -- but updateRequiredAction REPLACES the provider from " "the representation, so name, priority and config are all blanked: the enrolment " "action keeps working but loses its priority and display name", " body = dict(rep)\n for key in drift:\n body[key] = wanted[key]", " body = {}\n for key in drift:\n body[key] = wanted[key]"), ("the guarded default required actions are no longer audited", "stop checking that CONFIGURE_TOTP / UPDATE_PASSWORD / delete_credential still " "exist -- their disappearance is the signature of a requiredActions block in the " "realm import, which 26.0.6 treats as REPLACE, and it is a silent permanent " "lock-out of every platform-admin", " missing = [a for a in GUARDED_ACTIONS if a not in aliases]", " missing = []"), # ----------------------------------------------------------------- inputs / coupling ("the three passes are coupled", "abort the run at the first failing pass -- a policy write that fails for an " "unrelated reason would then suppress the browser-flow pass, which is the most " "safety-critical thing here", " problems.extend(found)\n if found:\n rc = 1", " problems.extend(found)\n if found:\n rc = 1\n" " break"), ("policy-key validation removed from the desired state", "accept any realm key from the ConfigMap -- an arbitrary realm attribute becomes " "writable through a file edit, so a hand-edited ConfigMap could set sslRequired, " "bruteForceProtected or any other top-level realm field on the live IdP", " if key not in POLICY_FIELDS:", " if False:"), ("fail-open on an unreadable desired state", "treat a missing ConfigMap as 'nothing declared' -- the run reports success while " "reconciling nothing, which is precisely the fail-open of the realm import this " "whole component exists to replace", " if not isinstance(raw, dict) or not raw:\n" " raise ValueError(\"%s does not hold a non-empty JSON object\" % path)", " if not isinstance(raw, dict) or not raw:\n return {}"), ("an undeclared authenticatorConfig reference is tolerated", "stop validating that every configAlias a flow references is actually declared -- " "the build then fails at the config-binding step, AFTER flows and executions have " "already been created, leaving leftovers on every run", " if ca and ca not in configs:\n" " raise ValueError(", " if False:\n raise ValueError("), ("the flow-cycle guard is removed", "drop the `seen` check -- a cyclic flow declaration then recurses until the " "interpreter dies, in a CronJob that retries", " if alias in seen:\n" " raise ValueError(\"flow %r is cyclic in the desired state\" % alias)", " if False:\n" " raise ValueError(\"flow %r is cyclic in the desired state\" % alias)"), ] def mutation_jobs() -> int: """Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs). conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself running several checks in parallel, so the nested pools do not oversubscribe the host. MUTATION_JOBS=1 reproduces the old serial loop exactly. """ raw = os.environ.get("MUTATION_JOBS", "").strip() if raw: try: n = int(raw) except ValueError: n = 0 if n < 1: raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw) return n try: cpus = len(os.sched_getaffinity(0)) except (AttributeError, OSError): cpus = os.cpu_count() or 1 return max(1, min(8, cpus)) def run_selftest(mutated_src: str) -> subprocess.CompletedProcess: with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh: fh.write(mutated_src) path = fh.name try: return subprocess.run([sys.executable, path, "--selftest"], capture_output=True, text=True, timeout=300, stdin=subprocess.DEVNULL) finally: Path(path).unlink(missing_ok=True) def main() -> int: if len(sys.argv) != 2: print("usage: mutations.py ", file=sys.stderr) return 2 src = Path(sys.argv[1]).read_text() try: jobs = mutation_jobs() except ValueError as e: print("mutations.py: %s" % e, file=sys.stderr) return 2 # Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the # serial loop did, and each mutation's source is built here. None = anchor failure. planned: list[tuple[str, str, str | None, str | None]] = [] for name, why, old, new in MUTATIONS: # An anchor that no longer matches is a FAILURE, not a skip: it means the code # moved and this mutation silently stopped proving anything. n = src.count(old) if n != 1: planned.append((name, why, "%s: anchor matches %d times (expected exactly 1) -- the " "code moved and this mutation no longer proves anything. " "Re-anchor it against the current source." % (name, n), None)) continue planned.append((name, why, None, src.replace(old, new, 1))) # Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own # mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are # consumed in MUTATIONS order, so every line below is printed in the same order, with the # same verdict, as the serial loop produced -- whichever child finishes first. problems: list[str] = [] killed = 0 pool = ThreadPoolExecutor(max_workers=jobs) try: futures = [pool.submit(run_selftest, mutated) if mutated is not None else None for (_n, _w, _a, mutated) in planned] for (name, why, anchor_problem, _m), fut in zip(planned, futures): if anchor_problem is not None: problems.append(anchor_problem) continue proc = fut.result() if proc.returncode == 0: problems.append("%s SURVIVED -- --selftest still passed with this regression " "applied, so nothing in the suite actually checks it. The " "mistake it models: %s" % (name, why)) continue killed += 1 out = proc.stdout + proc.stderr m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out) how = ("%s assertion(s) caught it" % m.group(1) if m else "the selftest aborted (caught, less cleanly)") print(" killed %-54s %s" % (name, how)) finally: # A timeout (or any other exception) still propagates and fails the check, as it did # serially; queued mutations are cancelled rather than run for nothing. pool.shutdown(wait=True, cancel_futures=True) print() if problems: print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven" % (len(problems), len(MUTATIONS)), file=sys.stderr) for p in problems: print(" FAIL %s" % p, file=sys.stderr) return 1 print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest " "(the suite is falsifiable, not decorative)" % killed) return 0 if __name__ == "__main__": raise SystemExit(main())