home-network-k3s-impl/conformance/tests/l0-static/keycloak-passkey-reconciler/mutations.py

#!/usr/bin/env python3
# =============================================================================
# mutations.py -- prove the passkey reconciler's --selftest IS FALSIFIABLE.
#
# WHY THIS EXISTS
# ---------------
# A selftest that passes proves nothing on its own: a selftest asserting `True ==
# True` also passes. This reconciler exists because a one-shot realm import
# REPORTED SUCCESS while the realm was never converged (Done=True / Started=False
# with the whole passkey block inert, measured on prod 2026-10-18), and the same
# trap applies one level up -- a green test suite that cannot fail is the same lie
# in a different place.
#
# It matters more here than for the sibling client reconciler, because this program
# CONSTRUCTS AUTHENTICATION FLOWS and then REBINDS THE REALM'S LOGIN PATH. An
# untested safety property on that surface is not a documentation gap, it is an
# outage waiting for a schedule tick.
#
# So this injects a series of PLAUSIBLE regressions into the shipped script, one at
# a time, and requires that --selftest FAILS for every one. A mutation that
# SURVIVES means the suite does not actually check that property, and the check
# fails, naming it.
#
# THE FIRST SIX ARE THE BLUE/GREEN SAFETY ARGUMENT ITSELF. The design builds a new
# flow under an unused alias, verifies it, and flips `browserFlow` in one atomic
# write -- never mutating the flow users are logging in through. Each of those
# properties is enforced in code; these mutations are the proof that the enforcement
# is load-bearing rather than commentary. Several model the obvious "simplification"
# someone will propose:
#   * "just build it under the declared alias, the suffix is ugly"
#   * "bind it first, then check -- the check is just belt and braces"
#   * "the protected-alias screen is redundant, the allow-list covers it"
# Every one of those would be a total SSO outage waiting to happen.
#
# ANCHORS ARE AUTHORED AGAINST THE EXTRACTED SCRIPT, not against the YAML manifest.
# The sibling mutations.py dedents by four because its anchors are copied out of the
# ConfigMap block scalar; these are copied out of the script as `python3 -` sees it,
# which is one less transformation to get wrong. check.sh passes the extracted file,
# so what is proven falsifiable is byte-for-byte what runs on the cluster either way.
#
#   python3 mutations.py <path-to-extracted-reconciler.py>
#
# Runtime is ~60s (one subprocess per mutation, each running the full selftest
# against an in-process stub Keycloak; no network, no cluster).
# =============================================================================
from __future__ import annotations

import re
import subprocess
import sys
import tempfile
from pathlib import Path

# (name, the mistake being modelled, anchor, replacement)
MUTATIONS: list[tuple[str, str, str, str]] = [
    # ---------------------------------------------------------------- blue/green safety
    ("build INTO the currently bound flow",
     "'the suffix is ugly, just build under the declared alias' -- every "
     "create-execution and set-requirement call would then land on the flow users are "
     "logging in through, and a run dying part-way leaves a HALF-BUILT LOGIN PATH for "
     "every SSO app and kubectl. This is the single regression the whole design exists "
     "to make impossible",
     "    def mapped(alias):\n"
     "        return \"%s-%s\" % (alias, suffix)",
     "    def mapped(alias):\n        return alias"),

    ("the protected-subtree screen is never called",
     "'the allow-list already covers it' -- it does not: the allow-list's write entries "
     "are alias-parameterised, so only this screen can say 'but not the flow that is "
     "bound'. Without it the reconciler may write anywhere in the authentication tree",
     "    _screen_write(method, path, body if isinstance(body, dict) else None)",
     "    pass"),

    ("the per-run reset makes the protected sets EMPTY, not UNDETERMINED",
     "reset them to frozenset() so the code reads more simply -- it silently removes "
     "the fail-closed property, so a build that ran before the live binding had been "
     "read would be PERMITTED, and could write straight into the live flow. (The "
     "module-level initial value is deliberately NOT the anchor here: run() resets "
     "these at the start of every run, so the initialiser is unreachable and mutating "
     "it would prove nothing.)",
     "    global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS\n"
     "    _PROTECTED_FLOW_ALIASES = None\n"
     "    _PROTECTED_EXECUTION_IDS = None",
     "    global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS\n"
     "    _PROTECTED_FLOW_ALIASES = frozenset()\n"
     "    _PROTECTED_EXECUTION_IDS = frozenset()"),

    ("bind BEFORE verifying the build",
     "'bind it first, the readback is just belt and braces' -- it is not: the readback "
     "is what catches a build that was accepted but came out wrong, including the "
     "conditional-OTP config landing under the wrong key. Binding first makes that the "
     "live login path before anyone notices",
     "    verify = compare_shape(final_shape, expected, base, realm, token, configs,\n"
     "                           \"the newly built flow %r\" % target)",
     "    verify = []"),

    ("a failed build is reported as SUCCESS",
     "swallow the build error and return a converged result -- the realm is then left "
     "on the old flow (which blue/green makes safe) but the run says it converged, so "
     "nobody ever learns the passkey flow was never built. That is the one-shot "
     "import's fail-open, reintroduced. NOTE: simply removing the early return here "
     "is NOT a regression -- verify-before-bind catches a half-built flow anyway, which "
     "is defence in depth working as intended -- so the mutation targets the reporting",
     "            problems.append(\n"
     "                \"flow: building %r FAILED (%s). Nothing was bound, so login is \"\n"
     "                \"UNAFFECTED -- the old flow is still live. The partial flow is left in \"\n"
     "                \"place, unbound and harmless\" % (candidate, e))\n"
     "            return 0, problems",
     "            return 1, []"),

    ("an existing candidate flow is reused WITHOUT verifying it",
     "'if the alias exists we already built it' -- a half-built leftover from a crashed "
     "run lives under exactly that alias, so this binds a partially constructed login "
     "path",
     "            if cand_shape is not None and not compare_shape(\n"
     "                    cand_shape, expected, base, realm, token, configs,\n"
     "                    \"the existing flow %r\" % candidate):",
     "            if cand_shape is not None:"),

    # ------------------------------------------------------------------- idempotence
    ("sub-flow aliases are compared as names",
     "compare the sub-flow alias like any other field -- the flows this program builds "
     "carry a hash suffix on every alias, so a built flow could NEVER be recognised as "
     "converged: it would be rebuilt and the login path reflipped every fifteen "
     "minutes, forever",
     "            out.append((level, ex[\"kind\"], \"\" if is_flow else ex[\"name\"],\n"
     "                        ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))",
     "            out.append((level, ex[\"kind\"], ex[\"name\"],\n"
     "                        ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))"),

    ("the converged short-circuit is removed",
     "drop the 'bound flow already matches git' early return -- every run rebuilds and "
     "reflips, which is a permanent write loop against the IdP and churns the realm's "
     "login path on a schedule",
     "        if not drift:\n"
     "            log(\"flow     ok (the bound flow %r matches git: %d execution(s), no \"\n"
     "                \"write)\" % (bound, len(expected)))\n"
     "            return 0, problems",
     "        if False:\n            return 0, problems"),

    ("the execution priority is dropped from the requirement write",
     "omit `priority` from the body -- Keycloak's updateExecutions deserialises a "
     "missing int as 0 and compares it against the stored priority, so this RESHUFFLES "
     "the execution order while only a requirement change was intended. The passkey "
     "branch would end up in the wrong place in the tree",
     "            body={\"id\": ex.get(\"id\"), \"requirement\": want_req,\n"
     "                  \"priority\": ex.get(\"priority\") or 0})",
     "            body={\"id\": ex.get(\"id\"), \"requirement\": want_req})"),

    ("the alias budget is unbounded",
     "loop forever looking for a free alias -- a persistently failing build then creates "
     "unbounded flow sprawl in the realm, and nothing here can delete any of it",
     "    for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):",
     "    for attempt in range(1, 10000):"),

    ("fail-open when the bound flow cannot be read",
     "carry on when the live flow's executions are unreadable -- the program then cannot "
     "know which flows are live, so it cannot prove a build would miss them, and it "
     "would build with the protected sets still undetermined",
     "            return 0, [\n"
     "                \"flow: the realm binds browserFlow to %r but its executions could not \"\n"
     "                \"be read, so this program cannot establish which flows are live. Flow \"\n"
     "                \"writes stay FAIL-CLOSED for this run -- nothing was built or bound\"\n"
     "                % bound]",
     "            _set_protected(set(), set())\n            bound_shape, bound_raw = None, None"),

    # ------------------------------------------------------- the conditional-OTP regression
    ("authenticator config contents no longer compared",
     "check only that a config is BOUND, not what it says -- this is THE conditional-OTP "
     "regression: the original bug wrote the role under "
     "`condition.evaluated.user.role` instead of `condUserRole`, so the role resolved "
     "to null, the REQUIRED conditional step threw 'Invalid role name submitted: null' "
     "and every platform-admin got a 403 from every SSO-gated app",
     "            if str(have_cfg.get(key)) != str(want_cfg[key]):",
     "            if False:"),

    ("undeclared config keys are ignored",
     "stop reporting keys the realm holds and git does not -- the stale "
     "`condition.evaluated.user.role` key is the signature of the original null-role "
     "bug, and a flow carrying both keys would look correct",
     "        extra = sorted(k for k in have_cfg if k not in want_cfg)",
     "        extra = []"),

    ("flow structure divergence detected but not reported",
     "leave the per-step comparison out -- the realm's login flow could be the old "
     "pre-passkey shape forever and every run would report the realm converged, which "
     "is the exact fail-open of the one-shot import this program replaces",
     "        if observed[idx][:4] != expected[idx][:4]:",
     "        if False:"),

    ("the browserFlow BINDING is no longer compared",
     "stop noticing that a correct flow is not the one bound -- a perfect, fully-built "
     "passkey flow that nothing logs in through changes nothing at all",
     "    if bound_shape is not None:\n"
     "        drift = compare_shape(bound_shape, expected, base, realm, token, configs,",
     "    if False:\n"
     "        drift = compare_shape(bound_shape, expected, base, realm, token, configs,"),

    # ------------------------------------------------------------- realm / policy writes
    ("partial policy PUT instead of read-modify-write",
     "send only the drifted policy keys -- Keycloak 26.0.6 rebuilds the WHOLE "
     "passwordless policy from the representation and substitutes its own DEFAULT for "
     "every key the body omits, so a partial PUT silently resets everything it did not "
     "mention while returning 204",
     "    body = dict(realm_rep)\n"
     "    for key in drift:\n"
     "        body[key] = wanted[key]",
     "    body = {RP_ENTITY_NAME: realm_rep.get(RP_ENTITY_NAME),\n"
     "            BROWSER_FLOW: realm_rep.get(BROWSER_FLOW)}\n"
     "    for key in drift:\n"
     "        body[key] = wanted[key]"),

    ("the policy pass rebinds browserFlow as a side effect",
     "'while we are here, let the policy write fix the flow binding too' -- the realm "
     "PUT does read rep.getBrowserFlow(), so this is genuinely in reach, and it would "
     "rebind the login path to an UNVERIFIED flow from a pass whose stated surface is "
     "the passkey policy",
     "    if body.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW):\n"
     "        raise RuntimeError(",
     "    body[BROWSER_FLOW] = \"browser\"\n"
     "    if False:\n"
     "        raise RuntimeError("),

    ("the RpEntityName gate removed from desired-state validation",
     "accept a desired state with an empty RpEntityName -- Keycloak 26.0.6 applies the "
     "passwordless policy ONLY when that field is non-empty, so every realm PUT would "
     "be accepted with 204 and change nothing: reported success, zero convergence",
     "    if not policy.get(RP_ENTITY_NAME):\n"
     "        raise ValueError(",
     "    if False:\n        raise ValueError("),

    ("the RpEntityName gate removed from the write path",
     "drop the second, defence-in-depth guard -- it is unreachable while the "
     "desired-state validation holds, which is exactly why deleting it would otherwise "
     "be an invisible regression",
     "    if not body.get(RP_ENTITY_NAME):\n"
     "        raise RuntimeError(",
     "    if False:\n        raise RuntimeError("),

    ("no readback verification on the policy write",
     "trust the 204 -- a server that accepts the PUT and does not apply it is reported "
     "as REPAIRED, which is 'reports success while the realm never converged' again, "
     "and it is precisely what the RpEntityName gate does when it is missed",
     "    still = policy_drift(after, wanted)",
     "    still = {}"),

    ("AAGUID comparison made order-sensitive",
     "compare the AAGUID allow-list in the order Keycloak returns it -- an allow-list "
     "is a set membership test, so an order-sensitive comparison rewrites a CONVERGED "
     "realm on every run forever while every run still reports success",
     "            if stored != sorted(declared_value):\n"
     "                drift[key] = (stored, sorted(declared_value))",
     "            if stored != declared_value:\n"
     "                drift[key] = (stored, declared_value)"),

    ("signature-algorithm order treated as insignificant",
     "sort the signature algorithms before comparing -- it is a PREFERENCE LIST handed "
     "to the authenticator, so ES256-then-RS256 is not the same declaration as "
     "RS256-then-ES256, and a reorder in the realm would never be corrected",
     "            if stored != list(declared_value):",
     "            if sorted(stored) != sorted(declared_value):"),

    ("unmanaged policy keys forced to a value",
     "iterate every writable key instead of only the declared ones -- a "
     "webAuthnPolicyPasswordless field the realm holds and git does not declare would "
     "be overwritten, turning 'not declared' into 'must equal my default'",
     "    for key in sorted(wanted):\n        declared_value = wanted[key]",
     "    for key in POLICY_FIELDS:\n        declared_value = wanted.get(key, \"\")"),

    # ------------------------------------------------------------------ required actions
    ("required-action PUT sends only the two drifted fields",
     "a minimal body looks safer -- but updateRequiredAction REPLACES the provider from "
     "the representation, so name, priority and config are all blanked: the enrolment "
     "action keeps working but loses its priority and display name",
     "    body = dict(rep)\n    for key in drift:\n        body[key] = wanted[key]",
     "    body = {}\n    for key in drift:\n        body[key] = wanted[key]"),

    ("the guarded default required actions are no longer audited",
     "stop checking that CONFIGURE_TOTP / UPDATE_PASSWORD / delete_credential still "
     "exist -- their disappearance is the signature of a requiredActions block in the "
     "realm import, which 26.0.6 treats as REPLACE, and it is a silent permanent "
     "lock-out of every platform-admin",
     "    missing = [a for a in GUARDED_ACTIONS if a not in aliases]",
     "    missing = []"),

    # ----------------------------------------------------------------- inputs / coupling
    ("the three passes are coupled",
     "abort the run at the first failing pass -- a policy write that fails for an "
     "unrelated reason would then suppress the browser-flow pass, which is the most "
     "safety-critical thing here",
     "        problems.extend(found)\n        if found:\n            rc = 1",
     "        problems.extend(found)\n        if found:\n            rc = 1\n"
     "            break"),

    ("policy-key validation removed from the desired state",
     "accept any realm key from the ConfigMap -- an arbitrary realm attribute becomes "
     "writable through a file edit, so a hand-edited ConfigMap could set sslRequired, "
     "bruteForceProtected or any other top-level realm field on the live IdP",
     "        if key not in POLICY_FIELDS:",
     "        if False:"),

    ("fail-open on an unreadable desired state",
     "treat a missing ConfigMap as 'nothing declared' -- the run reports success while "
     "reconciling nothing, which is precisely the fail-open of the realm import this "
     "whole component exists to replace",
     "    if not isinstance(raw, dict) or not raw:\n"
     "        raise ValueError(\"%s does not hold a non-empty JSON object\" % path)",
     "    if not isinstance(raw, dict) or not raw:\n        return {}"),

    ("an undeclared authenticatorConfig reference is tolerated",
     "stop validating that every configAlias a flow references is actually declared -- "
     "the build then fails at the config-binding step, AFTER flows and executions have "
     "already been created, leaving leftovers on every run",
     "            if ca and ca not in configs:\n"
     "                raise ValueError(",
     "            if False:\n                raise ValueError("),

    ("the flow-cycle guard is removed",
     "drop the `seen` check -- a cyclic flow declaration then recurses until the "
     "interpreter dies, in a CronJob that retries",
     "        if alias in seen:\n"
     "            raise ValueError(\"flow %r is cyclic in the desired state\" % alias)",
     "        if False:\n"
     "            raise ValueError(\"flow %r is cyclic in the desired state\" % alias)"),
]


def main() -> int:
    if len(sys.argv) != 2:
        print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
        return 2
    src = Path(sys.argv[1]).read_text()

    problems: list[str] = []
    killed = 0
    for name, why, old, new in MUTATIONS:
        # An anchor that no longer matches is a FAILURE, not a skip: it means the code
        # moved and this mutation silently stopped proving anything.
        n = src.count(old)
        if n != 1:
            problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
                            "code moved and this mutation no longer proves anything. "
                            "Re-anchor it against the current source." % (name, n))
            continue
        with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
            fh.write(src.replace(old, new, 1))
            path = fh.name
        try:
            proc = subprocess.run([sys.executable, path, "--selftest"],
                                  capture_output=True, text=True, timeout=300)
        finally:
            Path(path).unlink(missing_ok=True)
        if proc.returncode == 0:
            problems.append("%s SURVIVED -- --selftest still passed with this regression "
                            "applied, so nothing in the suite actually checks it. The "
                            "mistake it models: %s" % (name, why))
            continue
        killed += 1
        out = proc.stdout + proc.stderr
        m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
        how = ("%s assertion(s) caught it" % m.group(1) if m
               else "the selftest aborted (caught, less cleanly)")
        print("  killed  %-54s %s" % (name, how))

    print()
    if problems:
        print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven"
              % (len(problems), len(MUTATIONS)), file=sys.stderr)
        for p in problems:
            print("  FAIL %s" % p, file=sys.stderr)
        return 1
    print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest "
          "(the suite is falsifiable, not decorative)" % killed)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())