jakman85andClaude Opus 5.5
a775082ae1
- conformance/lib/run_l1_offline.py: checks run in a thread pool (--jobs / CONF_OFFLINE_JOBS,
default min(8, usable CPUs)); results consumed in submission order, so log lines, JUnit,
step summary and the 0/1/3 aggregation are unchanged. Exports a smaller MUTATION_JOBS to
the children when parallel, so nested pools do not oversubscribe.
- keycloak-{passkey,client-secret}-reconciler/mutations.py: anchors checked first (serial),
then the --selftest subprocesses run in a pool (MUTATION_JOBS, default min(8, CPUs));
verdicts reported in MUTATIONS order. Output byte-identical to the serial loop.
- ci/lint.sh: yamllint over LINT_JOBS contiguous slices concurrently, per-slice output
buffered and printed in order; stage rc = highest slice rc.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
517 lines
27 KiB
Python
517 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
# =============================================================================
|
|
# mutations.py -- prove the passkey reconciler's --selftest IS FALSIFIABLE.
|
|
#
|
|
# WHY THIS EXISTS
|
|
# ---------------
|
|
# A selftest that passes proves nothing on its own: a selftest asserting `True ==
|
|
# True` also passes. This reconciler exists because a one-shot realm import
|
|
# REPORTED SUCCESS while the realm was never converged (Done=True / Started=False
|
|
# with the whole passkey block inert, measured on prod 2026-10-18), and the same
|
|
# trap applies one level up -- a green test suite that cannot fail is the same lie
|
|
# in a different place.
|
|
#
|
|
# It matters more here than for the sibling client reconciler, because this program
|
|
# CONSTRUCTS AUTHENTICATION FLOWS and then REBINDS THE REALM'S LOGIN PATH. An
|
|
# untested safety property on that surface is not a documentation gap, it is an
|
|
# outage waiting for a schedule tick.
|
|
#
|
|
# So this injects a series of PLAUSIBLE regressions into the shipped script, one at
|
|
# a time, and requires that --selftest FAILS for every one. A mutation that
|
|
# SURVIVES means the suite does not actually check that property, and the check
|
|
# fails, naming it.
|
|
#
|
|
# THE FIRST SIX ARE THE BLUE/GREEN SAFETY ARGUMENT ITSELF. The design builds a new
|
|
# flow under an unused alias, verifies it, and flips `browserFlow` in one atomic
|
|
# write -- never mutating the flow users are logging in through. Each of those
|
|
# properties is enforced in code; these mutations are the proof that the enforcement
|
|
# is load-bearing rather than commentary. Several model the obvious "simplification"
|
|
# someone will propose:
|
|
# * "just build it under the declared alias, the suffix is ugly"
|
|
# * "bind it first, then check -- the check is just belt and braces"
|
|
# * "the protected-alias screen is redundant, the allow-list covers it"
|
|
# Every one of those would be a total SSO outage waiting to happen.
|
|
#
|
|
# ANCHORS ARE AUTHORED AGAINST THE EXTRACTED SCRIPT, not against the YAML manifest.
|
|
# The sibling mutations.py dedents by four because its anchors are copied out of the
|
|
# ConfigMap block scalar; these are copied out of the script as `python3 -` sees it,
|
|
# which is one less transformation to get wrong. check.sh passes the extracted file,
|
|
# so what is proven falsifiable is byte-for-byte what runs on the cluster either way.
|
|
#
|
|
# python3 mutations.py <path-to-extracted-reconciler.py>
|
|
#
|
|
# Runtime is ~85s serial (one subprocess per mutation, each running the full selftest
|
|
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
|
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
|
# verdicts are independent of the pool size.
|
|
# =============================================================================
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
# (name, the mistake being modelled, anchor, replacement)
|
|
MUTATIONS: list[tuple[str, str, str, str]] = [
|
|
# ---------------------------------------------------------------- blue/green safety
|
|
("build INTO the currently bound flow",
|
|
"'the suffix is ugly, just build under the declared alias' -- every "
|
|
"create-execution and set-requirement call would then land on the flow users are "
|
|
"logging in through, and a run dying part-way leaves a HALF-BUILT LOGIN PATH for "
|
|
"every SSO app and kubectl. This is the single regression the whole design exists "
|
|
"to make impossible",
|
|
" def mapped(alias):\n"
|
|
" return \"%s-%s\" % (alias, suffix)",
|
|
" def mapped(alias):\n return alias"),
|
|
|
|
("the protected-subtree screen is never called",
|
|
"'the allow-list already covers it' -- it does not: the allow-list's write entries "
|
|
"are alias-parameterised, so only this screen can say 'but not the flow that is "
|
|
"bound'. Without it the reconciler may write anywhere in the authentication tree",
|
|
" _screen_write(method, path, body if isinstance(body, dict) else None)",
|
|
" pass"),
|
|
|
|
("the per-run reset makes the protected sets EMPTY, not UNDETERMINED",
|
|
"reset them to frozenset() so the code reads more simply -- it silently removes "
|
|
"the fail-closed property, so a build that ran before the live binding had been "
|
|
"read would be PERMITTED, and could write straight into the live flow. (The "
|
|
"module-level initial value is deliberately NOT the anchor here: run() resets "
|
|
"these at the start of every run, so the initialiser is unreachable and mutating "
|
|
"it would prove nothing.)",
|
|
" global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS, _BUILD_EXECUTION_IDS\n"
|
|
" _PROTECTED_FLOW_ALIASES = None\n"
|
|
" _PROTECTED_EXECUTION_IDS = None",
|
|
" global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS, _BUILD_EXECUTION_IDS\n"
|
|
" _PROTECTED_FLOW_ALIASES = frozenset()\n"
|
|
" _PROTECTED_EXECUTION_IDS = frozenset()"),
|
|
|
|
("bind BEFORE verifying the build",
|
|
"'bind it first, the readback is just belt and braces' -- it is not: the readback "
|
|
"is what catches a build that was accepted but came out wrong, including the "
|
|
"conditional-OTP config landing under the wrong key. Binding first makes that the "
|
|
"live login path before anyone notices",
|
|
" verify = compare_shape(final_shape, expected, base, realm, token, configs,\n"
|
|
" \"the newly built flow %r\" % target)",
|
|
" verify = []"),
|
|
|
|
("a failed build is reported as SUCCESS",
|
|
"swallow the build error and return a converged result -- the realm is then left "
|
|
"on the old flow (which blue/green makes safe) but the run says it converged, so "
|
|
"nobody ever learns the passkey flow was never built. That is the one-shot "
|
|
"import's fail-open, reintroduced. NOTE: simply removing the early return here "
|
|
"is NOT a regression -- verify-before-bind catches a half-built flow anyway, which "
|
|
"is defence in depth working as intended -- so the mutation targets the reporting",
|
|
" problems.append(\n"
|
|
" \"flow: building %r FAILED (%s). Nothing was bound, so login is \"\n"
|
|
" \"UNAFFECTED -- the old flow is still live. The partial flow is left in \"\n"
|
|
" \"place, unbound and harmless\" % (candidate, e))\n"
|
|
" return 0, problems",
|
|
" return 1, []"),
|
|
|
|
("an existing candidate flow is reused WITHOUT verifying it",
|
|
"'if the alias exists we already built it' -- a half-built leftover from a crashed "
|
|
"run lives under exactly that alias, so this binds a partially constructed login "
|
|
"path",
|
|
" if cand_shape is not None and not compare_shape(\n"
|
|
" cand_shape, expected, base, realm, token, configs,\n"
|
|
" \"the existing flow %r\" % candidate):",
|
|
" if cand_shape is not None:"),
|
|
|
|
# ------------------------------------- 2026-10-29 review findings, each made falsifiable
|
|
("sub-flows identified by `alias` again",
|
|
"read `alias` off a sub-flow row, as the first version did -- Keycloak 26.0.6's "
|
|
"recurseExecutions sets only displayName (and flowId) on a sub-flow row, so every "
|
|
"sub-flow resolves to nothing and the protected subtree is the top-level flow alone: "
|
|
"a write into browser-mfa-forms would NOT be refused",
|
|
" name = ex.get(\"displayName\")\n"
|
|
" return name if isinstance(name, str) and name else None",
|
|
" name = ex.get(\"alias\")\n"
|
|
" return name if isinstance(name, str) and name else None"),
|
|
|
|
("an unnameable sub-flow is tolerated",
|
|
"carry on when a sub-flow row has no displayName -- the protected subtree is then "
|
|
"SMALLER than the live tree, which is the fail-open the guard exists to prevent",
|
|
" if name is None:\n"
|
|
" return None, None",
|
|
" if name is None:\n"
|
|
" name = \"\""),
|
|
|
|
("the requirement PUT is screened on its path only",
|
|
"trust the alias in PUT /flows/<alias>/executions -- updateExecutions resolves "
|
|
"body.id realm-wide and never checks it belongs to <alias>, so a candidate alias in "
|
|
"the path can carry a BOUND execution's id in the body",
|
|
" targets.append((\"execution\", (body or {}).get(\"id\")))",
|
|
" pass"),
|
|
|
|
("execution-addressed writes are not held to the build tree",
|
|
"'outside the bound subtree is enough' -- it is not: every OTHER flow in the realm is "
|
|
"outside it too, so a stray id would edit an unrelated flow",
|
|
" if what == \"execution\" and (_BUILD_EXECUTION_IDS is None\n"
|
|
" or target not in _BUILD_EXECUTION_IDS):",
|
|
" if False:"),
|
|
|
|
("the requirement PUT may carry any key",
|
|
"let the body carry flowId / displayName -- the same handler then RENAMES whatever "
|
|
"flow flowId names, bound or not",
|
|
" if extra:\n"
|
|
" raise Refused(",
|
|
" if False:\n"
|
|
" raise Refused("),
|
|
|
|
("git's bind switch is ignored",
|
|
"bind whenever verification passes, as the first version did -- the first live use of "
|
|
"an untested login flow becomes a schedule tick, and an operator's rollback is "
|
|
"undone within 15 minutes, forever",
|
|
" bind = git_bind and not no_bind",
|
|
" bind = not no_bind"),
|
|
|
|
("bind=false still binds inside the flow pass",
|
|
"keep the flag but forget to honour it where the bind happens",
|
|
" if not bind:\n"
|
|
" # THE BIND SWITCH IS OFF.",
|
|
" if False:\n"
|
|
" # THE BIND SWITCH IS OFF."),
|
|
|
|
("an absent bind switch means bind",
|
|
"default to true when the key is missing -- a ConfigMap predating the switch, or a "
|
|
"deleted line, would silently put an untested flow live",
|
|
" except FileNotFoundError:\n"
|
|
" return False",
|
|
" except FileNotFoundError:\n"
|
|
" return True"),
|
|
|
|
("an invalid bind switch is guessed at",
|
|
"read anything that is not \"true\" as false -- a typo then passes silently instead of "
|
|
"stopping the run, and the operator believes the flow is live",
|
|
" if raw == \"false\":\n"
|
|
" return False\n"
|
|
" raise ValueError(",
|
|
" return False\n"
|
|
" raise ValueError("),
|
|
|
|
("bind=false rebuilds beside a hand-edited bound flow",
|
|
"treat the hand-edited bound candidate as a 'half-built leftover' -- with the switch "
|
|
"off this builds an -r2 next to the operator's hotfix on every tick",
|
|
" if candidate in have and candidate == bound and not bind:",
|
|
" if False:"),
|
|
|
|
# ------------------------------------------------------------------- idempotence
|
|
("sub-flow aliases are compared as names",
|
|
"compare the sub-flow alias like any other field -- the flows this program builds "
|
|
"carry a hash suffix on every alias, so a built flow could NEVER be recognised as "
|
|
"converged: it would be rebuilt and the login path reflipped every fifteen "
|
|
"minutes, forever",
|
|
" out.append((level, ex[\"kind\"], \"\" if is_flow else ex[\"name\"],\n"
|
|
" ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))",
|
|
" out.append((level, ex[\"kind\"], ex[\"name\"],\n"
|
|
" ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))"),
|
|
|
|
("the converged short-circuit is removed",
|
|
"drop the 'bound flow already matches git' early return -- every run rebuilds and "
|
|
"reflips, which is a permanent write loop against the IdP and churns the realm's "
|
|
"login path on a schedule",
|
|
" if not drift:\n"
|
|
" log(\"flow ok (the bound flow %r matches git: %d execution(s), no \"\n"
|
|
" \"write)\" % (bound, len(expected)))\n"
|
|
" return 0, problems",
|
|
" if False:\n return 0, problems"),
|
|
|
|
("the execution priority is dropped from the requirement write",
|
|
"omit `priority` from the body -- Keycloak's updateExecutions deserialises a "
|
|
"missing int as 0 and compares it against the stored priority, so this RESHUFFLES "
|
|
"the execution order while only a requirement change was intended. The passkey "
|
|
"branch would end up in the wrong place in the tree",
|
|
" body={\"id\": ex.get(\"id\"), \"requirement\": want_req,\n"
|
|
" \"priority\": ex.get(\"priority\") or 0})",
|
|
" body={\"id\": ex.get(\"id\"), \"requirement\": want_req})"),
|
|
|
|
("the alias budget is unbounded",
|
|
"loop forever looking for a free alias -- a persistently failing build then creates "
|
|
"unbounded flow sprawl in the realm, and nothing here can delete any of it",
|
|
" for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):",
|
|
" for attempt in range(1, 10000):"),
|
|
|
|
("fail-open when the bound flow cannot be read",
|
|
"carry on when the live flow's executions are unreadable -- the program then cannot "
|
|
"know which flows are live, so it cannot prove a build would miss them, and it "
|
|
"would build with the protected sets still undetermined",
|
|
" return 0, [\n"
|
|
" \"flow: the realm binds browserFlow to %r but its executions could not \"\n"
|
|
" \"be read, so this program cannot establish which flows are live. Flow \"\n"
|
|
" \"writes stay FAIL-CLOSED for this run -- nothing was built or bound\"\n"
|
|
" % bound]",
|
|
" _set_protected(set(), set())\n bound_shape, bound_raw = None, None"),
|
|
|
|
# ------------------------------------------------------- the conditional-OTP regression
|
|
("authenticator config contents no longer compared",
|
|
"check only that a config is BOUND, not what it says -- this is THE conditional-OTP "
|
|
"regression: the original bug wrote the role under "
|
|
"`condition.evaluated.user.role` instead of `condUserRole`, so the role resolved "
|
|
"to null, the REQUIRED conditional step threw 'Invalid role name submitted: null' "
|
|
"and every platform-admin got a 403 from every SSO-gated app",
|
|
" if str(have_cfg.get(key)) != str(want_cfg[key]):",
|
|
" if False:"),
|
|
|
|
("undeclared config keys are ignored",
|
|
"stop reporting keys the realm holds and git does not -- the stale "
|
|
"`condition.evaluated.user.role` key is the signature of the original null-role "
|
|
"bug, and a flow carrying both keys would look correct",
|
|
" extra = sorted(k for k in have_cfg if k not in want_cfg)",
|
|
" extra = []"),
|
|
|
|
("flow structure divergence detected but not reported",
|
|
"leave the per-step comparison out -- the realm's login flow could be the old "
|
|
"pre-passkey shape forever and every run would report the realm converged, which "
|
|
"is the exact fail-open of the one-shot import this program replaces",
|
|
" if observed[idx][:4] != expected[idx][:4]:",
|
|
" if False:"),
|
|
|
|
("the browserFlow BINDING is no longer compared",
|
|
"stop noticing that a correct flow is not the one bound -- a perfect, fully-built "
|
|
"passkey flow that nothing logs in through changes nothing at all",
|
|
" if bound_shape is not None:\n"
|
|
" drift = compare_shape(bound_shape, expected, base, realm, token, configs,",
|
|
" if False:\n"
|
|
" drift = compare_shape(bound_shape, expected, base, realm, token, configs,"),
|
|
|
|
# ------------------------------------------------------------- realm / policy writes
|
|
("partial policy PUT instead of read-modify-write",
|
|
"send only the drifted policy keys -- Keycloak 26.0.6 rebuilds the WHOLE "
|
|
"passwordless policy from the representation and substitutes its own DEFAULT for "
|
|
"every key the body omits, so a partial PUT silently resets everything it did not "
|
|
"mention while returning 204",
|
|
" body = dict(realm_rep)\n"
|
|
" for key in drift:\n"
|
|
" body[key] = wanted[key]",
|
|
" body = {RP_ENTITY_NAME: realm_rep.get(RP_ENTITY_NAME),\n"
|
|
" BROWSER_FLOW: realm_rep.get(BROWSER_FLOW)}\n"
|
|
" for key in drift:\n"
|
|
" body[key] = wanted[key]"),
|
|
|
|
("the policy pass rebinds browserFlow as a side effect",
|
|
"'while we are here, let the policy write fix the flow binding too' -- the realm "
|
|
"PUT does read rep.getBrowserFlow(), so this is genuinely in reach, and it would "
|
|
"rebind the login path to an UNVERIFIED flow from a pass whose stated surface is "
|
|
"the passkey policy",
|
|
" if body.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW):\n"
|
|
" raise RuntimeError(",
|
|
" body[BROWSER_FLOW] = \"browser\"\n"
|
|
" if False:\n"
|
|
" raise RuntimeError("),
|
|
|
|
("the RpEntityName gate removed from desired-state validation",
|
|
"accept a desired state with an empty RpEntityName -- Keycloak 26.0.6 applies the "
|
|
"passwordless policy ONLY when that field is non-empty, so every realm PUT would "
|
|
"be accepted with 204 and change nothing: reported success, zero convergence",
|
|
" if not policy.get(RP_ENTITY_NAME):\n"
|
|
" raise ValueError(",
|
|
" if False:\n raise ValueError("),
|
|
|
|
("the RpEntityName gate removed from the write path",
|
|
"drop the second, defence-in-depth guard -- it is unreachable while the "
|
|
"desired-state validation holds, which is exactly why deleting it would otherwise "
|
|
"be an invisible regression",
|
|
" if not body.get(RP_ENTITY_NAME):\n"
|
|
" raise RuntimeError(",
|
|
" if False:\n raise RuntimeError("),
|
|
|
|
("no readback verification on the policy write",
|
|
"trust the 204 -- a server that accepts the PUT and does not apply it is reported "
|
|
"as REPAIRED, which is 'reports success while the realm never converged' again, "
|
|
"and it is precisely what the RpEntityName gate does when it is missed",
|
|
" still = policy_drift(after, wanted)",
|
|
" still = {}"),
|
|
|
|
("AAGUID comparison made order-sensitive",
|
|
"compare the AAGUID allow-list in the order Keycloak returns it -- an allow-list "
|
|
"is a set membership test, so an order-sensitive comparison rewrites a CONVERGED "
|
|
"realm on every run forever while every run still reports success",
|
|
" if stored != sorted(declared_value):\n"
|
|
" drift[key] = (stored, sorted(declared_value))",
|
|
" if stored != declared_value:\n"
|
|
" drift[key] = (stored, declared_value)"),
|
|
|
|
("signature-algorithm order treated as insignificant",
|
|
"sort the signature algorithms before comparing -- it is a PREFERENCE LIST handed "
|
|
"to the authenticator, so ES256-then-RS256 is not the same declaration as "
|
|
"RS256-then-ES256, and a reorder in the realm would never be corrected",
|
|
" if stored != list(declared_value):",
|
|
" if sorted(stored) != sorted(declared_value):"),
|
|
|
|
("unmanaged policy keys forced to a value",
|
|
"iterate every writable key instead of only the declared ones -- a "
|
|
"webAuthnPolicyPasswordless field the realm holds and git does not declare would "
|
|
"be overwritten, turning 'not declared' into 'must equal my default'",
|
|
" for key in sorted(wanted):\n declared_value = wanted[key]",
|
|
" for key in POLICY_FIELDS:\n declared_value = wanted.get(key, \"\")"),
|
|
|
|
# ------------------------------------------------------------------ required actions
|
|
("required-action PUT sends only the two drifted fields",
|
|
"a minimal body looks safer -- but updateRequiredAction REPLACES the provider from "
|
|
"the representation, so name, priority and config are all blanked: the enrolment "
|
|
"action keeps working but loses its priority and display name",
|
|
" body = dict(rep)\n for key in drift:\n body[key] = wanted[key]",
|
|
" body = {}\n for key in drift:\n body[key] = wanted[key]"),
|
|
|
|
("the guarded default required actions are no longer audited",
|
|
"stop checking that CONFIGURE_TOTP / UPDATE_PASSWORD / delete_credential still "
|
|
"exist -- their disappearance is the signature of a requiredActions block in the "
|
|
"realm import, which 26.0.6 treats as REPLACE, and it is a silent permanent "
|
|
"lock-out of every platform-admin",
|
|
" missing = [a for a in GUARDED_ACTIONS if a not in aliases]",
|
|
" missing = []"),
|
|
|
|
# ----------------------------------------------------------------- inputs / coupling
|
|
("the three passes are coupled",
|
|
"abort the run at the first failing pass -- a policy write that fails for an "
|
|
"unrelated reason would then suppress the browser-flow pass, which is the most "
|
|
"safety-critical thing here",
|
|
" problems.extend(found)\n if found:\n rc = 1",
|
|
" problems.extend(found)\n if found:\n rc = 1\n"
|
|
" break"),
|
|
|
|
("policy-key validation removed from the desired state",
|
|
"accept any realm key from the ConfigMap -- an arbitrary realm attribute becomes "
|
|
"writable through a file edit, so a hand-edited ConfigMap could set sslRequired, "
|
|
"bruteForceProtected or any other top-level realm field on the live IdP",
|
|
" if key not in POLICY_FIELDS:",
|
|
" if False:"),
|
|
|
|
("fail-open on an unreadable desired state",
|
|
"treat a missing ConfigMap as 'nothing declared' -- the run reports success while "
|
|
"reconciling nothing, which is precisely the fail-open of the realm import this "
|
|
"whole component exists to replace",
|
|
" if not isinstance(raw, dict) or not raw:\n"
|
|
" raise ValueError(\"%s does not hold a non-empty JSON object\" % path)",
|
|
" if not isinstance(raw, dict) or not raw:\n return {}"),
|
|
|
|
("an undeclared authenticatorConfig reference is tolerated",
|
|
"stop validating that every configAlias a flow references is actually declared -- "
|
|
"the build then fails at the config-binding step, AFTER flows and executions have "
|
|
"already been created, leaving leftovers on every run",
|
|
" if ca and ca not in configs:\n"
|
|
" raise ValueError(",
|
|
" if False:\n raise ValueError("),
|
|
|
|
("the flow-cycle guard is removed",
|
|
"drop the `seen` check -- a cyclic flow declaration then recurses until the "
|
|
"interpreter dies, in a CronJob that retries",
|
|
" if alias in seen:\n"
|
|
" raise ValueError(\"flow %r is cyclic in the desired state\" % alias)",
|
|
" if False:\n"
|
|
" raise ValueError(\"flow %r is cyclic in the desired state\" % alias)"),
|
|
]
|
|
|
|
|
|
def mutation_jobs() -> int:
|
|
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
|
|
|
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
|
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
|
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
|
"""
|
|
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
|
if raw:
|
|
try:
|
|
n = int(raw)
|
|
except ValueError:
|
|
n = 0
|
|
if n < 1:
|
|
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
|
return n
|
|
try:
|
|
cpus = len(os.sched_getaffinity(0))
|
|
except (AttributeError, OSError):
|
|
cpus = os.cpu_count() or 1
|
|
return max(1, min(8, cpus))
|
|
|
|
|
|
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
|
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
|
fh.write(mutated_src)
|
|
path = fh.name
|
|
try:
|
|
return subprocess.run([sys.executable, path, "--selftest"],
|
|
capture_output=True, text=True, timeout=300,
|
|
stdin=subprocess.DEVNULL)
|
|
finally:
|
|
Path(path).unlink(missing_ok=True)
|
|
|
|
|
|
def main() -> int:
|
|
if len(sys.argv) != 2:
|
|
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
|
return 2
|
|
src = Path(sys.argv[1]).read_text()
|
|
|
|
try:
|
|
jobs = mutation_jobs()
|
|
except ValueError as e:
|
|
print("mutations.py: %s" % e, file=sys.stderr)
|
|
return 2
|
|
|
|
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
|
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
|
planned: list[tuple[str, str, str | None, str | None]] = []
|
|
for name, why, old, new in MUTATIONS:
|
|
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
|
# moved and this mutation silently stopped proving anything.
|
|
n = src.count(old)
|
|
if n != 1:
|
|
planned.append((name, why,
|
|
"%s: anchor matches %d times (expected exactly 1) -- the "
|
|
"code moved and this mutation no longer proves anything. "
|
|
"Re-anchor it against the current source." % (name, n), None))
|
|
continue
|
|
planned.append((name, why, None, src.replace(old, new, 1)))
|
|
|
|
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
|
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
|
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
|
# same verdict, as the serial loop produced -- whichever child finishes first.
|
|
problems: list[str] = []
|
|
killed = 0
|
|
pool = ThreadPoolExecutor(max_workers=jobs)
|
|
try:
|
|
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
|
for (_n, _w, _a, mutated) in planned]
|
|
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
|
if anchor_problem is not None:
|
|
problems.append(anchor_problem)
|
|
continue
|
|
proc = fut.result()
|
|
if proc.returncode == 0:
|
|
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
|
"applied, so nothing in the suite actually checks it. The "
|
|
"mistake it models: %s" % (name, why))
|
|
continue
|
|
killed += 1
|
|
out = proc.stdout + proc.stderr
|
|
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
|
how = ("%s assertion(s) caught it" % m.group(1) if m
|
|
else "the selftest aborted (caught, less cleanly)")
|
|
print(" killed %-54s %s" % (name, how))
|
|
finally:
|
|
# A timeout (or any other exception) still propagates and fails the check, as it did
|
|
# serially; queued mutations are cancelled rather than run for nothing.
|
|
pool.shutdown(wait=True, cancel_futures=True)
|
|
|
|
print()
|
|
if problems:
|
|
print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven"
|
|
% (len(problems), len(MUTATIONS)), file=sys.stderr)
|
|
for p in problems:
|
|
print(" FAIL %s" % p, file=sys.stderr)
|
|
return 1
|
|
print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest "
|
|
"(the suite is falsifiable, not decorative)" % killed)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|