jakman85andClaude Opus 5.5
a775082ae1
- conformance/lib/run_l1_offline.py: checks run in a thread pool (--jobs / CONF_OFFLINE_JOBS,
default min(8, usable CPUs)); results consumed in submission order, so log lines, JUnit,
step summary and the 0/1/3 aggregation are unchanged. Exports a smaller MUTATION_JOBS to
the children when parallel, so nested pools do not oversubscribe.
- keycloak-{passkey,client-secret}-reconciler/mutations.py: anchors checked first (serial),
then the --selftest subprocesses run in a pool (MUTATION_JOBS, default min(8, CPUs));
verdicts reported in MUTATIONS order. Output byte-identical to the serial loop.
- ci/lint.sh: yamllint over LINT_JOBS contiguous slices concurrently, per-slice output
buffered and printed in order; stage rc = highest slice rc.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
251 lines
12 KiB
Python
Executable File
251 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# =============================================================================
|
|
# mutations.py -- prove the reconciler's --selftest IS FALSIFIABLE.
|
|
#
|
|
# WHY THIS EXISTS
|
|
# ---------------
|
|
# A selftest that passes proves nothing on its own: a selftest asserting `True ==
|
|
# True` also passes. The reconciler exists because a one-shot import REPORTED
|
|
# SUCCESS while the realm was broken, and the same trap applies one level up -- a
|
|
# green test suite that cannot fail is the same lie in a different place.
|
|
#
|
|
# So this injects a series of PLAUSIBLE regressions into the shipped script, one at
|
|
# a time, and requires that --selftest FAILS for every one. A mutation that
|
|
# SURVIVES means the suite does not actually check that property, and the check
|
|
# fails, naming it.
|
|
#
|
|
# Each mutation is a mistake a reasonable person could make while editing this
|
|
# code -- not a nonsense edit. Several were live design questions while it was
|
|
# written (order-sensitivity, additive-vs-exact URI sets, whether the two passes
|
|
# should share a failure path); the mutation is the record that the answer is
|
|
# enforced and not merely commented.
|
|
#
|
|
# Invoked by check.sh with the script EXTRACTED from the shipped ConfigMap, so what
|
|
# is proven falsifiable is byte-for-byte what runs on the cluster.
|
|
#
|
|
# python3 mutations.py <path-to-extracted-reconciler.py>
|
|
#
|
|
# Runtime is ~11s serial (one subprocess per mutation, each running the full selftest
|
|
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
|
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
|
# verdicts are independent of the pool size.
|
|
# =============================================================================
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
|
|
def dedent4(text: str) -> str:
|
|
"""Anchors are written with the YAML file's indentation. The script extracted from
|
|
the ConfigMap block scalar has 4 fewer leading spaces on every line, so anchors are
|
|
authored to match the YAML (easy to diff against the manifest) and dedented here."""
|
|
return "\n".join(ln[4:] if ln.startswith(" ") else ln for ln in text.split("\n"))
|
|
|
|
|
|
# (name, the mistake being modelled, anchor, replacement)
|
|
MUTATIONS: list[tuple[str, str, str, str]] = [
|
|
("order-sensitive URL comparison",
|
|
"compare redirectUris in the order Keycloak returns them -- Keycloak does not "
|
|
"preserve the authored order, so a converged realm is rewritten on every run "
|
|
"forever while every run still reports success",
|
|
"if sorted(stored) != sorted(declared_value):",
|
|
"if stored != declared_value:"),
|
|
|
|
("fat round-tripped URL PUT",
|
|
"PUT the whole client representation back instead of only the drifted fields -- "
|
|
"the live server rejects it with 400 'Cannot parse the JSON', and even if it did "
|
|
"not, the blast radius would cover every field of the client",
|
|
" rep = {\"clientId\": client.get(\"clientId\")}\n"
|
|
" for field in sorted(drift):\n"
|
|
" rep[field] = want[cid][field]",
|
|
" rep = dict(client)\n"
|
|
" for field in sorted(drift):\n"
|
|
" rep[field] = want[cid][field]"),
|
|
|
|
("write unconditionally (compare-then-write removed)",
|
|
"drop the converged short-circuit -- every client is rewritten on every run, so "
|
|
"the CronJob becomes a permanent write load against the IdP database",
|
|
" drift = url_drift(client, want[cid])\n if not drift:",
|
|
" drift = url_drift(client, want[cid]) or dict(\n"
|
|
" (f, (\"\", want[cid][f])) for f in want[cid])\n if False:"),
|
|
|
|
("additive-only URI reconciliation",
|
|
"union the declared URIs with the stored ones -- a redirect URI REMOVED from git "
|
|
"is never removed from the realm, so a stale wildcard keeps working and quietly "
|
|
"widens what the realm will redirect to",
|
|
" rep[field] = want[cid][field]",
|
|
" rep[field] = (sorted(set(want[cid][field]) | set(client.get(field) or []))\n"
|
|
" if field in URL_LIST_FIELDS else want[cid][field])"),
|
|
|
|
("URL pass fails OPEN on an unreadable input",
|
|
"treat a missing ConfigMap as 'nothing declared' -- the run reports success while "
|
|
"reconciling nothing, which is precisely the fail-open of the realm import",
|
|
" except OSError as e:\n"
|
|
" url_input_fail = (\"cannot read %s (%s) -- the keycloak-client-urls ConfigMap \"\n"
|
|
" \"is not mounted\" % (URLS_FILE, e.__class__.__name__))",
|
|
" except OSError:\n want_urls = {}"),
|
|
|
|
("URL pass skips public clients",
|
|
"copy the secret pass's publicClient skip into the URL pass -- the `kubernetes` "
|
|
"client is PKCE-public, so kubectl's redirect URIs would never converge",
|
|
" drift = url_drift(client, want[cid])",
|
|
" if client.get(\"publicClient\"):\n continue\n"
|
|
" drift = url_drift(client, want[cid])"),
|
|
|
|
("the two passes are coupled",
|
|
"abort the whole run when either input is bad -- a typo in a hostname ConfigMap "
|
|
"would stop client-SECRET repair, making credential healing depend on a file that "
|
|
"has nothing to do with credentials",
|
|
" if want_secrets is None and want_urls is None:",
|
|
" if want_secrets is None or want_urls is None:"),
|
|
|
|
("unmanaged fields forced to empty",
|
|
"iterate every writable field instead of only the declared ones -- a rootUrl or "
|
|
"adminUrl the realm holds and git does not declare would be WIPED, turning "
|
|
"'not declared' into 'must be empty'",
|
|
" for field, declared_value in sorted(wanted.items()):",
|
|
" for field in URL_FIELDS:\n"
|
|
" declared_value = wanted.get(\n"
|
|
" field, [] if field in URL_LIST_FIELDS else \"\")"),
|
|
|
|
("no readback verification",
|
|
"trust the 204 -- a server that accepts the PUT and does not apply it is reported "
|
|
"as REPAIRED, which is 'reports success while the realm never converged' again",
|
|
" still = url_drift(after[0], want[cid])",
|
|
" still = {}"),
|
|
|
|
("allow-list widened to the whole realm",
|
|
"add a realm-level endpoint 'just in case' -- /users, /groups, /roles and "
|
|
"/authentication stop being structurally unreachable",
|
|
" r\"/admin/realms/[^/]+/clients\",",
|
|
" r\"/admin/realms/[^/]+\",\n r\"/admin/realms/[^/]+/clients\","),
|
|
|
|
("desired-state field validation removed",
|
|
"accept any field name from the ConfigMap -- an arbitrary client field becomes "
|
|
"writable through a file edit, bypassing the five-field write surface entirely",
|
|
" if field not in URL_FIELDS:",
|
|
" if False:"),
|
|
|
|
("URL drift detected but never repaired",
|
|
"leave the audit-mode early-continue in place on the write path -- drift is "
|
|
"reported in the log and silently not fixed",
|
|
" if audit:\n continue\n # MINIMAL representation",
|
|
" if True:\n continue\n # MINIMAL representation"),
|
|
]
|
|
|
|
|
|
def mutation_jobs() -> int:
|
|
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
|
|
|
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
|
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
|
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
|
"""
|
|
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
|
if raw:
|
|
try:
|
|
n = int(raw)
|
|
except ValueError:
|
|
n = 0
|
|
if n < 1:
|
|
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
|
return n
|
|
try:
|
|
cpus = len(os.sched_getaffinity(0))
|
|
except (AttributeError, OSError):
|
|
cpus = os.cpu_count() or 1
|
|
return max(1, min(8, cpus))
|
|
|
|
|
|
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
|
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
|
fh.write(mutated_src)
|
|
path = fh.name
|
|
try:
|
|
return subprocess.run([sys.executable, path, "--selftest"],
|
|
capture_output=True, text=True, timeout=240,
|
|
stdin=subprocess.DEVNULL)
|
|
finally:
|
|
Path(path).unlink(missing_ok=True)
|
|
|
|
|
|
def main() -> int:
|
|
if len(sys.argv) != 2:
|
|
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
|
return 2
|
|
src = Path(sys.argv[1]).read_text()
|
|
|
|
try:
|
|
jobs = mutation_jobs()
|
|
except ValueError as e:
|
|
print("mutations.py: %s" % e, file=sys.stderr)
|
|
return 2
|
|
|
|
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
|
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
|
planned: list[tuple[str, str, str | None, str | None]] = []
|
|
for name, why, old, new in MUTATIONS:
|
|
old, new = dedent4(old), dedent4(new)
|
|
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
|
# moved and this mutation silently stopped proving anything.
|
|
n = src.count(old)
|
|
if n != 1:
|
|
planned.append((name, why,
|
|
"%s: anchor matches %d times (expected exactly 1) -- the "
|
|
"code moved and this mutation no longer proves anything. "
|
|
"Re-anchor it against the current source." % (name, n), None))
|
|
continue
|
|
planned.append((name, why, None, src.replace(old, new, 1)))
|
|
|
|
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
|
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
|
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
|
# same verdict, as the serial loop produced -- whichever child finishes first.
|
|
problems: list[str] = []
|
|
killed = 0
|
|
pool = ThreadPoolExecutor(max_workers=jobs)
|
|
try:
|
|
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
|
for (_n, _w, _a, mutated) in planned]
|
|
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
|
if anchor_problem is not None:
|
|
problems.append(anchor_problem)
|
|
continue
|
|
proc = fut.result()
|
|
if proc.returncode == 0:
|
|
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
|
"applied, so nothing in the suite actually checks it. The "
|
|
"mistake it models: %s" % (name, why))
|
|
continue
|
|
killed += 1
|
|
out = proc.stdout + proc.stderr
|
|
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
|
how = ("%s assertion(s) caught it" % m.group(1) if m
|
|
else "the selftest aborted (caught, less cleanly)")
|
|
print(" killed %-42s %s" % (name, how))
|
|
finally:
|
|
# A timeout (or any other exception) still propagates and fails the check, as it did
|
|
# serially; queued mutations are cancelled rather than run for nothing.
|
|
pool.shutdown(wait=True, cancel_futures=True)
|
|
|
|
print()
|
|
if problems:
|
|
print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven"
|
|
% (len(problems), len(MUTATIONS)), file=sys.stderr)
|
|
for p in problems:
|
|
print(" FAIL %s" % p, file=sys.stderr)
|
|
return 1
|
|
print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest "
|
|
"(the suite is falsifiable, not decorative)" % killed)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|