home-network-k3s-impl/conformance/tests/l0-static/keycloak-client-secret-reconciler/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — keycloak-client-secret-reconciler
# Serves: R-IDP-03 ("Identity config (realms, clients, groups, roles) is declarative."
#                   — spec docs/15-identity-and-access.md)
#         R-IDP-04 ("IdP secrets SOPS-encrypted; MFA available."
#                   — spec docs/15-identity-and-access.md)
#
# WHY THIS EXISTS — THE REALM IMPORT IS A ONE-SHOT THAT FAILS OPEN
# ---------------------------------------------------------------
# platform/keycloak/keycloak-realm-import.yaml declares 12 confidential OIDC clients
# whose secrets are `(env:NAME)` import placeholders, mapped by spec.placeholders to
# keys of the SOPS-encrypted keycloak-realm-clients Secret. Measured on both clusters
# on 2026-09-06:
#
#   * when the Keycloak Operator cannot resolve a placeholder it stores the LITERAL
#     placeholder text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS;
#   * the KeycloakRealmImport then rests at Done=True / Started=False and NEVER
#     re-runs, so it never self-heals.
#
# Consequence: on prod all 12 clients held the literal placeholder and every OIDC
# login on the cluster failed ("Invalid client or Invalid client credentials"); on dev
# exactly one (oauth2-proxy) was broken, because that Secret key was added AFTER dev's
# import had already completed and a completed import is never re-run.
#
# Nothing in the repo asserted the declaration was self-consistent, and nothing
# corrected the realm afterwards. This check is the static half of the fix (the
# reconciler CronJob is the runtime half, and this check proves that too).
#
# GIVEN the repository as authored,
# THEN  (1) DECLARATION IS COMPLETE: every confidential client in the realm import
#           declares a secret placeholder; every placeholder it uses is defined in
#           spec.placeholders; every placeholder resolves to the keycloak-realm-clients
#           Secret; and every referenced key EXISTS in the SOPS file. A client whose
#           key is missing is exactly the dev failure, and it is now unmergeable.
#       (2) THE NAMING CONTRACT HOLDS: each key is `<clientId>-client-secret`. That is
#           what lets the reconciler derive the client set from the mounted Secret with
#           no second declaration — so it must be enforced, not assumed. No orphan
#           placeholder, and no orphan Secret key (an orphan key would make every
#           reconciler run fail with NOT FOUND).
#       (3) PUBLIC CLIENTS DECLARE NO SECRET (the `kubernetes` client is PKCE-public;
#           giving it a secret would be a silent downgrade).
#       (4) ORDERING IS A REAL FLUX EDGE, NOT A COMMENT: the Flux Kustomization that
#           renders the Secret is a transitive `dependsOn` ancestor — with `wait: true`
#           — of the one that renders the realm import and the reconciler. The repo's
#           hard-won rule: a dependency that is only DOCUMENTED is not a dependency.
#       (5) THE RECONCILER IS WIRED AND SAFE: it is reconciled from BOTH cluster
#           entrypoints, mounts exactly the two Secrets it needs at the paths its code
#           defaults to, carries no ServiceAccount token, and contains no dollar sign
#           (it ships inside a substituted Flux path, where an unescaped dollar-brace
#           expansion is rewritten before any container runs).
#       (6) THE RECONCILER'S BEHAVIOUR IS PROVEN, NOT ASSERTED: this check EXTRACTS the
#           script from the ConfigMap and executes its `--selftest`, which drives the
#           real code paths against an in-process stub Keycloak and asserts
#           compare-then-write, idempotence (zero writes on a converged realm),
#           no-secret-logging, the endpoint allow-list (users/groups/roles are
#           unreachable) and fail-closed behaviour. What CI proves is byte-for-byte
#           what ships.
#      (6b) AND THAT PROOF IS FALSIFIABLE: mutations.py injects plausible regressions
#           into the extracted script one at a time and requires --selftest to CATCH
#           every one. A passing suite proves nothing if it cannot fail, and "reported
#           success while the realm was broken" is the exact failure this component
#           exists to end — so the same standard is applied to its own test.
#       (8) THE URL DESIRED STATE IS DERIVED, NOT DECLARED TWICE: the reconciler's
#           second pass converges the realm's environment-derived client URLs
#           (redirectUris, webOrigins, rootUrl, baseUrl, adminUrl) on
#           platform/keycloak/client-urls.yaml. That file is GENERATED from the realm
#           import by ci/render-keycloak-client-urls.py, and this check RUNS the
#           renderer with --check, so a hand edit that makes the two disagree is
#           unmergeable. It further asserts the two files are rendered by the SAME Flux
#           Kustomization (postBuild.substituteFrom is per-Kustomization, so a split
#           would let one ${BASE_DOMAIN} resolve to two different values), that the
#           desired state uses no substitution token the realm definition does not,
#           that every client it names exists in the import, and that it declares no
#           field outside the reconciler's five-field write surface.
#   AND (7) THIS CHECK IS LOAD-BEARING: it DECLARES `conformance.home.k3s/scope: whole-repo`
#           so ci/l0.sh runs it unconditionally via --include-scope, ci/gate.sh
#           blocks the merge when the l0 stage fails, ci/touched-requirements.sh maps
#           platform/keycloak/ to these requirement IDs, and an L2 test exists that
#           audits the LIVE realm. An unreferenced test is precisely the "looks applied
#           and isn't" failure this repo keeps hitting.
#
# NOT proven here (named honestly, in the style of the sibling L0 checks):
#   - that the LIVE realm's stored secrets OR redirect URIs match git. That is cluster
#     state; it is conformance/tests/l2-behavioral/keycloak-client-secrets-match-git,
#     which runs the reconciler's own --audit against the live realm.
#   - that the hostnames the realm import declares are the ones the Gateway/HTTPRoutes
#     actually serve. This check proves the realm converges on what GIT says; it cannot
#     prove git is right about the domain. A BASE_DOMAIN flip that misses an HTTPRoute is
#     a different invariant.
#   - that the SOPS ciphertext decrypts to a usable secret. Key PRESENCE is checkable
#     offline; the plaintext is not, and must not be.
#   - that the CronJob's image can be pulled or its schedule fires.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"

IMPORT_FILE="platform/keycloak/keycloak-realm-import.yaml"
SOPS_FILE="platform/keycloak/keycloak-realm-clients.sops.yaml"
RECON_FILE="platform/keycloak/client-secret-reconciler.yaml"
URLS_FILE="platform/keycloak/client-urls.yaml"

# Honest SKIP (never green) if the thing under test is absent entirely.
for f in "$IMPORT_FILE" "$SOPS_FILE" "$RECON_FILE" "$URLS_FILE"; do
  if [ ! -f "${ROOT}/${f}" ]; then
    echo "check SKIPPED: ${f} absent — nothing to audit" >&2
    exit 3
  fi
done

cd "${ROOT}"
"${PYTHON}" - "${ROOT}" <<'PY'
"""Static audit of the Keycloak client-secret declaration, its Flux ordering, and the
reconciler that heals it. PyYAML (the same dependency tests/l0-static/no-plaintext-secrets
and least-privilege-rbac already take) parses the manifests; lib/status.py is IMPORTED for
the Flux graph walk rather than forked, so there is one definition of "what a Flux path
renders" in the harness."""
from __future__ import annotations

import json
import re
import subprocess
import sys
import tempfile
from pathlib import Path

try:
    import yaml
except ImportError:                                  # pragma: no cover
    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
          file=sys.stderr)
    raise SystemExit(1)

ROOT = Path(sys.argv[1])
sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree
sys.path.insert(0, str(ROOT / "conformance" / "lib"))
import status as st                     # noqa: E402 — reuse the walk, don't fork it

IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
SOPS_FILE = "platform/keycloak/keycloak-realm-clients.sops.yaml"
RECON_FILE = "platform/keycloak/client-secret-reconciler.yaml"
URLS_FILE = "platform/keycloak/client-urls.yaml"
SECRET_NAME = "keycloak-realm-clients"
ADMIN_SECRET = "keycloak-admin-credentials"
URLS_CM_NAME = "keycloak-client-urls"
KEY_SUFFIX = "-client-secret"
SECRETS_MOUNT = "/etc/keycloak-realm-clients"
URLS_MOUNT = "/etc/keycloak-client-urls"
URLS_DATA_KEY = "client-urls.json"
RENDERER = "ci/render-keycloak-client-urls.py"
DIR_PATH = ROOT / "conformance" / "tests" / "l0-static" / "keycloak-client-secret-reconciler"
# The ONLY client fields the reconciler may write besides the secret. Kept in lockstep
# with the renderer and the script; asserted below to be exactly what they agree on.
URL_LIST_FIELDS = ("redirectUris", "webOrigins")
URL_STR_FIELDS = ("rootUrl", "baseUrl", "adminUrl")
URL_FIELDS = URL_LIST_FIELDS + URL_STR_FIELDS
DOLLAR = chr(36)

FAILURES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(msg)


def load(rel):
    return list(yaml.safe_load_all((ROOT / rel).read_text()))


# ================================================================= (1)-(3) declaration
realm_import = next((d for d in load(IMPORT_FILE)
                     if d and d.get("kind") == "KeycloakRealmImport"), None)
check(realm_import is not None, f"{IMPORT_FILE}: no KeycloakRealmImport document found")
if realm_import is None:
    print("CHECK FAIL: cannot parse the realm import", file=sys.stderr)
    raise SystemExit(1)

placeholders = realm_import["spec"].get("placeholders") or {}
clients = realm_import["spec"]["realm"].get("clients") or []
check(len(clients) > 0, f"{IMPORT_FILE}: parsed zero clients — the file shape changed and "
                        f"this check would silently audit nothing")

# The operator's placeholder syntax, spelled without a literal dollar so this file is
# itself safe to grep for one. Matches `<dollar>(env:NAME)`.
PLACEHOLDER_RE = re.compile(re.escape(DOLLAR) + r"\(env:([A-Za-z_][A-Za-z0-9_]*)\)")

used_placeholders: set[str] = set()
declared_pairs: dict[str, str] = {}          # clientId -> secret key in the Secret

for c in clients:
    cid = c.get("clientId", "<unnamed>")
    secret = c.get("secret")
    public = bool(c.get("publicClient"))
    if public:
        # (3) a PKCE public client must not carry a secret at all.
        check(secret is None,
              f"{IMPORT_FILE}: client '{cid}' is publicClient but declares a secret — a "
              f"public client holds no secret; declaring one is a silent downgrade")
        continue
    # (1) every confidential client must declare one.
    check(isinstance(secret, str) and secret,
          f"{IMPORT_FILE}: confidential client '{cid}' declares NO secret. The operator "
          f"would generate a random one that exists nowhere in git, so nothing could "
          f"ever reconcile it and the consuming app's mirrored credential would be wrong")
    if not isinstance(secret, str):
        continue
    m = PLACEHOLDER_RE.fullmatch(secret.strip())
    check(m is not None,
          f"{IMPORT_FILE}: client '{cid}' has a secret that is not an (env:NAME) "
          f"placeholder. A literal secret in git violates R-IDP-04/R-SEC-01; the only "
          f"permitted form is a placeholder resolved from the {SECRET_NAME} Secret")
    if m is None:
        continue
    name = m.group(1)
    used_placeholders.add(name)
    # (1) the placeholder must be DEFINED. An undefined one is stored literally by the
    #     operator, with a successful-looking import — the whole incident.
    ph = placeholders.get(name)
    check(ph is not None,
          f"{IMPORT_FILE}: client '{cid}' uses placeholder '{name}' but spec.placeholders "
          f"does not define it. The operator stores the LITERAL placeholder as the client "
          f"secret and STILL reports the import successful (measured 2026-09-06)")
    if ph is None:
        continue
    ref = (ph or {}).get("secret") or {}
    check(ref.get("name") == SECRET_NAME,
          f"{IMPORT_FILE}: placeholder '{name}' reads Secret '{ref.get('name')}', not "
          f"'{SECRET_NAME}'. The reconciler mounts only {SECRET_NAME}, so a second source "
          f"would be declared but never healed")
    key = ref.get("key")
    check(isinstance(key, str) and key,
          f"{IMPORT_FILE}: placeholder '{name}' names no Secret key")
    if not isinstance(key, str):
        continue
    # (2) the naming contract the reconciler derives its client set from.
    check(key == cid + KEY_SUFFIX,
          f"{IMPORT_FILE}: client '{cid}' reads Secret key '{key}', but the reconciler "
          f"derives the clientId from the key name, so it MUST be "
          f"'{cid}{KEY_SUFFIX}'. Rename the key (and the SOPS entry) or the reconciler "
          f"would look up a client that does not exist")
    declared_pairs[cid] = key

# (2) no orphan placeholder: a defined-but-unused placeholder is dead config that will
#     quietly rot out of sync with the Secret.
for name in sorted(set(placeholders) - used_placeholders):
    check(False,
          f"{IMPORT_FILE}: spec.placeholders defines '{name}' but no client uses it — "
          f"dead declaration; remove it or wire it to the client that needs it")

# ---- the SOPS Secret: keys are cleartext in a SOPS-encrypted file; values are not.
sops_doc = next((d for d in load(SOPS_FILE) if d and d.get("kind") == "Secret"), None)
check(sops_doc is not None, f"{SOPS_FILE}: no Secret document found")
sops_keys = set((sops_doc or {}).get("stringData") or {})
check(len(sops_keys) > 0,
      f"{SOPS_FILE}: parsed zero stringData keys — the file shape changed and this check "
      f"would silently audit nothing")
check((sops_doc or {}).get("metadata", {}).get("name") == SECRET_NAME,
      f"{SOPS_FILE}: the Secret is not named {SECRET_NAME}")

for cid, key in sorted(declared_pairs.items()):
    # (1) THE DEV FAILURE, made unmergeable: a client declared in the import whose key
    #     is absent from the Secret imports "successfully" with a literal placeholder.
    check(key in sops_keys,
          f"{SOPS_FILE}: client '{cid}' needs key '{key}' and it is NOT in the Secret. "
          f"The realm import would still report SUCCESS and store the literal placeholder "
          f"as that client's secret — this is exactly how dev's oauth2-proxy broke")

# (2) no orphan Secret key: the reconciler treats every `<x>-client-secret` key as a
#     client that MUST exist, and fails loudly if it does not. Catch it here instead.
for key in sorted(k for k in sops_keys if k.endswith(KEY_SUFFIX)):
    cid = key[: -len(KEY_SUFFIX)]
    check(cid in declared_pairs,
          f"{SOPS_FILE}: key '{key}' has no matching client in {IMPORT_FILE}. The "
          f"reconciler would look up client '{cid}', not find it, and fail every run")

# ===================================================== (4) ordering is a real Flux edge
# Parse contract mirrors lib/status.py's flux_paths(): docs split on a lone `---`; a doc
# is an entrypoint only if it carries the kustomize.toolkit apiVersion, `kind:
# Kustomization` and a spec-level `path:`. Overlay `patch: |-` bodies carry Kustomizations
# WITHOUT a path and are correctly skipped (they are patches, not entrypoints).
_DEPENDSON_RE = re.compile(r"^(\s*)dependsOn:\s*$", re.M)
_DEP_NAME_RE = re.compile(r"^\s*(?:-\s*)?name:\s*([^\s#]+)")
_WAIT_RE = re.compile(r"^\s{2,6}wait:\s*true\s*(?:#.*)?$", re.M)


def depends_on(doc: str) -> list[str]:
    m = _DEPENDSON_RE.search(doc)
    if not m:
        return []
    indent = len(m.group(1))
    out: list[str] = []
    for raw in doc[m.end():].splitlines():
        if raw.strip() == "" or raw.lstrip().startswith("#"):
            continue
        if len(raw) - len(raw.lstrip()) <= indent and not raw.lstrip().startswith("-"):
            break                                    # dedented out of the list
        nm = _DEP_NAME_RE.match(raw)
        if nm:
            out.append(nm.group(1))
    return out


kustomizations: dict[str, dict] = {}
for yfile in sorted((ROOT / "clusters").rglob("*.yaml")):
    text = yfile.read_text()
    if st.FLUX_API not in text:
        continue
    for doc in st._DOC_SPLIT_RE.split(text):
        if st.FLUX_API not in doc or not re.search(r"^kind:\s*Kustomization\s*$", doc, re.M):
            continue
        pm = st._PATH_RE.search(doc)
        if not pm:
            continue
        nm = st._NAME_RE.search(doc)
        name = nm.group(1) if nm else "?"
        kustomizations[name] = {
            "file": str(yfile.relative_to(ROOT)),
            "path": pm.group(1),
            "dependsOn": depends_on(doc),
            "wait": bool(_WAIT_RE.search(doc)),
        }

check(len(kustomizations) > 0,
      "no Flux Kustomization with a spec.path found under clusters/ — the parser or the "
      "repo layout changed; this check would silently audit nothing")


def direct_render(path_spec: str) -> set[str]:
    """Files a Flux `spec.path` renders ITSELF — a kustomize walk that deliberately does
    NOT expand nested Flux Kustomizations.

    lib/status.py's reconciled_files() follows every `spec.path` it discovers, which is
    right for "is this file applied to the cluster at all" but wrong for OWNERSHIP: the
    root Kustomization's path is clusters/<env>, so a transitive walk makes the root the
    owner of every file in the repo and the ordering assertion below becomes vacuous
    (measured: it reported the root as an owner of all three keycloak files). `resources:`
    parsing itself is st.kustomize_resources, not forked.
    """
    rel = path_spec.lstrip("./") or "."
    out: set[str] = set()
    seen: set[str] = set()

    def walk(d: str) -> None:
        if d in seen:
            return
        seen.add(d)
        kfile = ROOT / d / "kustomization.yaml"
        if not kfile.is_file():
            return
        out.add(str(kfile.relative_to(ROOT)))
        for item in st.kustomize_resources(kfile):
            if "://" in item:
                continue                       # remote base: not repo state
            target = (ROOT / d / item).resolve()
            try:
                trel = target.relative_to(ROOT).as_posix()
            except ValueError:
                continue                       # outside the repo
            if target.is_dir():
                walk(trel)
            elif target.is_file():
                out.add(trel)

    if (ROOT / rel).is_dir():
        walk(rel)
    return out


rendered_by = {n: direct_render(k["path"]) for n, k in kustomizations.items()}


def owners(rel_file: str) -> list[str]:
    return sorted(n for n, files in rendered_by.items() if rel_file in files)


def ancestors(name: str, seen=None) -> set[str]:
    """Transitive dependsOn closure of `name` (its dependencies, direct and indirect)."""
    seen = seen if seen is not None else set()
    for dep in kustomizations.get(name, {}).get("dependsOn", []):
        if dep in seen:
            continue
        seen.add(dep)
        ancestors(dep, seen)
    return seen


secret_owners = owners(SOPS_FILE)
import_owners = owners(IMPORT_FILE)
recon_owners = owners(RECON_FILE)

check(bool(secret_owners),
      f"{SOPS_FILE} is rendered by NO Flux Kustomization — the client secrets are in git "
      f"but never applied")
check(bool(import_owners),
      f"{IMPORT_FILE} is rendered by NO Flux Kustomization")
# (5) the reconciler must actually be reconciled, or it is decoration.
check(bool(recon_owners),
      f"{RECON_FILE} is rendered by NO Flux Kustomization — the healer would sit in git "
      f"and never run, which is the 'looks applied and isn't' failure it exists to end")

for consumer_file, consumer_owners in ((IMPORT_FILE, import_owners),
                                       (RECON_FILE, recon_owners)):
    for co in consumer_owners:
        for so in secret_owners:
            if so == co:
                continue          # same Kustomization: one atomic apply, ordering moot
            anc = ancestors(co)
            check(so in anc,
                  f"ORDERING IS ONLY DOCUMENTED: '{co}' renders {consumer_file} but does "
                  f"not (transitively) dependsOn '{so}', which renders the "
                  f"{SECRET_NAME} Secret. Without that edge the consumer can reconcile "
                  f"before the secret exists, and the Keycloak Operator's response to a "
                  f"missing placeholder is to store it LITERALLY and report success")
            if so in anc:
                check(kustomizations[so]["wait"],
                      f"'{so}' (which renders the {SECRET_NAME} Secret) does not set "
                      f"wait: true, so '{co}' can proceed before it is applied")

# ------ both environments must reconcile the reconciler, not just dev.
for env in ("dev", "prod"):
    entry = f"clusters/{env}"
    if not (ROOT / entry).is_dir():
        continue
    applied, _ = st.reconciled_files(ROOT, entry)
    check(RECON_FILE in applied,
          f"{RECON_FILE} is not reachable from the {entry} entrypoint — the healer would "
          f"exist on one cluster only, and prod is where all 12 clients were broken")
    check(IMPORT_FILE in applied, f"{IMPORT_FILE} is not reachable from {entry}")

# ============================================ (5) the reconciler manifest is safe + wired
recon_docs = [d for d in load(RECON_FILE) if d]
cm = next((d for d in recon_docs if d.get("kind") == "ConfigMap"), None)
cj = next((d for d in recon_docs if d.get("kind") == "CronJob"), None)
check(cm is not None, f"{RECON_FILE}: no ConfigMap carrying the reconciler script")
check(cj is not None, f"{RECON_FILE}: no CronJob running the reconciler")

script = ""
if cm is not None:
    data = cm.get("data") or {}
    check(len(data) == 1,
          f"{RECON_FILE}: the reconciler ConfigMap must carry exactly one script key "
          f"(found {sorted(data)}) — a second copy would be untested by this gate")
    script = next(iter(data.values()), "")
    check(len(script.splitlines()) > 50,
          f"{RECON_FILE}: the extracted script is implausibly short "
          f"({len(script.splitlines())} lines) — the ConfigMap shape changed and the "
          f"selftest below would prove nothing")
    # THE nextcloud TRAP. This file is rendered by a Flux Kustomization with
    # postBuild.substituteFrom, so envsubst rewrites it before any container runs. The
    # script is written with NO dollar sign at all, so the hazard is structural rather
    # than escaped and no future edit has to reason about escaping.
    check(DOLLAR not in script,
          f"{RECON_FILE}: the reconciler script contains a dollar sign. It ships inside a "
          f"SUBSTITUTED Flux path, where a dollar-brace expansion is rewritten at "
          f"reconcile time (see apps/wave-1/nextcloud.yaml). Rewrite without it — Python "
          f"never needs one; chr(36) spells it where a literal is genuinely required")

if cj is not None:
    pod = cj["spec"]["jobTemplate"]["spec"]["template"]["spec"]
    ctr = pod["containers"][0]
    vols = {v["name"]: v for v in pod.get("volumes", [])}
    mounts = {m["name"]: m for m in ctr.get("volumeMounts", [])}
    secret_vols = [v for v in vols.values()
                   if (v.get("secret") or {}).get("secretName") == SECRET_NAME]
    check(len(secret_vols) == 1,
          f"{RECON_FILE}: the CronJob must mount the {SECRET_NAME} Secret — that mount IS "
          f"the desired state it reconciles towards")
    if secret_vols:
        mp = mounts.get(secret_vols[0]["name"], {}).get("mountPath")
        check(mp == SECRETS_MOUNT,
              f"{RECON_FILE}: {SECRET_NAME} is mounted at {mp}, but the script defaults "
              f"to {SECRETS_MOUNT} and no env var overrides it")
    cm_vols = [v for v in vols.values()
               if (v.get("configMap") or {}).get("name") == (cm or {}).get(
                   "metadata", {}).get("name")]
    check(len(cm_vols) == 1,
          f"{RECON_FILE}: the CronJob does not mount the reconciler ConfigMap, so it "
          f"would run nothing")
    # The URL desired state must be MOUNTED at the path the script defaults to, for the
    # same reason as the Secret: the pod holds no ServiceAccount token and cannot read
    # the KeycloakRealmImport CR, so an unmounted ConfigMap means pass 2 fails closed on
    # every run forever -- correct behaviour, but a permanently red reconciler.
    url_vols = [v for v in vols.values()
                if (v.get("configMap") or {}).get("name") == URLS_CM_NAME]
    check(len(url_vols) == 1,
          f"{RECON_FILE}: the CronJob must mount the {URLS_CM_NAME} ConfigMap — that "
          f"mount IS the desired state the URL pass reconciles towards, and the pod has "
          f"no Kubernetes API access to read the realm import instead")
    if url_vols:
        mp = mounts.get(url_vols[0]["name"], {}).get("mountPath")
        check(mp == URLS_MOUNT,
              f"{RECON_FILE}: {URLS_CM_NAME} is mounted at {mp}, but the script defaults "
              f"to {URLS_MOUNT}/{URLS_DATA_KEY} and no env var overrides it")
        check(mounts.get(url_vols[0]["name"], {}).get("readOnly") is True,
              f"{RECON_FILE}: the {URLS_CM_NAME} mount must be readOnly — the reconciler "
              f"reads its desired state, it never writes it")
    env_srcs = {e["name"]: e for e in ctr.get("env", [])}
    for var in ("KEYCLOAK_ADMIN_USER", "KEYCLOAK_ADMIN_PASSWORD"):
        ref = (env_srcs.get(var, {}).get("valueFrom") or {}).get("secretKeyRef") or {}
        check(ref.get("name") == ADMIN_SECRET,
              f"{RECON_FILE}: {var} must come from the {ADMIN_SECRET} Secret "
              f"(got {ref.get('name')!r}) — the admin credential is never inlined")
    for var in ("KEYCLOAK_BASE_URL", "KEYCLOAK_REALM"):
        check(env_srcs.get(var, {}).get("value"),
              f"{RECON_FILE}: {var} is not set; the script fails closed without it, but "
              f"the CronJob should not ship broken")
    # R-SEC-02: it talks only to Keycloak's REST API and needs no Kubernetes API rights.
    check(pod.get("automountServiceAccountToken") is False,
          f"{RECON_FILE}: the reconciler needs NO Kubernetes API access (its inputs are "
          f"mounted Secrets) — set automountServiceAccountToken: false")
    check(cj["spec"].get("concurrencyPolicy") == "Forbid",
          f"{RECON_FILE}: set concurrencyPolicy: Forbid so a run stalled on an "
          f"unavailable Keycloak is not lapped by the next tick")
    img = ctr.get("image", "")
    check(re.search(r":[0-9]", img),
          f"{RECON_FILE}: container image {img!r} is not pinned to an explicit version")

# ================================================ (8) the URL desired state is DERIVED
# The reconciler's second pass makes the realm's environment-derived client URLs match
# platform/keycloak/client-urls.yaml. That file must not be an independent declaration of
# every public hostname: two declarations that can disagree is the same shape as the bug
# this whole change closes. It is GENERATED from the realm import, and the generation is
# gated HERE -- a hand edit to either file that makes them disagree is unmergeable.
urls_docs = [d for d in load(URLS_FILE) if d]
urls_cm = next((d for d in urls_docs if d.get("kind") == "ConfigMap"), None)
check(urls_cm is not None, f"{URLS_FILE}: no ConfigMap document found")
check((urls_cm or {}).get("metadata", {}).get("name") == URLS_CM_NAME,
      f"{URLS_FILE}: the ConfigMap must be named {URLS_CM_NAME} (the CronJob mounts it "
      f"by that name)")

urls_want: dict[str, dict] = {}
if urls_cm is not None:
    data = urls_cm.get("data") or {}
    check(set(data) == {URLS_DATA_KEY},
          f"{URLS_FILE}: the ConfigMap must carry exactly the key {URLS_DATA_KEY} "
          f"(found {sorted(data)}) — the script reads that filename from its mount")
    raw = data.get(URLS_DATA_KEY, "")
    try:
        urls_want = json.loads(raw)
    except ValueError as e:
        check(False, f"{URLS_FILE}: {URLS_DATA_KEY} is not valid JSON ({e}) — the "
                     f"reconciler would fail closed on every run")
    check(isinstance(urls_want, dict) and bool(urls_want),
          f"{URLS_FILE}: {URLS_DATA_KEY} must be a non-empty JSON object of clients")

# THE GATE: the committed file must be exactly what the renderer produces from the realm
# import. This is what makes the realm import the single source of truth for hostnames
# rather than merely the place they were first typed.
if (ROOT / RENDERER).is_file():
    proc = subprocess.run([sys.executable, str(ROOT / RENDERER), "--check"],
                          capture_output=True, text=True, timeout=120, cwd=str(ROOT))
    check(proc.returncode == 0,
          f"{URLS_FILE} is NOT what {RENDERER} renders from {IMPORT_FILE}. The URL "
          f"reconciler would converge the live realm on a desired state the realm import "
          f"does not declare — two sources of truth for every public hostname, which is "
          f"the failure class this change exists to close. Regenerate with: "
          f"python3 {RENDERER} --write\n        "
          + "\n        ".join((proc.stdout + proc.stderr).strip().splitlines()[:12]))
else:
    check(False, f"{RENDERER} is missing — {URLS_FILE} would become a hand-maintained "
                 f"second declaration of every hostname with nothing keeping it honest")

# Every client named in the desired state must exist in the realm import (otherwise every
# reconciler run reports NOT FOUND forever), and every field must be one the reconciler is
# actually allowed to write.
import_client_ids = {c.get("clientId") for c in clients}
for cid in sorted(urls_want):
    check(cid in import_client_ids,
          f"{URLS_FILE}: declares URLs for client '{cid}', which {IMPORT_FILE} does not "
          f"declare. The reconciler would look it up, not find it, and fail every run")
    entry = urls_want[cid] if isinstance(urls_want.get(cid), dict) else {}
    check(bool(entry), f"{URLS_FILE}: client '{cid}' declares no URL fields")
    for field in sorted(entry):
        check(field in URL_FIELDS,
              f"{URLS_FILE}: client '{cid}' declares field '{field}', which is not one of "
              f"the writable URL fields ({', '.join(URL_FIELDS)}). Anything else would be "
              f"a client-representation field written into the realm by a file edit")

# The three places that name the writable field set must agree, or the gate is checking a
# different contract than the one that ships.
for field in URL_FIELDS:
    check(field in script,
          f"{RECON_FILE}: the shipped script never mentions the writable URL field "
          f"'{field}', but this check and {RENDERER} both treat it as writable — the "
          f"three definitions of the write surface have diverged")

# A redirect URI is only as good as the hostname it points at. If the realm import uses a
# Flux substitution token, the desired-state ConfigMap must carry the SAME token (not a
# resolved value) and must be rendered by the SAME Kustomization, or the two are
# substituted differently and disagree about the domain at reconcile time.
# Scan the DATA PAYLOAD, not the whole file: both files carry header prose that discusses
# substitution tokens by name, and matching those would be a false positive.
TOKEN_RE = re.compile(re.escape(DOLLAR) + r"\{([A-Z_][A-Z0-9_]*)\}")
import_tokens = set(TOKEN_RE.findall(
    yaml.safe_dump((realm_import or {}).get("spec", {}).get("realm", {}))))
urls_tokens = set(TOKEN_RE.findall(
    ((urls_cm or {}).get("data") or {}).get(URLS_DATA_KEY, "")))
check(urls_tokens <= import_tokens,
      f"{URLS_FILE} uses substitution token(s) {sorted(urls_tokens - import_tokens)} that "
      f"the realm definition in {IMPORT_FILE} does not — the desired state would resolve "
      f"to a hostname the realm import never declared")

urls_owners = owners(URLS_FILE)
check(bool(urls_owners),
      f"{URLS_FILE} is rendered by NO Flux Kustomization — the reconciler would mount a "
      f"ConfigMap that does not exist and fail closed on every run")
check(set(urls_owners) == set(import_owners),
      f"{URLS_FILE} is rendered by {urls_owners} but {IMPORT_FILE} by {import_owners}. "
      f"They MUST be rendered by the same Flux Kustomization: postBuild.substituteFrom is "
      f"per-Kustomization, so a split would let the same "
      f"{DOLLAR}{{BASE_DOMAIN}} resolve to two different values — the realm converging on "
      f"a hostname the HTTPRoutes do not serve")
for env in ("dev", "prod"):
    entry = f"clusters/{env}"
    if not (ROOT / entry).is_dir():
        continue
    applied, _ = st.reconciled_files(ROOT, entry)
    check(URLS_FILE in applied,
          f"{URLS_FILE} is not reachable from the {entry} entrypoint — the URL pass would "
          f"fail closed on that cluster forever")

# ============================== (6) the reconciler's BEHAVIOUR, proven by its own selftest
if script:
    with tempfile.TemporaryDirectory() as td:
        p = Path(td) / "reconcile.py"
        p.write_text(script)
        proc = subprocess.run([sys.executable, str(p), "--selftest"],
                              capture_output=True, text=True, timeout=180)
        tail = (proc.stdout + proc.stderr).strip().splitlines()
        check(proc.returncode == 0,
              "the reconciler's --selftest FAILED (compare-then-write / idempotence / "
              "no-secret-logging / endpoint allow-list / fail-closed):\n        "
              + "\n        ".join(tail[-12:]))
        if proc.returncode == 0:
            print("  reconciler --selftest: " + (tail[-1] if tail else "(no output)"))

        # ---- (6b) THE SELFTEST IS FALSIFIABLE, not decorative -----------------------
        # A passing suite proves nothing unless it can be shown to FAIL. mutations.py
        # injects plausible regressions one at a time -- order-sensitive comparison, a
        # fat round-tripped PUT, additive-only URI sets, a widened allow-list, coupled
        # passes, fail-open on an unreadable input -- and requires --selftest to catch
        # every one. A mutation that SURVIVES means the suite does not check that
        # property, and this check fails naming it.
        #
        # This matters more than usual here: the whole incident was a component that
        # REPORTED SUCCESS while the realm was broken. A green test that cannot fail is
        # the same lie one level up. ~20s, no network, no cluster.
        mut = DIR_PATH / "mutations.py"
        if mut.is_file():
            mproc = subprocess.run([sys.executable, str(mut), str(p)],
                                   capture_output=True, text=True, timeout=600)
            mtail = (mproc.stdout + mproc.stderr).strip().splitlines()
            check(mproc.returncode == 0,
                  "the reconciler's --selftest is NOT FALSIFIABLE — at least one injected "
                  "regression survived it, so the suite does not prove what it claims:\n"
                  "        " + "\n        ".join(mtail[-14:]))
            if mproc.returncode == 0:
                print("  reconciler mutations: "
                      + (mtail[-1] if mtail else "(no output)"))
        else:
            check(False,
                  "conformance/tests/l0-static/keycloak-client-secret-reconciler/"
                  "mutations.py is missing — nothing proves the --selftest can fail, and "
                  "an unfalsifiable proof is the failure mode this component exists to end")

# ======================================================= (7) this check is load-bearing
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
TOUCHED = ((ROOT / "ci" / "touched-requirements.sh").read_text()
           if (ROOT / "ci" / "touched-requirements.sh").is_file() else "")
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism.
# It used to assert `"keycloak-client-secret-reconciler/check.sh" in L0`: ci/l0.sh carried a hand-written
# "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone
# (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED
# to write it). The same property is now asserted two ways: this test DECLARES
# `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into
# the run via `--include-scope`. Neither half alone is enough, so both are checked here; the
# mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than
# passing) is proven end-to-end and falsifiably by
# conformance/tests/l0-static/whole-repo-invariants-wired.
SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "keycloak-client-secret-reconciler" / "chainsaw-test.yaml"
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None,
      "keycloak-client-secret-reconciler/chainsaw-test.yaml does not declare "
      'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only '
      "when a change happens to touch one of the requirements it is labelled with. "
      "This invariant spans platform/, platform-config/ and clusters/, so a change that breaks it need not attribute R-IDP-03/04 — which is how 12 unusable prod OIDC clients sat behind an entirely green cluster on 2026-09-06")
check("--include-scope" in L0,
      "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is "
      "a purely requirement-scoped run again and the scope annotation is inert decoration — "
      "this check would then run only by coincidence")
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
      "ci/gate.sh does not set the merge block when the l0 stage fails")
check(re.search(r"^\s*platform/keycloak/\*\)", TOUCHED, re.M),
      "ci/touched-requirements.sh has no platform/keycloak/* entry in its path->requirement "
      "map, so editing the realm import or this reconciler attributes NO requirement and "
      "the L2 keycloak tests are filtered out of the merge gate's touched subset")
l2 = ROOT / "conformance/tests/l2-behavioral/keycloak-client-secrets-match-git/chainsaw-test.yaml"
check(l2.is_file(),
      "the LIVE-realm counterpart (conformance/tests/l2-behavioral/"
      "keycloak-client-secrets-match-git) is missing — this static check cannot see a "
      "realm that drifted after apply, which is the whole failure mode")
if l2.is_file():
    l2_text = l2.read_text()
    check("KEYCLOAK_CLIENT_SECRETS_AUDIT_OK" in l2_text,
          "the L2 test does not assert the reconciler's secret audit sentinel, so it could "
          "pass on a Job that never ran the audit")
    check("KEYCLOAK_CLIENT_URLS_AUDIT_OK" in l2_text,
          "the L2 test does not assert the reconciler's URL audit sentinel. The live realm "
          "could hold pre-migration redirect URIs and the test would still pass — which is "
          "exactly the state prod was measured in")
    check(URLS_CM_NAME in l2_text,
          f"the L2 audit Job does not mount the {URLS_CM_NAME} ConfigMap, so its URL pass "
          f"would fail closed and the test would report a false failure")

# The alert pair must cover the reconciler's silence. Extended, not duplicated: one CronJob
# gets one Failing/Stale pair, because two pairs keyed to the same job_name regex would
# both fire for a single cause.
ALERTS = "platform/monitoring/alerts.yaml"
alerts_text = (ROOT / ALERTS).read_text() if (ROOT / ALERTS).is_file() else ""
for rule in ("KeycloakClientSecretReconcilerFailing", "KeycloakClientSecretReconcilerStale"):
    check(rule in alerts_text,
          f"{ALERTS} has no {rule} rule — the reconciler's silence would be unobservable, "
          f"and a healer nobody notices is absent is the 'looks applied and isn't' failure")
check("redirect" in alerts_text.lower(),
      f"{ALERTS}: the reconciler alert descriptions never mention redirect URIs, so an "
      f"operator paged at 3am would look only for a client-secret problem while the actual "
      f"drift was a hostname")

# The renderer must be a real, runnable, tracked file — not a documented intention.
check((ROOT / RENDERER).is_file(),
      f"{RENDERER} is missing; without it {URLS_FILE} cannot be regenerated and the "
      f"single-source-of-truth gate above has nothing to compare against")

# ------------------------------------------------------------------------------ report
print(f"realm import: {len(clients)} client(s), {len(declared_pairs)} confidential with a "
      f"declared secret, {len(placeholders)} placeholder(s)")
print(f"{SECRET_NAME} Secret: {len(sops_keys)} key(s) in git (names only; values stay "
      f"SOPS-encrypted)")
print("flux ordering: " + ", ".join(
    f"{f.split('/')[-1]} <- {owners(f)}" for f in (SOPS_FILE, IMPORT_FILE, RECON_FILE)))

print()
if not FAILURES:
    print(f"check OK: keycloak-client-secret-reconciler — {CHECKS} assertions "
          f"(R-IDP-03, R-IDP-04)")
    raise SystemExit(0)
print(f"CHECK FAIL (keycloak-client-secret-reconciler): {len(FAILURES)} failed, "
      f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
for f in FAILURES:
    print(f"  FAIL {f}", file=sys.stderr)
raise SystemExit(1)
PY