# ============================================================================= # L2 BEHAVIORAL — every OIDC client's STORED secret AND redirect URIs equal what git # declares (R-IDP-02, R-IDP-04) [RUNNABLE] # Spec: docs/15 identity-and-access; ADR-0014. # Impl: platform/keycloak/client-secret-reconciler.yaml (the CronJob + its script). # # WHY THIS TEST EXISTS # -------------------- # The KeycloakRealmImport resolves each confidential client's secret from a `(env:NAME)` # placeholder mapped to a key of the SOPS-encrypted keycloak-realm-clients Secret. When # the Keycloak Operator cannot resolve a placeholder it stores the LITERAL placeholder # text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS; the CR then rests # at Done=True / Started=False and never re-runs. Measured 2026-09-06: prod had all 12 # clients holding the literal placeholder — every OIDC login on the cluster failed with # "Invalid client or Invalid client credentials" — and dev had one (oauth2-proxy), whose # Secret key was added after dev's import had already completed. # # Every SIGNAL the cluster offered was green: the import Job succeeded, the CR read # Done=True, the Keycloak pods were healthy, and the existing keycloak-config-as-code # test (which asserts Done=True) PASSED throughout. The only observable was a human # failing to log in. This test is that observable, automated: it compares STORED against # DECLARED for every client and fails on any difference, so the class cannot sit # undetected for months again. # # GIVEN the `homelab` realm and the SOPS-encrypted keycloak-realm-clients Secret, # WHEN the reconciler's script is run in --audit mode (READ-ONLY: it cannot write, and # its endpoint allow-list makes users/groups/roles unreachable), # -> THEN every declared client's stored secret matches git, and none holds an # unresolved import placeholder. Emits KEYCLOAK_CLIENT_SECRETS_AUDIT_OK. # -> AND every declared client URL — redirectUris, webOrigins, rootUrl, baseUrl, # adminUrl — matches git. Emits KEYCLOAK_CLIENT_URLS_AUDIT_OK. # # THE URL HALF EXISTS FOR THE SAME REASON, ONE FIELD OVER. The realm import is templated # per environment, so moving prod's BASE_DOMAIN re-renders every declared redirect URI in # git — and reaches the realm never, because a completed import does not re-run. Measured # on prod: stored redirect URIs still on the PRE-migration hostname while git declared the # new one. Every OIDC login would fail with a redirect_uri mismatch, and once again every # signal would be green: the CR reads Done=True, the pods are healthy, and the # secret-only version of this very test would have PASSED. Asserting only the secret # sentinel is what would have let that ship. # # WHY IT RUNS THE SHIPPED SCRIPT rather than re-implementing the comparison: a detector # that duplicates the healer's logic drifts from it. This audits with the same code, in # the same mode the L0 gate selftests, so a passing audit is evidence about the healer # too. It runs as a Job IN the keycloak namespace (not in the ephemeral chainsaw # namespace) because the desired state is the mounted keycloak-realm-clients Secret plus # the keycloak-client-urls ConfigMap, and the admin credential is # keycloak-admin-credentials — all namespace-local, which is also why neither this Job nor # the CronJob needs any Kubernetes API access. # # chainsaw v0.2.12: Job + status assert; the leftover-Job delete keeps a re-run from # colliding with an immutable Job left by a previous failure. # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: name: keycloak-client-secrets-match-git labels: conformance.home.k3s/R-IDP-02: "true" conformance.home.k3s/R-IDP-04: "true" annotations: conformance.home.k3s/requirements: "R-IDP-02, R-IDP-04" conformance.home.k3s/layer: "L2" conformance.home.k3s/safety-critical: "false" conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- Every OIDC client's stored secret equals the SOPS-declared one and none holds an unresolved import placeholder — the observable that was missing when all 12 prod clients were broken behind a green realm import. spec: timeouts: apply: 60s assert: 5m exec: 5m steps: - name: given-the-reconciler-is-reconciled-onto-the-cluster description: >- The audit Job runs the CronJob's own script from its ConfigMap; if Flux has not applied the reconciler, there is nothing to audit WITH and that is the failure. try: - assert: resource: apiVersion: v1 kind: ConfigMap metadata: name: keycloak-client-secret-reconciler namespace: keycloak - assert: resource: apiVersion: batch/v1 kind: CronJob metadata: name: keycloak-client-secret-reconciler namespace: keycloak spec: concurrencyPolicy: Forbid - name: then-every-stored-client-secret-matches-git try: - script: content: | kubectl -n keycloak delete job keycloak-client-secret-audit \ --ignore-not-found --wait=true - apply: resource: apiVersion: batch/v1 kind: Job metadata: name: keycloak-client-secret-audit namespace: keycloak labels: app.kubernetes.io/name: keycloak-client-secret-audit app.kubernetes.io/part-of: conformance spec: backoffLimit: 0 activeDeadlineSeconds: 240 template: metadata: labels: app.kubernetes.io/name: keycloak-client-secret-audit spec: restartPolicy: Never automountServiceAccountToken: false securityContext: runAsNonRoot: true runAsUser: 65532 runAsGroup: 65532 seccompProfile: type: RuntimeDefault containers: - name: audit image: python:3.13.15-alpine3.23 command: - python3 - /opt/reconciler/reconcile-keycloak-clients.py # READ-ONLY. --audit never issues a write on any path; the # script's --selftest proves that (L0 gate). - --audit env: - name: KEYCLOAK_BASE_URL value: http://keycloak-service.keycloak.svc.cluster.local:8080 - name: KEYCLOAK_REALM value: homelab - name: PYTHONDONTWRITEBYTECODE value: "1" - name: PYTHONUNBUFFERED value: "1" - name: KEYCLOAK_ADMIN_USER valueFrom: secretKeyRef: name: keycloak-admin-credentials key: admin-user - name: KEYCLOAK_ADMIN_PASSWORD valueFrom: secretKeyRef: name: keycloak-admin-credentials key: admin-password volumeMounts: - name: script mountPath: /opt/reconciler readOnly: true - name: client-secrets mountPath: /etc/keycloak-realm-clients readOnly: true # The URL pass's desired state. Without this mount the audit # fails closed on pass 2 and this test would report a false # failure rather than auditing the realm's redirect URIs. - name: client-urls mountPath: /etc/keycloak-client-urls readOnly: true securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: drop: - ALL resources: requests: cpu: 10m memory: 48Mi limits: memory: 128Mi volumes: - name: script configMap: name: keycloak-client-secret-reconciler defaultMode: 0444 - name: client-secrets secret: secretName: keycloak-realm-clients defaultMode: 0400 - name: client-urls configMap: name: keycloak-client-urls defaultMode: 0444 - assert: resource: apiVersion: batch/v1 kind: Job metadata: name: keycloak-client-secret-audit namespace: keycloak status: succeeded: 1 # The Job's exit status already encodes the verdict, but the sentinel is what the # run LEDGER records (conformance/runs/*/run.log): a JUnit alone cannot tell a real # audit from a Job that succeeded without auditing anything. - script: content: | set -u out="$(kubectl -n keycloak logs job/keycloak-client-secret-audit \ --tail=-1 2>&1 || true)" echo "$out" echo "$out" | grep -q 'KEYCLOAK_CLIENT_SECRETS_AUDIT_OK' \ || { echo "FAIL: the audit did not report every client secret matching git"; exit 1; } echo "$out" | grep -q 'UNRESOLVED IMPORT PLACEHOLDER' \ && { echo "FAIL: a client still holds the literal import placeholder"; exit 1; } # The URL half. A realm still holding PRE-migration redirect URIs is the # state prod was measured in: every OIDC login fails with a redirect_uri # mismatch while the KeycloakRealmImport CR still reads Done=True. Asserting # only the secret sentinel would pass on exactly that realm. echo "$out" | grep -q 'KEYCLOAK_CLIENT_URLS_AUDIT_OK' \ || { echo "FAIL: the audit did not report every client URL matching git"; exit 1; } echo "$out" | grep -q 'DRIFT -- realm has' \ && { echo "FAIL: a client URL still differs from the declared one"; exit 1; } echo KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK check: (contains($stdout, 'KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK')): true catch: - describe: apiVersion: batch/v1 kind: Job namespace: keycloak name: keycloak-client-secret-audit - podLogs: namespace: keycloak selector: app.kubernetes.io/name=keycloak-client-secret-audit tail: 60 finally: - script: content: | kubectl -n keycloak delete job keycloak-client-secret-audit \ --ignore-not-found --wait=false