keycloak: coverage tests + dashboard + reword cross-cutting tags

conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml

@@ -0,0 +1,70 @@
+# =============================================================================
+# L2 BEHAVIORAL — identity config as code  (R-IDP-03, R-IDP-04, R-IDP-06)   [RUNNABLE]
+# Spec: docs/15 identity-and-access; ADR-0014. Impl: platform/keycloak/keycloak-realm-import.yaml.
+#
+# GIVEN the `homelab` realm (groups platform-admins/media-users/viewers, OIDC clients, brute-force,
+#       and the conditional MFA browser flow) is declared as a KeycloakRealmImport in git,
+# WHEN  the Keycloak Operator reconciles it
+#   ->  THEN the import reaches Done=True — realm/groups/clients/roles + the MFA flow are applied
+#       PURELY from git (R-IDP-03 config-as-code; R-IDP-04 MFA flow bound; R-IDP-06 groups/roles for
+#       least-privilege). No console clicking.
+#
+# Goes GREEN once Keycloak is up + the realm client-secret hand-off (keycloak-realm-clients) is filled
+# so the placeholder resolves and the import completes. chainsaw v0.2.12: script + block contains check.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: keycloak-config-as-code
+  labels:
+    conformance.home.k3s/R-IDP-03: "true"
+    conformance.home.k3s/R-IDP-04: "true"
+    conformance.home.k3s/R-IDP-06: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04, R-IDP-06"
+    conformance.home.k3s/layer: "L2"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      The homelab realm (groups, clients, MFA flow) reconciles to Done purely from the git-managed
+      KeycloakRealmImport — no manual console configuration.
+spec:
+  timeouts:
+    exec: 5m
+  steps:
+    - name: realm-import-reconciles-from-git
+      try:
+        - script:
+            content: |
+              set -u
+              # Poll the KeycloakRealmImport until the operator reports Done=True.
+              for i in $(seq 1 30); do
+                st="$(kubectl -n keycloak get keycloakrealmimport homelab \
+                        -o jsonpath='{.status.conditions[?(@.type=="Done")].status}' 2>/dev/null || true)"
+                echo "poll $i: Done=$st"
+                if [ "$st" = "True" ]; then
+                  echo "REALM_IMPORT_DONE_OK"
+                  break
+                fi
+                sleep 10
+              done
+            check:
+              (contains($stdout, 'REALM_IMPORT_DONE_OK')): true
+    - name: privileged-group-and-mfa-flow-present
+      try:
+        - script:
+            content: |
+              set -u
+              # The realm import declares the browser-mfa flow + platform-admins group. Confirm the
+              # realm CR / import spec carries them (declared-in-git evidence; the live realm state is
+              # exercised by the login flows).
+              flow="$(kubectl -n keycloak get keycloakrealmimport homelab \
+                        -o jsonpath='{.spec.realm.browserFlow}' 2>/dev/null || true)"
+              grp="$(kubectl -n keycloak get keycloakrealmimport homelab \
+                        -o jsonpath='{.spec.realm.groups[*].name}' 2>/dev/null || true)"
+              echo "browserFlow=$flow groups=$grp"
+              echo "$flow" | grep -q 'browser-mfa' || { echo "FAIL: MFA browser flow not bound"; exit 1; }
+              echo "$grp" | grep -q 'platform-admins' || { echo "FAIL: platform-admins group missing"; exit 1; }
+              echo "MFA_AND_GROUPS_OK"
+            check:
+              (contains($stdout, 'MFA_AND_GROUPS_OK')): true

conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml

@@ -0,0 +1,49 @@
+# =============================================================================
+# L2 BEHAVIORAL — IdP health is visible on a dashboard  (R-OBS-03)      [RUNNABLE]
+# Spec: docs/10 observability. Impl: platform/keycloak/grafana-dashboard.yaml (a grafana_dashboard
+# ConfigMap auto-provisioned into Grafana by the sidecar).
+#
+# GIVEN a Keycloak health dashboard is declared as a grafana_dashboard ConfigMap,
+# WHEN  Grafana's sidecar discovers it (searchNamespace: ALL)
+#   ->  THEN the dashboard is provisioned and Grafana is up — Keycloak health is visible on a dashboard
+#       (R-OBS-03). The presence of the labelled ConfigMap + a ready Grafana is the reconciliation proof.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: keycloak-dashboard
+  labels:
+    conformance.home.k3s/R-OBS-03: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-OBS-03"
+    conformance.home.k3s/layer: "L2"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      A grafana_dashboard ConfigMap for Keycloak health exists and Grafana is up to render it.
+spec:
+  timeouts:
+    assert: 3m
+  steps:
+    - name: keycloak-dashboard-configmap-provisioned
+      try:
+        - assert:
+            resource:
+              apiVersion: v1
+              kind: ConfigMap
+              metadata:
+                name: keycloak-dashboard
+                namespace: keycloak
+                labels:
+                  grafana_dashboard: "1"
+    - name: grafana-is-up-to-render-it
+      try:
+        - assert:
+            resource:
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: kube-prometheus-stack-grafana
+                namespace: monitoring
+              status:
+                availableReplicas: 1

conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml

@@ -0,0 +1,58 @@
+# =============================================================================
+# L2 BEHAVIORAL — IdP datastore is DR-critical  (R-IDP-05)              [RUNNABLE]
+# Spec: docs/15 identity-and-access; ADR-0014. Impl: platform/keycloak/postgres-cluster.yaml
+# (CloudNativePG Cluster on the 3-replica `longhorn` StorageClass).
+#
+# GIVEN Keycloak's PostgreSQL runs as a CloudNativePG Cluster,
+# WHEN  we inspect it
+#   ->  THEN it is healthy (>=1 ready instance) AND its data lives on the replicated `longhorn`
+#       StorageClass — i.e. the identity datastore is DR-critical / replicated (R-IDP-05). Keycloak is
+#       stateless and rebuilds from this DB + the git realm import.
+# (Full off-cluster backup/restore is R-STATE-03/Velero territory, hand-off OQ-BKP.)
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: keycloak-db-dr
+  labels:
+    conformance.home.k3s/R-IDP-05: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-IDP-05"
+    conformance.home.k3s/layer: "L2"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      Keycloak's CloudNativePG database is healthy and stored on the 3-replica longhorn StorageClass.
+spec:
+  timeouts:
+    exec: 5m
+  steps:
+    - name: db-cluster-is-healthy
+      try:
+        - script:
+            content: |
+              set -u
+              for i in $(seq 1 30); do
+                ready="$(kubectl -n keycloak get clusters.postgresql.cnpg.io keycloak-db \
+                          -o jsonpath='{.status.readyInstances}' 2>/dev/null || true)"
+                echo "poll $i: readyInstances=$ready"
+                if [ "${ready:-0}" -ge 1 ]; then echo DB_READY_OK; break; fi
+                sleep 10
+              done
+            check:
+              (contains($stdout, 'DB_READY_OK')): true
+    - name: db-data-is-on-replicated-longhorn
+      try:
+        - script:
+            content: |
+              set -u
+              sc="$(kubectl -n keycloak get pvc -l cnpg.io/cluster=keycloak-db \
+                      -o jsonpath='{.items[0].spec.storageClassName}' 2>/dev/null || true)"
+              echo "db pvc storageClass=$sc"
+              if [ "$sc" = "longhorn" ]; then
+                echo DB_ON_LONGHORN_OK
+              else
+                echo "FAIL: Keycloak DB PVC not on the replicated longhorn StorageClass"; exit 1
+              fi
+            check:
+              (contains($stdout, 'DB_ON_LONGHORN_OK')): true

conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml

@@ -0,0 +1,81 @@
+# =============================================================================
+# L2 BEHAVIORAL — the IdP is monitored  (R-OBS-01)                      [RUNNABLE]
+# Spec: docs/10 observability. Impl: platform/keycloak/servicemonitor.yaml (Keycloak ServiceMonitor,
+# carried via platform-config per the F-MON1 split) + the CNPG PodMonitor.
+#
+# GIVEN a ServiceMonitor for Keycloak,
+# WHEN  Prometheus reconciles its scrape config
+#   ->  THEN Prometheus carries a `serviceMonitor/keycloak` scrape pool in its active targets — the IdP
+#       is monitored (metrics/health), R-OBS-01. (The KeycloakDown / KeycloakDatabaseDown alerts in
+#       platform/monitoring/alerts.yaml consume this signal + kube-state-metrics.)
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: keycloak-monitored
+  labels:
+    conformance.home.k3s/R-OBS-01: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-OBS-01"
+    conformance.home.k3s/layer: "L2"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      Prometheus discovers the Keycloak ServiceMonitor and carries it as an active scrape pool.
+spec:
+  timeouts:
+    apply: 60s
+    assert: 3m
+    exec: 3m
+  steps:
+    - name: keycloak-servicemonitor-exists
+      try:
+        - assert:
+            resource:
+              apiVersion: monitoring.coreos.com/v1
+              kind: ServiceMonitor
+              metadata:
+                name: keycloak
+                namespace: keycloak
+    - name: prometheus-carries-the-keycloak-scrape-pool
+      try:
+        - apply:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: keycloak-target-probe
+              spec:
+                backoffLimit: 3
+                activeDeadlineSeconds: 150
+                template:
+                  spec:
+                    restartPolicy: Never
+                    containers:
+                      - name: probe
+                        image: busybox:1.36.1
+                        command:
+                          - sh
+                          - -c
+                          - |
+                            set -u
+                            prom="http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090"
+                            for i in $(seq 1 12); do
+                              out="$(wget -q -O - "$prom/api/v1/targets" 2>/dev/null || true)"
+                              if echo "$out" | grep -q 'serviceMonitor/keycloak'; then
+                                echo KEYCLOAK_TARGET_PRESENT_OK; exit 0
+                              fi
+                              echo "poll $i: keycloak scrape pool not yet present"; sleep 10
+                            done
+                            echo "FAIL: Prometheus has no serviceMonitor/keycloak scrape pool"; exit 1
+        - assert:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: keycloak-target-probe
+              status:
+                succeeded: 1
+      finally:
+        - script:
+            content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"'

conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml

@@ -0,0 +1,85 @@
+# =============================================================================
+# L2 BEHAVIORAL — native-OIDC SSO via the central IdP  (R-IDP-01)       [RUNNABLE]
+# Spec: docs/15 identity-and-access; ADR-0014. Impl: Grafana as a Keycloak OIDC client
+# (platform/monitoring/kube-prometheus-stack.yaml auth.generic_oauth) — the self-contained SSO proof.
+#
+# GIVEN Grafana is deployed and configured as an OIDC client of the Keycloak `homelab` realm,
+# WHEN  an unauthenticated browser starts the OIDC login (GET /login/generic_oauth)
+#   ->  THEN Grafana redirects (302) to the CENTRAL IdP's authorize endpoint on
+#       keycloak.home.keithjackson.dev/realms/homelab — proving SSO is driven by the one IdP
+#       (R-IDP-01: central Identity Provider as the single user directory).
+#
+# Goes GREEN once the grafana-oidc client-secret hand-off is filled and Keycloak + Grafana are up.
+# chainsaw v0.2.12: Job + status assert; no test-level finally.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: oidc-sso-grafana
+  labels:
+    conformance.home.k3s/R-IDP-01: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-IDP-01"
+    conformance.home.k3s/layer: "L2"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      Grafana's OIDC login redirects to the central Keycloak realm — SSO is driven by the one IdP.
+spec:
+  timeouts:
+    apply: 60s
+    assert: 3m
+    exec: 3m
+  steps:
+    - name: given-grafana-is-an-oidc-client
+      try:
+        - assert:
+            resource:
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: kube-prometheus-stack-grafana
+                namespace: monitoring
+              status:
+                availableReplicas: 1
+    - name: then-login-redirects-to-keycloak-realm
+      try:
+        - apply:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: grafana-oidc-probe
+              spec:
+                backoffLimit: 2
+                activeDeadlineSeconds: 150
+                template:
+                  spec:
+                    restartPolicy: Never
+                    containers:
+                      - name: probe
+                        image: busybox:1.36.1
+                        command:
+                          - sh
+                          - -c
+                          - |
+                            set -u
+                            svc="http://kube-prometheus-stack-grafana.monitoring.svc.cluster.local"
+                            out="$(wget -S --max-redirect=0 -O /dev/null "$svc/login/generic_oauth" 2>&1 || true)"
+                            echo "$out"
+                            if echo "$out" | grep -qi 'keycloak.home.keithjackson.dev/realms/homelab'; then
+                              echo GRAFANA_OIDC_REDIRECT_OK
+                            else
+                              echo "FAIL: Grafana did not redirect to the Keycloak homelab realm"; exit 1
+                            fi
+        - assert:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: grafana-oidc-probe
+              status:
+                succeeded: 1
+      finally:
+        - script:
+            content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"'

platform/keycloak/grafana-dashboard.yaml

@@ -0,0 +1,73 @@
+---
+# Keycloak health dashboard (R-OBS-03, docs/10 — "issues and health visible on dashboards"). A
+# ConfigMap labelled grafana_dashboard is auto-discovered by the Grafana sidecar (searchNamespace:
+# ALL, label grafana_dashboard — see platform/monitoring/kube-prometheus-stack.yaml) and provisioned
+# into Grafana. Applied by the `platform` Kustomization (plain ConfigMap, no CRD → no split needed).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: keycloak-dashboard
+  namespace: keycloak
+  labels:
+    grafana_dashboard: "1"
+data:
+  keycloak-health.json: |
+    {
+      "uid": "keycloak-health",
+      "title": "Keycloak — Identity / SSO health",
+      "tags": ["keycloak", "identity", "platform-health"],
+      "timezone": "browser",
+      "schemaVersion": 39,
+      "version": 1,
+      "refresh": "30s",
+      "time": {"from": "now-6h", "to": "now"},
+      "panels": [
+        {
+          "id": 1,
+          "type": "stat",
+          "title": "Keycloak pods ready",
+          "gridPos": {"h": 6, "w": 8, "x": 0, "y": 0},
+          "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}},
+          "targets": [
+            {"expr": "sum(kube_pod_status_ready{namespace=\"keycloak\", pod=~\"keycloak-[0-9]+\", condition=\"true\"})", "legendFormat": "ready"}
+          ]
+        },
+        {
+          "id": 2,
+          "type": "stat",
+          "title": "Keycloak DB (CNPG) pods ready",
+          "gridPos": {"h": 6, "w": 8, "x": 8, "y": 0},
+          "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}},
+          "targets": [
+            {"expr": "sum(kube_pod_status_ready{namespace=\"keycloak\", pod=~\"keycloak-db-.*\", condition=\"true\"})", "legendFormat": "db ready"}
+          ]
+        },
+        {
+          "id": 3,
+          "type": "timeseries",
+          "title": "Keycloak scrape target up",
+          "gridPos": {"h": 6, "w": 8, "x": 16, "y": 0},
+          "targets": [
+            {"expr": "up{namespace=\"keycloak\"}", "legendFormat": "{{pod}}"}
+          ]
+        },
+        {
+          "id": 4,
+          "type": "timeseries",
+          "title": "JVM heap used (Keycloak)",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
+          "targets": [
+            {"expr": "sum by (pod) (jvm_memory_used_bytes{namespace=\"keycloak\", area=\"heap\"})", "legendFormat": "{{pod}}"}
+          ]
+        },
+        {
+          "id": 5,
+          "type": "timeseries",
+          "title": "HTTP requests (Keycloak, rate)",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
+          "targets": [
+            {"expr": "sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[5m]))", "legendFormat": "req/s"}
+          ]
+        }
+      ]
+    }

platform/keycloak/keycloak-realm-clients.sops.yaml.example

@@ -5,7 +5,7 @@
 # WHAT THIS IS: the OIDC client secrets for the two confidential clients the realm import
 # (keycloak-realm-import.yaml) provisions — grafana and oauth2-proxy. The realm import injects
 # these values via spec.placeholders ($(env:GRAFANA_CLIENT_SECRET) / $(env:OAUTH2_PROXY_CLIENT_SECRET)),
-# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-SEC-01 / R-IDP-04).
+# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-IDP-04; docs/07).
 #
 # HAND-OFF STEPS (operator):
 #  (a) Generate two client secrets:

platform/keycloak/keycloak-realm-import.yaml

@@ -4,7 +4,7 @@
 # declaratively by the Keycloak Operator; CARRIED BY platform-config (dependsOn platform).
 #
 # Client secrets are NOT in this file: they are injected from the SOPS-encrypted keycloak-realm-clients
-# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-SEC-01 / R-IDP-04).
+# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-IDP-04; SOPS, docs/07).
 # Until that Secret is filled (operator hand-off, see keycloak-realm-clients.sops.yaml.example), this
 # import stays PENDING on the unresolved placeholder.
 apiVersion: k8s.keycloak.org/v2alpha1

platform/keycloak/kustomization.yaml

@@ -26,3 +26,5 @@ resources:
   - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml
   # --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
   - keycloak-admin-credentials.sops.yaml
+  # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
+  - grafana-dashboard.yaml

platform/monitoring/alerts.yaml

@@ -226,7 +226,7 @@ spec:
             severity: critical
           annotations:
             summary: "Egress VPN tunnel down"
-            description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (R-SEC-03 leak risk)."
+            description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)."
             runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down"
         # (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun
         # retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd
@@ -241,7 +241,7 @@ spec:
             severity: warning
           annotations:
             summary: "Spike in policy-denied network flows"
-            description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (R-SEC-03)."
+            description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)."
             runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows"
 
 # =====================================================================================