From 4bebd4b6d2b88aaca8874ddd897fd7aa1f4567a5 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 21:07:27 -0500 Subject: [PATCH] keycloak: coverage tests + dashboard + reword cross-cutting tags --- .../chainsaw-test.yaml | 70 +++++++++++++++ .../keycloak-dashboard/chainsaw-test.yaml | 49 +++++++++++ .../keycloak-db-dr/chainsaw-test.yaml | 58 +++++++++++++ .../keycloak-monitored/chainsaw-test.yaml | 81 ++++++++++++++++++ .../oidc-sso-grafana/chainsaw-test.yaml | 85 +++++++++++++++++++ platform/keycloak/grafana-dashboard.yaml | 73 ++++++++++++++++ .../keycloak-realm-clients.sops.yaml.example | 2 +- platform/keycloak/keycloak-realm-import.yaml | 2 +- platform/keycloak/kustomization.yaml | 2 + platform/monitoring/alerts.yaml | 4 +- 10 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml create mode 100644 platform/keycloak/grafana-dashboard.yaml diff --git a/conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml new file mode 100644 index 0000000..d9b6963 --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml @@ -0,0 +1,70 @@ +# ============================================================================= +# L2 BEHAVIORAL — identity config as code (R-IDP-03, R-IDP-04, R-IDP-06) [RUNNABLE] +# Spec: docs/15 identity-and-access; ADR-0014. Impl: platform/keycloak/keycloak-realm-import.yaml. +# +# GIVEN the `homelab` realm (groups platform-admins/media-users/viewers, OIDC clients, brute-force, +# and the conditional MFA browser flow) is declared as a KeycloakRealmImport in git, +# WHEN the Keycloak Operator reconciles it +# -> THEN the import reaches Done=True — realm/groups/clients/roles + the MFA flow are applied +# PURELY from git (R-IDP-03 config-as-code; R-IDP-04 MFA flow bound; R-IDP-06 groups/roles for +# least-privilege). No console clicking. +# +# Goes GREEN once Keycloak is up + the realm client-secret hand-off (keycloak-realm-clients) is filled +# so the placeholder resolves and the import completes. chainsaw v0.2.12: script + block contains check. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-config-as-code + labels: + conformance.home.k3s/R-IDP-03: "true" + conformance.home.k3s/R-IDP-04: "true" + conformance.home.k3s/R-IDP-06: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04, R-IDP-06" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + The homelab realm (groups, clients, MFA flow) reconciles to Done purely from the git-managed + KeycloakRealmImport — no manual console configuration. +spec: + timeouts: + exec: 5m + steps: + - name: realm-import-reconciles-from-git + try: + - script: + content: | + set -u + # Poll the KeycloakRealmImport until the operator reports Done=True. + for i in $(seq 1 30); do + st="$(kubectl -n keycloak get keycloakrealmimport homelab \ + -o jsonpath='{.status.conditions[?(@.type=="Done")].status}' 2>/dev/null || true)" + echo "poll $i: Done=$st" + if [ "$st" = "True" ]; then + echo "REALM_IMPORT_DONE_OK" + break + fi + sleep 10 + done + check: + (contains($stdout, 'REALM_IMPORT_DONE_OK')): true + - name: privileged-group-and-mfa-flow-present + try: + - script: + content: | + set -u + # The realm import declares the browser-mfa flow + platform-admins group. Confirm the + # realm CR / import spec carries them (declared-in-git evidence; the live realm state is + # exercised by the login flows). + flow="$(kubectl -n keycloak get keycloakrealmimport homelab \ + -o jsonpath='{.spec.realm.browserFlow}' 2>/dev/null || true)" + grp="$(kubectl -n keycloak get keycloakrealmimport homelab \ + -o jsonpath='{.spec.realm.groups[*].name}' 2>/dev/null || true)" + echo "browserFlow=$flow groups=$grp" + echo "$flow" | grep -q 'browser-mfa' || { echo "FAIL: MFA browser flow not bound"; exit 1; } + echo "$grp" | grep -q 'platform-admins' || { echo "FAIL: platform-admins group missing"; exit 1; } + echo "MFA_AND_GROUPS_OK" + check: + (contains($stdout, 'MFA_AND_GROUPS_OK')): true diff --git a/conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml new file mode 100644 index 0000000..81c694b --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml @@ -0,0 +1,49 @@ +# ============================================================================= +# L2 BEHAVIORAL — IdP health is visible on a dashboard (R-OBS-03) [RUNNABLE] +# Spec: docs/10 observability. Impl: platform/keycloak/grafana-dashboard.yaml (a grafana_dashboard +# ConfigMap auto-provisioned into Grafana by the sidecar). +# +# GIVEN a Keycloak health dashboard is declared as a grafana_dashboard ConfigMap, +# WHEN Grafana's sidecar discovers it (searchNamespace: ALL) +# -> THEN the dashboard is provisioned and Grafana is up — Keycloak health is visible on a dashboard +# (R-OBS-03). The presence of the labelled ConfigMap + a ready Grafana is the reconciliation proof. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-dashboard + labels: + conformance.home.k3s/R-OBS-03: "true" + annotations: + conformance.home.k3s/requirements: "R-OBS-03" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + A grafana_dashboard ConfigMap for Keycloak health exists and Grafana is up to render it. +spec: + timeouts: + assert: 3m + steps: + - name: keycloak-dashboard-configmap-provisioned + try: + - assert: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: keycloak-dashboard + namespace: keycloak + labels: + grafana_dashboard: "1" + - name: grafana-is-up-to-render-it + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kube-prometheus-stack-grafana + namespace: monitoring + status: + availableReplicas: 1 diff --git a/conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml new file mode 100644 index 0000000..9691bb0 --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml @@ -0,0 +1,58 @@ +# ============================================================================= +# L2 BEHAVIORAL — IdP datastore is DR-critical (R-IDP-05) [RUNNABLE] +# Spec: docs/15 identity-and-access; ADR-0014. Impl: platform/keycloak/postgres-cluster.yaml +# (CloudNativePG Cluster on the 3-replica `longhorn` StorageClass). +# +# GIVEN Keycloak's PostgreSQL runs as a CloudNativePG Cluster, +# WHEN we inspect it +# -> THEN it is healthy (>=1 ready instance) AND its data lives on the replicated `longhorn` +# StorageClass — i.e. the identity datastore is DR-critical / replicated (R-IDP-05). Keycloak is +# stateless and rebuilds from this DB + the git realm import. +# (Full off-cluster backup/restore is R-STATE-03/Velero territory, hand-off OQ-BKP.) +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-db-dr + labels: + conformance.home.k3s/R-IDP-05: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-05" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Keycloak's CloudNativePG database is healthy and stored on the 3-replica longhorn StorageClass. +spec: + timeouts: + exec: 5m + steps: + - name: db-cluster-is-healthy + try: + - script: + content: | + set -u + for i in $(seq 1 30); do + ready="$(kubectl -n keycloak get clusters.postgresql.cnpg.io keycloak-db \ + -o jsonpath='{.status.readyInstances}' 2>/dev/null || true)" + echo "poll $i: readyInstances=$ready" + if [ "${ready:-0}" -ge 1 ]; then echo DB_READY_OK; break; fi + sleep 10 + done + check: + (contains($stdout, 'DB_READY_OK')): true + - name: db-data-is-on-replicated-longhorn + try: + - script: + content: | + set -u + sc="$(kubectl -n keycloak get pvc -l cnpg.io/cluster=keycloak-db \ + -o jsonpath='{.items[0].spec.storageClassName}' 2>/dev/null || true)" + echo "db pvc storageClass=$sc" + if [ "$sc" = "longhorn" ]; then + echo DB_ON_LONGHORN_OK + else + echo "FAIL: Keycloak DB PVC not on the replicated longhorn StorageClass"; exit 1 + fi + check: + (contains($stdout, 'DB_ON_LONGHORN_OK')): true diff --git a/conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml new file mode 100644 index 0000000..e4b2230 --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml @@ -0,0 +1,81 @@ +# ============================================================================= +# L2 BEHAVIORAL — the IdP is monitored (R-OBS-01) [RUNNABLE] +# Spec: docs/10 observability. Impl: platform/keycloak/servicemonitor.yaml (Keycloak ServiceMonitor, +# carried via platform-config per the F-MON1 split) + the CNPG PodMonitor. +# +# GIVEN a ServiceMonitor for Keycloak, +# WHEN Prometheus reconciles its scrape config +# -> THEN Prometheus carries a `serviceMonitor/keycloak` scrape pool in its active targets — the IdP +# is monitored (metrics/health), R-OBS-01. (The KeycloakDown / KeycloakDatabaseDown alerts in +# platform/monitoring/alerts.yaml consume this signal + kube-state-metrics.) +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-monitored + labels: + conformance.home.k3s/R-OBS-01: "true" + annotations: + conformance.home.k3s/requirements: "R-OBS-01" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Prometheus discovers the Keycloak ServiceMonitor and carries it as an active scrape pool. +spec: + timeouts: + apply: 60s + assert: 3m + exec: 3m + steps: + - name: keycloak-servicemonitor-exists + try: + - assert: + resource: + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: keycloak + namespace: keycloak + - name: prometheus-carries-the-keycloak-scrape-pool + try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: keycloak-target-probe + spec: + backoffLimit: 3 + activeDeadlineSeconds: 150 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36.1 + command: + - sh + - -c + - | + set -u + prom="http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090" + for i in $(seq 1 12); do + out="$(wget -q -O - "$prom/api/v1/targets" 2>/dev/null || true)" + if echo "$out" | grep -q 'serviceMonitor/keycloak'; then + echo KEYCLOAK_TARGET_PRESENT_OK; exit 0 + fi + echo "poll $i: keycloak scrape pool not yet present"; sleep 10 + done + echo "FAIL: Prometheus has no serviceMonitor/keycloak scrape pool"; exit 1 + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: keycloak-target-probe + status: + succeeded: 1 + finally: + - script: + content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"' diff --git a/conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml b/conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml new file mode 100644 index 0000000..ac92c8c --- /dev/null +++ b/conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml @@ -0,0 +1,85 @@ +# ============================================================================= +# L2 BEHAVIORAL — native-OIDC SSO via the central IdP (R-IDP-01) [RUNNABLE] +# Spec: docs/15 identity-and-access; ADR-0014. Impl: Grafana as a Keycloak OIDC client +# (platform/monitoring/kube-prometheus-stack.yaml auth.generic_oauth) — the self-contained SSO proof. +# +# GIVEN Grafana is deployed and configured as an OIDC client of the Keycloak `homelab` realm, +# WHEN an unauthenticated browser starts the OIDC login (GET /login/generic_oauth) +# -> THEN Grafana redirects (302) to the CENTRAL IdP's authorize endpoint on +# keycloak.home.keithjackson.dev/realms/homelab — proving SSO is driven by the one IdP +# (R-IDP-01: central Identity Provider as the single user directory). +# +# Goes GREEN once the grafana-oidc client-secret hand-off is filled and Keycloak + Grafana are up. +# chainsaw v0.2.12: Job + status assert; no test-level finally. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: oidc-sso-grafana + labels: + conformance.home.k3s/R-IDP-01: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-01" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Grafana's OIDC login redirects to the central Keycloak realm — SSO is driven by the one IdP. +spec: + timeouts: + apply: 60s + assert: 3m + exec: 3m + steps: + - name: given-grafana-is-an-oidc-client + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kube-prometheus-stack-grafana + namespace: monitoring + status: + availableReplicas: 1 + - name: then-login-redirects-to-keycloak-realm + try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: grafana-oidc-probe + spec: + backoffLimit: 2 + activeDeadlineSeconds: 150 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36.1 + command: + - sh + - -c + - | + set -u + svc="http://kube-prometheus-stack-grafana.monitoring.svc.cluster.local" + out="$(wget -S --max-redirect=0 -O /dev/null "$svc/login/generic_oauth" 2>&1 || true)" + echo "$out" + if echo "$out" | grep -qi 'keycloak.home.keithjackson.dev/realms/homelab'; then + echo GRAFANA_OIDC_REDIRECT_OK + else + echo "FAIL: Grafana did not redirect to the Keycloak homelab realm"; exit 1 + fi + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: grafana-oidc-probe + status: + succeeded: 1 + finally: + - script: + content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"' diff --git a/platform/keycloak/grafana-dashboard.yaml b/platform/keycloak/grafana-dashboard.yaml new file mode 100644 index 0000000..d1d701f --- /dev/null +++ b/platform/keycloak/grafana-dashboard.yaml @@ -0,0 +1,73 @@ +--- +# Keycloak health dashboard (R-OBS-03, docs/10 — "issues and health visible on dashboards"). A +# ConfigMap labelled grafana_dashboard is auto-discovered by the Grafana sidecar (searchNamespace: +# ALL, label grafana_dashboard — see platform/monitoring/kube-prometheus-stack.yaml) and provisioned +# into Grafana. Applied by the `platform` Kustomization (plain ConfigMap, no CRD → no split needed). +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-dashboard + namespace: keycloak + labels: + grafana_dashboard: "1" +data: + keycloak-health.json: | + { + "uid": "keycloak-health", + "title": "Keycloak — Identity / SSO health", + "tags": ["keycloak", "identity", "platform-health"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-6h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Keycloak pods ready", + "gridPos": {"h": 6, "w": 8, "x": 0, "y": 0}, + "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"keycloak\", pod=~\"keycloak-[0-9]+\", condition=\"true\"})", "legendFormat": "ready"} + ] + }, + { + "id": 2, + "type": "stat", + "title": "Keycloak DB (CNPG) pods ready", + "gridPos": {"h": 6, "w": 8, "x": 8, "y": 0}, + "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"keycloak\", pod=~\"keycloak-db-.*\", condition=\"true\"})", "legendFormat": "db ready"} + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Keycloak scrape target up", + "gridPos": {"h": 6, "w": 8, "x": 16, "y": 0}, + "targets": [ + {"expr": "up{namespace=\"keycloak\"}", "legendFormat": "{{pod}}"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "JVM heap used (Keycloak)", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "targets": [ + {"expr": "sum by (pod) (jvm_memory_used_bytes{namespace=\"keycloak\", area=\"heap\"})", "legendFormat": "{{pod}}"} + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "HTTP requests (Keycloak, rate)", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "targets": [ + {"expr": "sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[5m]))", "legendFormat": "req/s"} + ] + } + ] + } diff --git a/platform/keycloak/keycloak-realm-clients.sops.yaml.example b/platform/keycloak/keycloak-realm-clients.sops.yaml.example index a098e1f..ff0bbc5 100644 --- a/platform/keycloak/keycloak-realm-clients.sops.yaml.example +++ b/platform/keycloak/keycloak-realm-clients.sops.yaml.example @@ -5,7 +5,7 @@ # WHAT THIS IS: the OIDC client secrets for the two confidential clients the realm import # (keycloak-realm-import.yaml) provisions — grafana and oauth2-proxy. The realm import injects # these values via spec.placeholders ($(env:GRAFANA_CLIENT_SECRET) / $(env:OAUTH2_PROXY_CLIENT_SECRET)), -# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-SEC-01 / R-IDP-04). +# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-IDP-04; docs/07). # # HAND-OFF STEPS (operator): # (a) Generate two client secrets: diff --git a/platform/keycloak/keycloak-realm-import.yaml b/platform/keycloak/keycloak-realm-import.yaml index 026a4d0..894844f 100644 --- a/platform/keycloak/keycloak-realm-import.yaml +++ b/platform/keycloak/keycloak-realm-import.yaml @@ -4,7 +4,7 @@ # declaratively by the Keycloak Operator; CARRIED BY platform-config (dependsOn platform). # # Client secrets are NOT in this file: they are injected from the SOPS-encrypted keycloak-realm-clients -# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-SEC-01 / R-IDP-04). +# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-IDP-04; SOPS, docs/07). # Until that Secret is filled (operator hand-off, see keycloak-realm-clients.sops.yaml.example), this # import stays PENDING on the unresolved placeholder. apiVersion: k8s.keycloak.org/v2alpha1 diff --git a/platform/keycloak/kustomization.yaml b/platform/keycloak/kustomization.yaml index b5db95b..d07e761 100644 --- a/platform/keycloak/kustomization.yaml +++ b/platform/keycloak/kustomization.yaml @@ -26,3 +26,5 @@ resources: - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml # --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) --- - keycloak-admin-credentials.sops.yaml + # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) --- + - grafana-dashboard.yaml diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index 434e4ab..5ae3901 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -226,7 +226,7 @@ spec: severity: critical annotations: summary: "Egress VPN tunnel down" - description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (R-SEC-03 leak risk)." + description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)." runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down" # (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun # retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd @@ -241,7 +241,7 @@ spec: severity: warning annotations: summary: "Spike in policy-denied network flows" - description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (R-SEC-03)." + description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)." runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows" # =====================================================================================