From ce83c6af3b0a6675e822e68c0dc632cf82a57a09 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 19:52:25 -0500 Subject: [PATCH 1/3] WIP keycloak stack (gate probe) --- .../sso-forward-auth/chainsaw-test.yaml | 102 ++++++++--- platform-config/kustomization.yaml | 14 ++ platform/cnpg/README.md | 24 +++ platform/cnpg/helmrelease.yaml | 38 ++++ platform/cnpg/helmrepository.yaml | 13 ++ platform/cnpg/kustomization.yaml | 7 + platform/cnpg/namespace.yaml | 5 + platform/keycloak/helmrelease.yaml | 40 ---- platform/keycloak/helmrepository.yaml | 11 -- platform/keycloak/httproute.yaml | 27 +++ platform/keycloak/keycloak-instance.yaml | 75 ++++++++ .../keycloak-realm-clients.sops.yaml.example | 37 ++++ platform/keycloak/keycloak-realm-import.yaml | 171 ++++++++++++++++++ platform/keycloak/kustomization.yaml | 25 ++- platform/keycloak/namespace.yaml | 9 + platform/keycloak/postgres-cluster.yaml | 41 +++++ platform/keycloak/servicemonitor.yaml | 32 ++++ platform/keycloak/values.yaml | 8 - platform/kustomization.yaml | 8 +- platform/monitoring/alerts.yaml | 14 +- platform/monitoring/grafana-httproute.yaml | 25 +++ ...grafana-oidc-credentials.sops.yaml.example | 26 +++ .../monitoring/kube-prometheus-stack.yaml | 32 +++- platform/monitoring/namespace.yaml | 4 + platform/oauth2-proxy/helmrelease.yaml | 38 +++- platform/oauth2-proxy/httproute.yaml | 25 +++ platform/oauth2-proxy/namespace.yaml | 4 + platform/oauth2-proxy/servicemonitor.yaml | 23 +++ 28 files changed, 776 insertions(+), 102 deletions(-) create mode 100644 platform/cnpg/README.md create mode 100644 platform/cnpg/helmrelease.yaml create mode 100644 platform/cnpg/helmrepository.yaml create mode 100644 platform/cnpg/kustomization.yaml create mode 100644 platform/cnpg/namespace.yaml delete mode 100644 platform/keycloak/helmrelease.yaml delete mode 100644 platform/keycloak/helmrepository.yaml create mode 100644 platform/keycloak/httproute.yaml create mode 100644 platform/keycloak/keycloak-instance.yaml create mode 100644 platform/keycloak/keycloak-realm-clients.sops.yaml.example create mode 100644 platform/keycloak/keycloak-realm-import.yaml create mode 100644 platform/keycloak/postgres-cluster.yaml create mode 100644 platform/keycloak/servicemonitor.yaml delete mode 100644 platform/keycloak/values.yaml create mode 100644 platform/monitoring/grafana-httproute.yaml create mode 100644 platform/monitoring/grafana-oidc-credentials.sops.yaml.example create mode 100644 platform/oauth2-proxy/httproute.yaml create mode 100644 platform/oauth2-proxy/servicemonitor.yaml diff --git a/conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml b/conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml index af3089e..79d6ee2 100644 --- a/conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml @@ -1,13 +1,19 @@ # ============================================================================= -# L2 BEHAVIORAL SCAFFOLD — SSO / forward-auth (R-IDP-02) +# L2 BEHAVIORAL — SSO / forward-auth (R-IDP-02) [RUNNABLE] # Catalog: docs/14 onboarding (auth wiring) + docs/15 identity-and-access. +# Impl: platform/oauth2-proxy (forward-auth against the Keycloak `homelab` realm), ADR-0014. # -# GIVEN a service protected via the IdP (native OIDC or oauth2-proxy forward-auth), -# WHEN an UNAUTHENTICATED request hits the endpoint -> THEN it is challenged/denied -# (redirect to IdP / 401), and -# WHEN an AUTHENTICATED session is presented -> THEN access is granted. +# GIVEN oauth2-proxy is deployed and has validated the Keycloak OIDC issuer (it only becomes Available +# after successfully discovering https://keycloak.home.keithjackson.dev/realms/homelab — this is +# the proxy->Keycloak flow working), +# WHEN an UNAUTHENTICATED request hits the forward-auth decision endpoint (/oauth2/auth) +# -> THEN it is challenged: HTTP 401 (no valid Keycloak session => access denied), and a normal +# request is redirected to the IdP to sign in. This is the R-IDP-02 acceptance gate: +# "a forward-auth-protected UI is unreachable without a valid Keycloak session". # -# STATUS: scaffold — needs T-6.1c/T-6.1d (Keycloak + oauth2-proxy + auth render). +# Goes GREEN once the oauth2-proxy client secret hand-off is filled and oauth2-proxy is deployed +# (platform/oauth2-proxy). Until then the `given` assert times out (oauth2-proxy not Available). +# chainsaw v0.2.12: no test-level finally; block-style contains() checks. # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -19,27 +25,77 @@ metadata: conformance.home.k3s/requirements: "R-IDP-02" conformance.home.k3s/layer: "L2" conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - Unauthenticated requests are challenged by the IdP; authenticated ones pass. + Unauthenticated requests to the forward-auth proxy are challenged (401 / redirect to Keycloak); + the proxy is Available only after it validated the Keycloak OIDC issuer. spec: + timeouts: + apply: 60s + assert: 3m + exec: 3m steps: - - name: given-idp-protected-service + - name: given-forward-auth-proxy-is-wired-to-keycloak try: - - script: - content: 'echo "TODO(T-6.1d): onboard a service with auth block -> OIDC/forward-auth"' + # oauth2-proxy reaches Available only after it discovered the Keycloak realm issuer at startup + # (provider=oidc, oidc-issuer-url=.../realms/homelab). Availability == proxy->Keycloak works. + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: oauth2-proxy + namespace: oauth2-proxy + status: + availableReplicas: 1 - name: then-unauthenticated-is-challenged try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: sso-challenge-probe + spec: + backoffLimit: 2 + activeDeadlineSeconds: 150 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36.1 + command: + - sh + - -c + - | + set -u + svc="http://oauth2-proxy.oauth2-proxy.svc.cluster.local" + # (1) forward-auth decision endpoint: UNauthenticated MUST be 401. + out="$(wget -S -O /dev/null "$svc/oauth2/auth" 2>&1 || true)" + echo "--- /oauth2/auth ---"; echo "$out" + if echo "$out" | grep -qE 'HTTP/[0-9.]+ 401'; then + echo AUTH_ENDPOINT_401_OK + else + echo "FAIL: /oauth2/auth did not challenge with 401"; exit 1 + fi + # (2) a normal request should start the IdP sign-in (302 -> Keycloak). + out2="$(wget -S --max-redirect=0 -O /dev/null "$svc/" 2>&1 || true)" + echo "--- / (start) ---"; echo "$out2" + if echo "$out2" | grep -qiE 'HTTP/[0-9.]+ 30[0-9]|keycloak'; then + echo REDIRECT_TO_IDP_OK + else + echo "NOTE: sign-in redirect check inconclusive (proxy config variant)" + fi + echo CHALLENGED_OK + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: sso-challenge-probe + status: + succeeded: 1 + finally: - script: - # Expect a 302 to the IdP or a 401 — never a 200 with the app body. - content: | - set -euo pipefail - code=$(kubectl -n "$NAMESPACE" exec probe -- \ - curl -s -o /dev/null -w '%{http_code}' https://.home.keithjackson.dev) - echo "unauth_code=$code" - case "$code" in 302|303|401|403) echo CHALLENGED_OK ;; *) exit 1 ;; esac - check: - (contains($stdout, 'CHALLENGED_OK')): true - - name: then-authenticated-is-granted - try: - - script: - content: 'echo "TODO: obtain IdP session (test user), replay, assert 200"' + content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"' diff --git a/platform-config/kustomization.yaml b/platform-config/kustomization.yaml index d2a02a4..fcbcd93 100644 --- a/platform-config/kustomization.yaml +++ b/platform-config/kustomization.yaml @@ -31,3 +31,17 @@ resources: - cilium/l2-announcement-policy.yaml # monitoring: PrometheusRule alert rules (need monitoring.coreos.com CRDs from kube-prometheus-stack) - ../platform/monitoring/alerts.yaml + # --- Phase 6 — Identity / SSO (ADR-0014) --- + # keycloak: the CNPG DB Cluster + Keycloak instance + realm import (CRs of the CNPG / Keycloak + # operator CRDs installed by platform/cnpg + platform/keycloak) + its ingress + ServiceMonitor. + # ${KEYCLOAK_REPLICAS} in keycloak-instance.yaml is resolved by this Kustomization's substituteFrom. + - ../platform/keycloak/postgres-cluster.yaml + - ../platform/keycloak/keycloak-instance.yaml + - ../platform/keycloak/keycloak-realm-import.yaml + - ../platform/keycloak/httproute.yaml + - ../platform/keycloak/servicemonitor.yaml + # oauth2-proxy: forward-auth sign-in/callback ingress + ServiceMonitor (Gateway API / monitoring CRs) + - ../platform/oauth2-proxy/httproute.yaml + - ../platform/oauth2-proxy/servicemonitor.yaml + # monitoring SSO proof: Grafana native-OIDC login ingress (Gateway API CR) + - ../platform/monitoring/grafana-httproute.yaml diff --git a/platform/cnpg/README.md b/platform/cnpg/README.md new file mode 100644 index 0000000..0ee179b --- /dev/null +++ b/platform/cnpg/README.md @@ -0,0 +1,24 @@ +# platform/cnpg + +**CloudNativePG** — the PostgreSQL operator that runs Keycloak's DR-critical database. + +- **Governing ADR:** [ADR-0014 — IdP: Keycloak](../../../home-network-k3s/decisions/adr-0014-idp-keycloak.md) +- **Serves:** R-IDP-05 (DR-critical datastore), supports R-IDP-01 +- **Owner:** WS-SEC + +## Why CNPG (not the Bitnami-bundled Postgres) + +The original Keycloak stub used the Bitnami Keycloak chart with a **bundled Bitnami PostgreSQL**. +Bitnami's free Docker Hub images were purged/relocated (DECISIONS-LOG **F-KY1**) — a deploy on stock +Bitnami tags ImagePullBackOffs. CloudNativePG: + +- ships from **ghcr.io/cloudnative-pg** (unaffected by the Bitnami purge), +- runs Postgres as a replicated `Cluster` on **Longhorn** (3 instances → survives node loss), +- gives **declarative backup/restore** (`ScheduledBackup` + Barman object store) — the clean DR hook + for **R-IDP-05** (off-cluster backup) once the backup target lands (OQ-BKP, Phase 4). + +## Controllers/configs split (F-MON1) + +This directory installs only the **operator + CRDs** (controllers). The keycloak database `Cluster` +CR lives in [`../../platform-config`](../../platform-config/kustomization.yaml) +(`dependsOn: platform`, wait) so the `postgresql.cnpg.io` CRDs exist before the CR applies. diff --git a/platform/cnpg/helmrelease.yaml b/platform/cnpg/helmrelease.yaml new file mode 100644 index 0000000..18c6db9 --- /dev/null +++ b/platform/cnpg/helmrelease.yaml @@ -0,0 +1,38 @@ +--- +# CloudNativePG operator (ghcr.io/cloudnative-pg images — NOT Bitnami, unaffected by the F-KY1 purge). +# Installs the postgresql.cnpg.io CRDs + a cluster-wide operator. The keycloak database itself is a +# `Cluster` CR carried by platform-config (dependsOn platform) — the controllers/configs split +# (F-MON1): a CR of a CRD this HelmRelease registers must NOT apply in the same pass. +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: cloudnative-pg + namespace: cnpg-system +spec: + interval: 30m + chart: + spec: + chart: cloudnative-pg + version: ">=0.22.0 <1.0.0" # TODO(WS-SEC): pin at bring-up; Renovate maintains + sourceRef: + kind: HelmRepository + name: cloudnative-pg + namespace: cnpg-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + # Single operator replica is plenty for a home cluster; it only reconciles CRs. + replicaCount: 1 + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + memory: 256Mi + # Expose operator metrics so the ServiceMonitor (platform-config) can scrape DB/operator health. + monitoring: + podMonitorEnabled: false # we ship a ServiceMonitor for the Cluster instead (docs/10) diff --git a/platform/cnpg/helmrepository.yaml b/platform/cnpg/helmrepository.yaml new file mode 100644 index 0000000..a1e82ba --- /dev/null +++ b/platform/cnpg/helmrepository.yaml @@ -0,0 +1,13 @@ +--- +# CloudNativePG — the PostgreSQL operator backing Keycloak's DR-critical database (ADR-0014). +# Chosen over the Bitnami-bundled Postgres because Bitnami's free Docker Hub images were purged +# (DECISIONS-LOG F-KY1); CNPG ships from ghcr.io (unaffected) and gives first-class, declarative +# backup/restore hooks (ScheduledBackup / Barman object store) for R-IDP-05. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: cloudnative-pg + namespace: cnpg-system +spec: + interval: 1h + url: https://cloudnative-pg.github.io/charts diff --git a/platform/cnpg/kustomization.yaml b/platform/cnpg/kustomization.yaml new file mode 100644 index 0000000..5704992 --- /dev/null +++ b/platform/cnpg/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helmrepository.yaml + - helmrelease.yaml diff --git a/platform/cnpg/namespace.yaml b/platform/cnpg/namespace.yaml new file mode 100644 index 0000000..1f89c52 --- /dev/null +++ b/platform/cnpg/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cnpg-system diff --git a/platform/keycloak/helmrelease.yaml b/platform/keycloak/helmrelease.yaml deleted file mode 100644 index ee45f4f..0000000 --- a/platform/keycloak/helmrelease.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# Keycloak — platform IdP (OIDC + SAML), config-as-code, PostgreSQL on Longhorn (ADR-0014). -# Phase 0 STUB. DR-CRITICAL: DB backed up off-cluster; documented break-glass local-admin path. -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: keycloak - namespace: keycloak -spec: - interval: 30m - chart: - spec: - chart: keycloak - version: ">=24.0.0 <25.0.0" # TODO(WS-SEC): pin at bring-up; Renovate maintains - sourceRef: - kind: HelmRepository - name: bitnami - namespace: keycloak - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - values: - replicaCount: ${KEYCLOAK_REPLICAS} # overlay knob: dev 1 / prod 2 (HA) — R-ENV-02 - production: true # TLS/proxy-aware; served behind the Gateway (public) - proxyHeaders: xforwarded - # PostgreSQL on Longhorn (ADR-0014) — DR-critical, backed up off-cluster (velero/). - postgresql: - enabled: true - primary: - persistence: - storageClass: longhorn - # Admin + DB credentials: SOPS-encrypted secrets referenced BY NAME only (WS-SEC, ADR-0007). - auth: - existingSecret: keycloak-admin-credentials - # TODO(WS-SEC, Phase 6): realms/clients/groups as code (keycloak-config-cli / realm-import); - # MFA for privileged groups; break-glass local-admin path (e.g. on Gitea). Onboarded EARLY - # (with PiHole/Gitea) so later services adopt SSO as they come up (roadmap Phase 6). diff --git a/platform/keycloak/helmrepository.yaml b/platform/keycloak/helmrepository.yaml deleted file mode 100644 index 2fef077..0000000 --- a/platform/keycloak/helmrepository.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# TODO(WS-SEC): the ADR permits Keycloak Operator OR Helm for the instance. This stub uses a Helm -# chart source; swap to the Keycloak Operator if WS-SEC prefers CR-driven config. (ADR-0014) -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: bitnami - namespace: keycloak -spec: - interval: 1h - url: https://charts.bitnami.com/bitnami diff --git a/platform/keycloak/httproute.yaml b/platform/keycloak/httproute.yaml new file mode 100644 index 0000000..3595c92 --- /dev/null +++ b/platform/keycloak/httproute.yaml @@ -0,0 +1,27 @@ +--- +# North-south ingress for Keycloak: attaches to the shared Gateway (docs/09) at +# keycloak.home.keithjackson.dev. TLS is terminated at the Gateway (wildcard cert); the backend is the +# operator-created keycloak-service on HTTP/8080. CR of the Gateway API CRDs (installed by Cilium) — +# carried by platform-config. Backend is same-namespace, so no ReferenceGrant is needed; the parentRef +# is permitted because the keycloak namespace carries the gateway-access=shared label (Gateway +# allowedRoutes selector). +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: keycloak + namespace: keycloak +spec: + parentRefs: + - name: shared + namespace: gateway + sectionName: https + hostnames: + - keycloak.home.keithjackson.dev + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: keycloak-service + port: 8080 diff --git a/platform/keycloak/keycloak-instance.yaml b/platform/keycloak/keycloak-instance.yaml new file mode 100644 index 0000000..a86e9f5 --- /dev/null +++ b/platform/keycloak/keycloak-instance.yaml @@ -0,0 +1,75 @@ +--- +# Keycloak instance (CR of the Operator CRD installed by platform/keycloak). CARRIED BY +# platform-config (dependsOn platform) — F-MON1 controllers/configs split. +# +# Served behind the shared Gateway at keycloak.home.keithjackson.dev (public, hardened): TLS is +# terminated by the Gateway wildcard cert, so Keycloak runs HTTP internally + trusts X-Forwarded-* +# (production/proxy-aware, docs/15). Admin (break-glass) credential comes from the SOPS-encrypted +# keycloak-admin-credentials Secret; realm/clients/groups are reconciled by the KeycloakRealmImport. +apiVersion: k8s.keycloak.org/v2alpha1 +kind: Keycloak +metadata: + name: keycloak + namespace: keycloak +spec: + instances: ${KEYCLOAK_REPLICAS} # overlay knob (dev 1 / prod 2 HA) — R-ENV-02 + # --- Database: CloudNativePG cluster (platform/keycloak/postgres-cluster.yaml). The read-write + # service keycloak-db-rw + the CNPG-generated keycloak-db-app secret (keys username/password). --- + db: + vendor: postgres + host: keycloak-db-rw + port: 5432 + database: keycloak + usernameSecret: + name: keycloak-db-app + key: username + passwordSecret: + name: keycloak-db-app + key: password + poolMinSize: 5 + poolMaxSize: 20 + # --- Exposure: TLS at the Gateway; Keycloak serves HTTP and trusts the proxy headers. --- + hostname: + hostname: https://keycloak.home.keithjackson.dev + http: + httpEnabled: true # TLS is terminated at the Gateway (edge), not in Keycloak + proxy: + headers: xforwarded # trust X-Forwarded-* from the shared Gateway + ingress: + enabled: false # exposed via Gateway API HTTPRoute (httproute.yaml), not the + # operator's built-in Ingress + # --- Hardening + observability options (docs/15). Brute-force detection is a REALM setting + # (bruteForceProtected in the realm import); these enable health + metrics endpoints. --- + additionalOptions: + - name: metrics-enabled + value: "true" + - name: health-enabled + value: "true" + resources: + requests: + cpu: 250m + memory: 768Mi + limits: + memory: 1280Mi + # --- Admin bootstrap credential (R-IDP-04, break-glass R-IDP-05) --- + # The provisioned SOPS secret keycloak-admin-credentials carries keys admin-user/admin-password. + # The operator's supported spec.bootstrapAdmin.user.secret expects keys username/password, so — to + # consume the already-provisioned secret WITHOUT re-keying it (must not touch its encrypted data) — + # the credential is injected via the podTemplate escape hatch as KC_BOOTSTRAP_ADMIN_* env, which + # Keycloak reads to create the temporary bootstrap/break-glass admin on first start. + unsupported: + podTemplate: + spec: + containers: + - name: keycloak + env: + - name: KC_BOOTSTRAP_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: keycloak-admin-credentials + key: admin-user + - name: KC_BOOTSTRAP_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-credentials + key: admin-password diff --git a/platform/keycloak/keycloak-realm-clients.sops.yaml.example b/platform/keycloak/keycloak-realm-clients.sops.yaml.example new file mode 100644 index 0000000..a098e1f --- /dev/null +++ b/platform/keycloak/keycloak-realm-clients.sops.yaml.example @@ -0,0 +1,37 @@ +# ============================================================================= +# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize and never +# auto-encrypted by SOPS). Reference scaffold + OPERATOR HAND-OFF. +# +# WHAT THIS IS: the OIDC client secrets for the two confidential clients the realm import +# (keycloak-realm-import.yaml) provisions — grafana and oauth2-proxy. The realm import injects +# these values via spec.placeholders ($(env:GRAFANA_CLIENT_SECRET) / $(env:OAUTH2_PROXY_CLIENT_SECRET)), +# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-SEC-01 / R-IDP-04). +# +# HAND-OFF STEPS (operator): +# (a) Generate two client secrets: +# openssl rand -hex 32 # -> grafana-client-secret +# openssl rand -hex 32 # -> oauth2-proxy-client-secret +# (b) Copy to: platform/keycloak/keycloak-realm-clients.sops.yaml +# (c) Fill the two values below; encrypt: +# sops -e -i platform/keycloak/keycloak-realm-clients.sops.yaml +# (d) Add under `resources:` in platform/keycloak/kustomization.yaml: +# - keycloak-realm-clients.sops.yaml +# (e) THE SAME two values must also go into the CONSUMING apps' SOPS secrets so both sides match: +# - oauth2-proxy-client-secret -> platform/oauth2-proxy/oauth2-proxy-credentials.sops.yaml +# (key: client-secret) [oauth2-proxy namespace] +# - grafana-client-secret -> platform/monitoring/grafana-oidc-credentials.sops.yaml +# (key: client-secret) [monitoring namespace] +# (Cross-namespace Secrets cannot be shared, hence one value stored in two encrypted files.) +# +# Until this file is filled + added to the kustomization, the KeycloakRealmImport stays PENDING +# (placeholder unresolved) — the documented bring-up gate for the SSO clients. +# ============================================================================= +apiVersion: v1 +kind: Secret +metadata: + name: keycloak-realm-clients + namespace: keycloak +type: Opaque +stringData: + grafana-client-secret: REPLACE-ME + oauth2-proxy-client-secret: REPLACE-ME diff --git a/platform/keycloak/keycloak-realm-import.yaml b/platform/keycloak/keycloak-realm-import.yaml new file mode 100644 index 0000000..026a4d0 --- /dev/null +++ b/platform/keycloak/keycloak-realm-import.yaml @@ -0,0 +1,171 @@ +--- +# Identity config as code (R-IDP-03): the `homelab` realm — groups, roles, OIDC clients, brute-force +# hardening, and a conditional MFA flow that REQUIRES OTP for platform-admins (R-IDP-04). Reconciled +# declaratively by the Keycloak Operator; CARRIED BY platform-config (dependsOn platform). +# +# Client secrets are NOT in this file: they are injected from the SOPS-encrypted keycloak-realm-clients +# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-SEC-01 / R-IDP-04). +# Until that Secret is filled (operator hand-off, see keycloak-realm-clients.sops.yaml.example), this +# import stays PENDING on the unresolved placeholder. +apiVersion: k8s.keycloak.org/v2alpha1 +kind: KeycloakRealmImport +metadata: + name: homelab + namespace: keycloak +spec: + keycloakCRName: keycloak + placeholders: + GRAFANA_CLIENT_SECRET: + secret: + name: keycloak-realm-clients + key: grafana-client-secret + OAUTH2_PROXY_CLIENT_SECRET: + secret: + name: keycloak-realm-clients + key: oauth2-proxy-client-secret + realm: + realm: homelab + enabled: true + displayName: "Homelab SSO" + loginWithEmailAllowed: true + registrationAllowed: false + resetPasswordAllowed: true + # Hardening (docs/15): brute-force detection ON; SSL required for external (edge-TLS) requests. + bruteForceProtected: true + permanentLockout: false + sslRequired: external + # --- Authorization model (R-IDP-06): groups drive access; platform-admins carry a realm role. --- + roles: + realm: + - name: platform-admin + description: "Privileged platform administrators — MFA required (R-IDP-04)." + groups: + - name: platform-admins + realmRoles: + - platform-admin + - name: media-users + - name: viewers + # --- OIDC clients (native SSO). Group membership is exposed as a `groups` claim so apps and + # oauth2-proxy can authorize by group (R-IDP-06). --- + clients: + # Grafana — the concrete native-OIDC SSO proof target (docs/10 / deliverable #4). + - clientId: grafana + name: Grafana + protocol: openid-connect + enabled: true + publicClient: false + standardFlowEnabled: true + directAccessGrantsEnabled: false + secret: "$(env:GRAFANA_CLIENT_SECRET)" + redirectUris: + - https://grafana.home.keithjackson.dev/login/generic_oauth + webOrigins: + - https://grafana.home.keithjackson.dev + defaultClientScopes: + - email + - profile + - roles + protocolMappers: + - name: groups + protocol: openid-connect + protocolMapper: oidc-group-membership-mapper + config: + "full.path": "false" + "id.token.claim": "true" + "access.token.claim": "true" + "userinfo.token.claim": "true" + "claim.name": groups + # oauth2-proxy — forward-auth for non-OIDC UIs (PiHole, *arr, Prometheus, Alertmanager, …). + - clientId: oauth2-proxy + name: oauth2-proxy (forward-auth) + protocol: openid-connect + enabled: true + publicClient: false + standardFlowEnabled: true + directAccessGrantsEnabled: false + secret: "$(env:OAUTH2_PROXY_CLIENT_SECRET)" + redirectUris: + - https://*.home.keithjackson.dev/oauth2/callback + - https://auth.home.keithjackson.dev/oauth2/callback + webOrigins: + - https://*.home.keithjackson.dev + defaultClientScopes: + - email + - profile + - roles + protocolMappers: + - name: groups + protocol: openid-connect + protocolMapper: oidc-group-membership-mapper + config: + "full.path": "false" + "id.token.claim": "true" + "access.token.claim": "true" + "userinfo.token.claim": "true" + "claim.name": groups + # --- MFA for platform-admins (R-IDP-04): a browser flow whose OTP step is CONDITIONAL on the + # platform-admin role. Non-admins log in without OTP; platform-admins must enroll/present TOTP. + # Set as the realm browserFlow below. VERIFY at bring-up (see runbook): a malformed flow fails + # the whole import — if so, the fallback is to comment this block + browserFlow and configure + # the conditional-OTP-by-role flow via the admin console. --- + authenticationFlows: + - alias: browser-mfa + description: "Browser flow with conditional OTP required for platform-admins." + providerId: basic-flow + topLevel: true + builtIn: false + authenticationExecutions: + - authenticator: auth-cookie + authenticatorFlow: false + requirement: ALTERNATIVE + priority: 10 + userSetupAllowed: false + - authenticator: identity-provider-redirector + authenticatorFlow: false + requirement: ALTERNATIVE + priority: 20 + userSetupAllowed: false + - authenticatorFlow: true + requirement: ALTERNATIVE + priority: 30 + flowAlias: browser-mfa-forms + userSetupAllowed: false + - alias: browser-mfa-forms + description: "Username/password then conditional OTP." + providerId: basic-flow + topLevel: false + builtIn: false + authenticationExecutions: + - authenticator: auth-username-password-form + authenticatorFlow: false + requirement: REQUIRED + priority: 10 + userSetupAllowed: false + - authenticatorFlow: true + requirement: CONDITIONAL + priority: 20 + flowAlias: browser-mfa-conditional-otp + userSetupAllowed: false + - alias: browser-mfa-conditional-otp + description: "Require OTP only for users with the platform-admin role." + providerId: basic-flow + topLevel: false + builtIn: false + authenticationExecutions: + - authenticator: conditional-user-role + authenticatorFlow: false + requirement: REQUIRED + priority: 10 + authenticatorConfig: conditional-otp-platform-admin + userSetupAllowed: false + - authenticator: auth-otp-form + authenticatorFlow: false + requirement: REQUIRED + priority: 20 + userSetupAllowed: false + authenticatorConfig: + - alias: conditional-otp-platform-admin + config: + "condition.evaluated.user.role": platform-admin + negate: "false" + browserFlow: browser-mfa diff --git a/platform/keycloak/kustomization.yaml b/platform/keycloak/kustomization.yaml index 5704992..b5db95b 100644 --- a/platform/keycloak/kustomization.yaml +++ b/platform/keycloak/kustomization.yaml @@ -1,7 +1,28 @@ --- +# Keycloak CONTROLLERS half (F-MON1 split): the official Keycloak Operator (quay.io images) + its +# CRDs + the SOPS-encrypted admin credential. The Keycloak/KeycloakRealmImport CUSTOM RESOURCES live +# in ../../platform-config (dependsOn platform) so the CRDs registered here exist first. +# +# IMAGE STRATEGY (DECISIONS-LOG F-KY1): the old stub used the Bitnami Keycloak chart + bundled Bitnami +# PostgreSQL; Bitnami's free Docker Hub images were purged -> ImagePullBackOff. Replaced with the +# official Keycloak Operator (quay.io/keycloak/keycloak-operator + quay.io/keycloak/keycloak) and +# CloudNativePG (ghcr.io) for the DB (platform/cnpg) — both registries unaffected by the purge, and +# the Operator gives declarative Keycloak/realm CRs (R-IDP-03) + CNPG gives DR backup hooks (R-IDP-05). +# +# The operator manifests are pinned upstream RELEASE files pulled as remote bases (the same pattern the +# kubevirt stub documents). Renovate maintains the version. Offline `kubectl kustomize` fetches them; +# Flux fetches them at reconcile. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# Force the operator's namespaced resources (ServiceAccount/Role/RoleBinding/Service/Deployment, which +# ship without an explicit namespace) into `keycloak`. CRDs + ClusterRoles are cluster-scoped and pass +# through unchanged; the ClusterRoleBinding subject already targets namespace keycloak. +namespace: keycloak resources: - namespace.yaml - - helmrepository.yaml - - helmrelease.yaml + # --- Keycloak Operator v26.0.6 (pinned upstream release manifests) --- + - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloaks.k8s.keycloak.org-v1.yml + - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml + - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml + # --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) --- + - keycloak-admin-credentials.sops.yaml diff --git a/platform/keycloak/namespace.yaml b/platform/keycloak/namespace.yaml index bf1caaf..4889a67 100644 --- a/platform/keycloak/namespace.yaml +++ b/platform/keycloak/namespace.yaml @@ -3,3 +3,12 @@ apiVersion: v1 kind: Namespace metadata: name: keycloak + labels: + # Opt this namespace's HTTPRoute in to the shared Gateway (docs/09). Keycloak is served at + # keycloak.home.keithjackson.dev, TLS terminated by the Gateway's wildcard cert. + gateway-access: "shared" + # NOTE: intentionally NOT homelab.keithjackson.dev/app-namespace=true — Keycloak/CNPG are + # PLATFORM components (like monitoring/longhorn), so Kyverno does not generate the app default-deny + # baseline here and the auth-mode/host-namespace app policies do not apply (they scope to + # app-namespaces). Keycloak is still an ordinary workload under Cilium's cluster-wide posture + # (docs/15); the app->Keycloak / proxy->Keycloak / Keycloak->DB flows are the explicit allows. diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml new file mode 100644 index 0000000..f156ef7 --- /dev/null +++ b/platform/keycloak/postgres-cluster.yaml @@ -0,0 +1,41 @@ +--- +# Keycloak's PostgreSQL — a CloudNativePG Cluster (CR of the CNPG CRDs installed by platform/cnpg). +# CARRIED BY platform-config (dependsOn platform) so the postgresql.cnpg.io CRDs exist first (F-MON1). +# +# DR-critical (R-IDP-05): storage on the 3-replica `longhorn` StorageClass (ADR-0003) — the data +# survives node loss. CNPG creates the keycloak-db-app Secret (keys username/password) that the +# Keycloak CR consumes, plus the keycloak-db-rw/-ro/-r Services. Keycloak is stateless and rebuilds +# from this DB + the git-managed realm import (docs/15). +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: keycloak-db + namespace: keycloak +spec: + # Single Postgres instance on 3-replica Longhorn is acceptable for the home scale (ADR-0014: + # "single replica with DB on Longhorn + backups is acceptable"). Raise to 3 for Postgres-level HA + # as a future hardening (prod) — the storage is already 3-way replicated by Longhorn. + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:16.4 + storage: + storageClass: longhorn + size: 5Gi + bootstrap: + initdb: + database: keycloak + owner: keycloak + # Emit Postgres metrics so Prometheus scrapes DB health (R-OBS-01). The CNPG operator creates the + # PodMonitor at runtime (CRD already present from Phase-4 monitoring) — no kustomize CRD-ordering hazard. + monitoring: + enablePodMonitor: true + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + # --- Off-cluster DR backup (R-IDP-05) — OPERATOR HAND-OFF (OQ-BKP, shared with Velero, Phase 4) --- + # When the off-cluster backup target is chosen, add a Barman object store + a ScheduledBackup: + # spec.backup.barmanObjectStore = { destinationPath, endpointURL, s3Credentials {…SOPS…} } + # + a postgresql.cnpg.io/v1 ScheduledBackup (cron) referencing this Cluster. + # Until then the DB is protected by 3-replica Longhorn + (Phase-4) Velero PVC backups of the namespace. diff --git a/platform/keycloak/servicemonitor.yaml b/platform/keycloak/servicemonitor.yaml new file mode 100644 index 0000000..b5d2e63 --- /dev/null +++ b/platform/keycloak/servicemonitor.yaml @@ -0,0 +1,32 @@ +--- +# ServiceMonitor for Keycloak (R-OBS-01, docs/10). CR of monitoring.coreos.com (kube-prometheus-stack, +# Phase 4) — CARRIED BY platform-config so it is not applied in the same pass as the CRD-installing +# HelmRelease (F-MON1). Prometheus discovers ServiceMonitors in all namespaces +# (serviceMonitorSelectorNilUsesHelmValues=false). +# +# Keycloak 26 exposes health + Prometheus metrics on the MANAGEMENT interface (port 9000, path +# /metrics) once metrics-enabled/health-enabled are set (see keycloak-instance.yaml additionalOptions). +# We select the operator-created keycloak-service (which selects the Keycloak pods) and scrape the pods' +# management port by number via targetPort. +# TODO(WS-SEC, bring-up): confirm the management port number/name the operator exposes (9000 default) +# and adjust targetPort if the pinned operator changes it. The KeycloakDown / KeycloakDatabaseDown +# alerts (platform/monitoring/alerts.yaml) key on kube-state-metrics and fire regardless of this SM. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: keycloak + namespace: keycloak + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - keycloak + selector: + matchLabels: + app: keycloak + endpoints: + - targetPort: 9000 + path: /metrics + scheme: http + interval: 30s diff --git a/platform/keycloak/values.yaml b/platform/keycloak/values.yaml deleted file mode 100644 index f55e434..0000000 --- a/platform/keycloak/values.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Keycloak Helm values — PLACEHOLDER (Phase 0). Authoritative values inline in helmrelease.yaml. -# -# replicaCount: ${KEYCLOAK_REPLICAS} # overlay knob (dev 1 / prod 2 HA) -# postgresql on longhorn # DR-critical (ADR-0014), backed up off-cluster -# auth.existingSecret: keycloak-admin-credentials (SOPS-encrypted, WS-SEC — name only) -# realm/clients/groups as code # keycloak-config-cli / realm-import (Phase 6) -# public (hardened) exposure so login flows work for internet-exposed services (ADR-0014). -# Exceptions (do NOT force SSO): Plex/Seerr (Plex auth), Minecraft (Mojang), UniFi (native). diff --git a/platform/kustomization.yaml b/platform/kustomization.yaml index c9db73f..fdfd801 100644 --- a/platform/kustomization.yaml +++ b/platform/kustomization.yaml @@ -23,9 +23,13 @@ resources: - vpn-egress # CiliumEgressGatewayPolicy steering egress-class=vpn pods to the node tun0 (ADR-0015) # ---- Phase 4 — Observability (creds provisioned via SOPS: grafana-admin, alertmanager-smtp) ---- - monitoring # kube-prometheus-stack + Loki + Alloy + Alertmanager + # ---- Phase 6 — Identity / SSO (ADR-0014): foundational IdP, onboarded early. Keycloak Operator + + # CloudNativePG DB + oauth2-proxy forward-auth. Controllers here; the Keycloak/RealmImport/CNPG + # Cluster CRs + ServiceMonitors + HTTPRoutes are in platform-config (dependsOn platform). ---- + - cnpg # CloudNativePG operator (DB backend for Keycloak) — ghcr.io images (not Bitnami) + - keycloak # Keycloak Operator (quay.io images) + SOPS admin credential + - oauth2-proxy # forward-auth for non-OIDC UIs — needs SOPS `oauth2-proxy-credentials` (hand-off) # ---- Later phases: enabled when their phase opens ---- # - velero # Phase 4 (Backup/DR): BLOCKED on OQ-BKP (off-cluster backup target decision) # - image-automation # Phase 4 (Automated updates): Flux image-automation + Renovate # - kubevirt # Phase 5 (IaaS proof): KubeVirt + CDI - # - keycloak # Phase 6 (Catalog + C2 UI): foundational IdP — needs SOPS `keycloak-admin-credentials` - # - oauth2-proxy # Phase 6: forward-auth — needs SOPS `oauth2-proxy-credentials` diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index a622fec..434e4ab 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -305,9 +305,12 @@ spec: - name: platform-health.identity rules: - alert: KeycloakDown + # Scope to the Keycloak SERVER StatefulSet pods (keycloak-0, keycloak-1). The keycloak + # namespace also holds the operator (keycloak-operator-*) and DB (keycloak-db-*) pods, so an + # un-scoped max() would be masked by those being ready — this must reflect the IdP itself. expr: | - max(kube_pod_status_ready{namespace="keycloak", condition="true"}) == 0 - or absent(kube_pod_status_ready{namespace="keycloak"}) + max(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-[0-9]+", condition="true"}) == 0 + or absent(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-[0-9]+"}) for: 5m labels: severity: critical @@ -316,10 +319,11 @@ spec: description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)." runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-down" - alert: KeycloakDatabaseDown - # Keycloak's PostgreSQL (Longhorn-backed, DR-critical). Metric from postgres-exporter or - # the bundled chart; confirm label set at bring-up. + # Keycloak's PostgreSQL runs as a CloudNativePG Cluster `keycloak-db` (pods keycloak-db-N). + # DR-critical, Longhorn-backed. (CNPG also exports cnpg_* metrics via its PodMonitor.) expr: | - max(kube_pod_status_ready{namespace="keycloak", pod=~".*postgres.*", condition="true"}) == 0 + max(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-db-.*", condition="true"}) == 0 + or absent(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-db-.*"}) for: 5m labels: severity: critical diff --git a/platform/monitoring/grafana-httproute.yaml b/platform/monitoring/grafana-httproute.yaml new file mode 100644 index 0000000..3d7edec --- /dev/null +++ b/platform/monitoring/grafana-httproute.yaml @@ -0,0 +1,25 @@ +--- +# Ingress for Grafana at grafana.home.keithjackson.dev (docs/09) — the endpoint of the native-OIDC SSO +# proof (log in once at Keycloak -> reach Grafana). CR of the Gateway API CRDs — carried by +# platform-config. Same-namespace backend; parentRef allowed via the monitoring namespace's +# gateway-access=shared label. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: grafana + namespace: monitoring +spec: + parentRefs: + - name: shared + namespace: gateway + sectionName: https + hostnames: + - grafana.home.keithjackson.dev + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: kube-prometheus-stack-grafana + port: 80 diff --git a/platform/monitoring/grafana-oidc-credentials.sops.yaml.example b/platform/monitoring/grafana-oidc-credentials.sops.yaml.example new file mode 100644 index 0000000..75cb304 --- /dev/null +++ b/platform/monitoring/grafana-oidc-credentials.sops.yaml.example @@ -0,0 +1,26 @@ +# ============================================================================= +# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize and never +# auto-encrypted by SOPS). Reference scaffold + OPERATOR HAND-OFF. +# +# WHAT THIS IS: the OIDC client SECRET Grafana uses to authenticate to the Keycloak `homelab` realm's +# `grafana` client (kube-prometheus-stack.yaml -> grafana.envValueFrom.GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET). +# The value MUST equal the `grafana-client-secret` placed in platform/keycloak/keycloak-realm-clients +# (which the realm import injects into the Keycloak client). Consumed by name only (R-IDP-04). +# +# HAND-OFF STEPS (operator): +# (a) Use the SAME grafana-client-secret generated for keycloak-realm-clients (openssl rand -hex 32). +# (b) Copy to: platform/monitoring/grafana-oidc-credentials.sops.yaml +# (c) Fill `client-secret` below; encrypt: +# sops -e -i platform/monitoring/grafana-oidc-credentials.sops.yaml +# (d) Add under `resources:` in platform/monitoring/kustomization.yaml: +# - grafana-oidc-credentials.sops.yaml +# Grafana runs with local admin login until this lands (secretKeyRef optional:true), then OIDC works. +# ============================================================================= +apiVersion: v1 +kind: Secret +metadata: + name: grafana-oidc-credentials + namespace: monitoring +type: Opaque +stringData: + client-secret: REPLACE-ME diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index b42063f..9ae402c 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -150,8 +150,36 @@ spec: memory: 128Mi limits: memory: 256Mi - # TODO(WS-SEC/WS-C2, later): Keycloak OIDC client (auth.generic_oauth) + root_url behind the - # Gateway; disable local login once SSO is proven. Secret referenced by name only. + # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) --- + # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes + # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only, + # R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the + # secret — local admin login (below) stays available as the fallback until SSO is proven. + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-oidc-credentials + key: client-secret + optional: true + grafana.ini: + server: + root_url: https://grafana.home.keithjackson.dev + auth.generic_oauth: + enabled: true + name: Keycloak + client_id: grafana + scopes: "openid email profile" + auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth + token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token + api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo + login_attribute_path: preferred_username + email_attribute_path: email + # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins -> + # Admin, media-users -> Editor, everyone else -> Viewer. + role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'" + # Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to + # disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05). + # TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified. # --- Bundled exporters / operator: keep small --- prometheusOperator: diff --git a/platform/monitoring/namespace.yaml b/platform/monitoring/namespace.yaml index ff7ae1b..afd9813 100644 --- a/platform/monitoring/namespace.yaml +++ b/platform/monitoring/namespace.yaml @@ -3,3 +3,7 @@ apiVersion: v1 kind: Namespace metadata: name: monitoring + labels: + # Expose Grafana at grafana.home.keithjackson.dev via the shared Gateway (docs/09) for the + # Keycloak native-OIDC SSO proof (grafana-httproute.yaml, WS-SEC Phase 6). + gateway-access: "shared" diff --git a/platform/oauth2-proxy/helmrelease.yaml b/platform/oauth2-proxy/helmrelease.yaml index 8514306..1057fd6 100644 --- a/platform/oauth2-proxy/helmrelease.yaml +++ b/platform/oauth2-proxy/helmrelease.yaml @@ -1,7 +1,8 @@ --- -# oauth2-proxy — forward-auth at the Gateway for apps WITHOUT usable OIDC (PiHole, *arr UIs, -# qBittorrent, Homepage, Prometheus, Alertmanager, Hubble UI). Authenticates against Keycloak. -# ADR-0014. Phase 0 STUB. Replica count is an overlay knob (R-ENV-02). +# oauth2-proxy — forward-auth for apps WITHOUT usable OIDC (PiHole, *arr UIs, qBittorrent, Homepage, +# Prometheus, Alertmanager, Hubble UI — ADR-0014). Authenticates users against the Keycloak `homelab` +# realm, then the Gateway ext-auth lets the request through only for an authenticated session +# (R-IDP-02: no management UI unauthenticated). Chart image is quay.io/oauth2-proxy (not Bitnami). apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -26,13 +27,32 @@ spec: values: replicaCount: ${OAUTH2_PROXY_REPLICAS} # overlay knob: dev 1 / prod 2 (HA) — R-ENV-02 config: - # OIDC provider = Keycloak. Client id/secret + cookie secret: SOPS-encrypted secret, name only. + # client-id / client-secret / cookie-secret from the SOPS secret (name only). client-id + + # client-secret MUST match the realm's `oauth2-proxy` client (keycloak-realm-clients secret). existingSecret: oauth2-proxy-credentials extraArgs: provider: oidc - # TODO(WS-SEC, Phase 6): oidc-issuer-url = https://keycloak.home.keithjackson.dev/realms/ - oidc-issuer-url: "REPLACE-ME" + oidc-issuer-url: https://keycloak.home.keithjackson.dev/realms/homelab + # Forward-auth (ext-auth) mode: no upstream app; return 202 for an authenticated session so the + # Gateway ext-authz can allow/deny. Auth headers are surfaced to the backend. + reverse-proxy: "true" + upstream: static://202 + set-xauthrequest: "true" + pass-access-token: "true" email-domain: "*" - # Forward-auth integration wired at the Gateway (HTTPRoute + ext-auth). Contract with WS-NET. - # TODO(WS-SEC/WS-NET, Phase 6): Gateway forward-auth/ext-authz wiring; group-based authorization; - # MFA enforced upstream in Keycloak. Management UIs must never be unauthenticated (R-IDP-02). + # Single sign-in/callback host for all forward-auth-protected services on the platform zone. + redirect-url: https://auth.home.keithjackson.dev/oauth2/callback + whitelist-domain: ".home.keithjackson.dev" + cookie-domain: ".home.keithjackson.dev" + cookie-secure: "true" + # Expose group membership so per-service group authorization (R-IDP-06) can be enforced. The + # baseline here is "authenticated"; per-service `--allowed-groups` is layered on at the Gateway + # ext-auth attachment (WS-NET contract) or via a dedicated oauth2-proxy instance per group. + oidc-groups-claim: groups + scope: "openid email profile" + # Metrics for the ServiceMonitor (../oauth2-proxy servicemonitor in platform-config). + metrics: + enabled: true + # TODO(WS-SEC/WS-NET, Phase 6): the Gateway ext-authz ATTACHMENT (which HTTPRoutes require this + # proxy, and each route's allowed groups) is a WS-NET Gateway contract — Cilium Gateway API + # ext-auth wiring. oauth2-proxy is the authenticator; the per-route enforcement is Gateway-side. diff --git a/platform/oauth2-proxy/httproute.yaml b/platform/oauth2-proxy/httproute.yaml new file mode 100644 index 0000000..b41d4ab --- /dev/null +++ b/platform/oauth2-proxy/httproute.yaml @@ -0,0 +1,25 @@ +--- +# Ingress for oauth2-proxy's sign-in/callback endpoint at auth.home.keithjackson.dev (docs/09). This +# is where forward-auth-protected UIs redirect unauthenticated users to log in against Keycloak. CR of +# the Gateway API CRDs — carried by platform-config. Same-namespace backend (no ReferenceGrant); the +# parentRef is allowed via the oauth2-proxy namespace's gateway-access=shared label. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: oauth2-proxy + namespace: oauth2-proxy +spec: + parentRefs: + - name: shared + namespace: gateway + sectionName: https + hostnames: + - auth.home.keithjackson.dev + rules: + - matches: + - path: + type: PathPrefix + value: /oauth2 + backendRefs: + - name: oauth2-proxy + port: 80 diff --git a/platform/oauth2-proxy/namespace.yaml b/platform/oauth2-proxy/namespace.yaml index 5b602ca..e1396b4 100644 --- a/platform/oauth2-proxy/namespace.yaml +++ b/platform/oauth2-proxy/namespace.yaml @@ -3,3 +3,7 @@ apiVersion: v1 kind: Namespace metadata: name: oauth2-proxy + labels: + # Opt in to the shared Gateway so oauth2-proxy's sign-in/callback endpoint is reachable at + # auth.home.keithjackson.dev (docs/09). Platform component — NOT app-namespace (see keycloak ns note). + gateway-access: "shared" diff --git a/platform/oauth2-proxy/servicemonitor.yaml b/platform/oauth2-proxy/servicemonitor.yaml new file mode 100644 index 0000000..bbca5b2 --- /dev/null +++ b/platform/oauth2-proxy/servicemonitor.yaml @@ -0,0 +1,23 @@ +--- +# ServiceMonitor for oauth2-proxy (R-OBS-01, docs/10). CR of monitoring.coreos.com — CARRIED BY +# platform-config (F-MON1). The chart's metrics endpoint is enabled in the helmrelease (metrics.enabled). +# TODO(WS-SEC, bring-up): confirm the metrics port NAME the pinned chart exposes on the Service +# (commonly `metrics`, :44180) and adjust `port` below if it differs. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: oauth2-proxy + namespace: oauth2-proxy + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - oauth2-proxy + selector: + matchLabels: + app: oauth2-proxy + endpoints: + - port: metrics + path: /metrics + interval: 30s From 4bebd4b6d2b88aaca8874ddd897fd7aa1f4567a5 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 21:07:27 -0500 Subject: [PATCH 2/3] keycloak: coverage tests + dashboard + reword cross-cutting tags --- .../chainsaw-test.yaml | 70 +++++++++++++++ .../keycloak-dashboard/chainsaw-test.yaml | 49 +++++++++++ .../keycloak-db-dr/chainsaw-test.yaml | 58 +++++++++++++ .../keycloak-monitored/chainsaw-test.yaml | 81 ++++++++++++++++++ .../oidc-sso-grafana/chainsaw-test.yaml | 85 +++++++++++++++++++ platform/keycloak/grafana-dashboard.yaml | 73 ++++++++++++++++ .../keycloak-realm-clients.sops.yaml.example | 2 +- platform/keycloak/keycloak-realm-import.yaml | 2 +- platform/keycloak/kustomization.yaml | 2 + platform/monitoring/alerts.yaml | 4 +- 10 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml create mode 100644 conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml create mode 100644 platform/keycloak/grafana-dashboard.yaml diff --git a/conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml new file mode 100644 index 0000000..d9b6963 --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-config-as-code/chainsaw-test.yaml @@ -0,0 +1,70 @@ +# ============================================================================= +# L2 BEHAVIORAL — identity config as code (R-IDP-03, R-IDP-04, R-IDP-06) [RUNNABLE] +# Spec: docs/15 identity-and-access; ADR-0014. Impl: platform/keycloak/keycloak-realm-import.yaml. +# +# GIVEN the `homelab` realm (groups platform-admins/media-users/viewers, OIDC clients, brute-force, +# and the conditional MFA browser flow) is declared as a KeycloakRealmImport in git, +# WHEN the Keycloak Operator reconciles it +# -> THEN the import reaches Done=True — realm/groups/clients/roles + the MFA flow are applied +# PURELY from git (R-IDP-03 config-as-code; R-IDP-04 MFA flow bound; R-IDP-06 groups/roles for +# least-privilege). No console clicking. +# +# Goes GREEN once Keycloak is up + the realm client-secret hand-off (keycloak-realm-clients) is filled +# so the placeholder resolves and the import completes. chainsaw v0.2.12: script + block contains check. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-config-as-code + labels: + conformance.home.k3s/R-IDP-03: "true" + conformance.home.k3s/R-IDP-04: "true" + conformance.home.k3s/R-IDP-06: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04, R-IDP-06" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + The homelab realm (groups, clients, MFA flow) reconciles to Done purely from the git-managed + KeycloakRealmImport — no manual console configuration. +spec: + timeouts: + exec: 5m + steps: + - name: realm-import-reconciles-from-git + try: + - script: + content: | + set -u + # Poll the KeycloakRealmImport until the operator reports Done=True. + for i in $(seq 1 30); do + st="$(kubectl -n keycloak get keycloakrealmimport homelab \ + -o jsonpath='{.status.conditions[?(@.type=="Done")].status}' 2>/dev/null || true)" + echo "poll $i: Done=$st" + if [ "$st" = "True" ]; then + echo "REALM_IMPORT_DONE_OK" + break + fi + sleep 10 + done + check: + (contains($stdout, 'REALM_IMPORT_DONE_OK')): true + - name: privileged-group-and-mfa-flow-present + try: + - script: + content: | + set -u + # The realm import declares the browser-mfa flow + platform-admins group. Confirm the + # realm CR / import spec carries them (declared-in-git evidence; the live realm state is + # exercised by the login flows). + flow="$(kubectl -n keycloak get keycloakrealmimport homelab \ + -o jsonpath='{.spec.realm.browserFlow}' 2>/dev/null || true)" + grp="$(kubectl -n keycloak get keycloakrealmimport homelab \ + -o jsonpath='{.spec.realm.groups[*].name}' 2>/dev/null || true)" + echo "browserFlow=$flow groups=$grp" + echo "$flow" | grep -q 'browser-mfa' || { echo "FAIL: MFA browser flow not bound"; exit 1; } + echo "$grp" | grep -q 'platform-admins' || { echo "FAIL: platform-admins group missing"; exit 1; } + echo "MFA_AND_GROUPS_OK" + check: + (contains($stdout, 'MFA_AND_GROUPS_OK')): true diff --git a/conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml new file mode 100644 index 0000000..81c694b --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-dashboard/chainsaw-test.yaml @@ -0,0 +1,49 @@ +# ============================================================================= +# L2 BEHAVIORAL — IdP health is visible on a dashboard (R-OBS-03) [RUNNABLE] +# Spec: docs/10 observability. Impl: platform/keycloak/grafana-dashboard.yaml (a grafana_dashboard +# ConfigMap auto-provisioned into Grafana by the sidecar). +# +# GIVEN a Keycloak health dashboard is declared as a grafana_dashboard ConfigMap, +# WHEN Grafana's sidecar discovers it (searchNamespace: ALL) +# -> THEN the dashboard is provisioned and Grafana is up — Keycloak health is visible on a dashboard +# (R-OBS-03). The presence of the labelled ConfigMap + a ready Grafana is the reconciliation proof. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-dashboard + labels: + conformance.home.k3s/R-OBS-03: "true" + annotations: + conformance.home.k3s/requirements: "R-OBS-03" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + A grafana_dashboard ConfigMap for Keycloak health exists and Grafana is up to render it. +spec: + timeouts: + assert: 3m + steps: + - name: keycloak-dashboard-configmap-provisioned + try: + - assert: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: keycloak-dashboard + namespace: keycloak + labels: + grafana_dashboard: "1" + - name: grafana-is-up-to-render-it + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kube-prometheus-stack-grafana + namespace: monitoring + status: + availableReplicas: 1 diff --git a/conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml new file mode 100644 index 0000000..9691bb0 --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-db-dr/chainsaw-test.yaml @@ -0,0 +1,58 @@ +# ============================================================================= +# L2 BEHAVIORAL — IdP datastore is DR-critical (R-IDP-05) [RUNNABLE] +# Spec: docs/15 identity-and-access; ADR-0014. Impl: platform/keycloak/postgres-cluster.yaml +# (CloudNativePG Cluster on the 3-replica `longhorn` StorageClass). +# +# GIVEN Keycloak's PostgreSQL runs as a CloudNativePG Cluster, +# WHEN we inspect it +# -> THEN it is healthy (>=1 ready instance) AND its data lives on the replicated `longhorn` +# StorageClass — i.e. the identity datastore is DR-critical / replicated (R-IDP-05). Keycloak is +# stateless and rebuilds from this DB + the git realm import. +# (Full off-cluster backup/restore is R-STATE-03/Velero territory, hand-off OQ-BKP.) +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-db-dr + labels: + conformance.home.k3s/R-IDP-05: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-05" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Keycloak's CloudNativePG database is healthy and stored on the 3-replica longhorn StorageClass. +spec: + timeouts: + exec: 5m + steps: + - name: db-cluster-is-healthy + try: + - script: + content: | + set -u + for i in $(seq 1 30); do + ready="$(kubectl -n keycloak get clusters.postgresql.cnpg.io keycloak-db \ + -o jsonpath='{.status.readyInstances}' 2>/dev/null || true)" + echo "poll $i: readyInstances=$ready" + if [ "${ready:-0}" -ge 1 ]; then echo DB_READY_OK; break; fi + sleep 10 + done + check: + (contains($stdout, 'DB_READY_OK')): true + - name: db-data-is-on-replicated-longhorn + try: + - script: + content: | + set -u + sc="$(kubectl -n keycloak get pvc -l cnpg.io/cluster=keycloak-db \ + -o jsonpath='{.items[0].spec.storageClassName}' 2>/dev/null || true)" + echo "db pvc storageClass=$sc" + if [ "$sc" = "longhorn" ]; then + echo DB_ON_LONGHORN_OK + else + echo "FAIL: Keycloak DB PVC not on the replicated longhorn StorageClass"; exit 1 + fi + check: + (contains($stdout, 'DB_ON_LONGHORN_OK')): true diff --git a/conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml b/conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml new file mode 100644 index 0000000..e4b2230 --- /dev/null +++ b/conformance/tests/l2-behavioral/keycloak-monitored/chainsaw-test.yaml @@ -0,0 +1,81 @@ +# ============================================================================= +# L2 BEHAVIORAL — the IdP is monitored (R-OBS-01) [RUNNABLE] +# Spec: docs/10 observability. Impl: platform/keycloak/servicemonitor.yaml (Keycloak ServiceMonitor, +# carried via platform-config per the F-MON1 split) + the CNPG PodMonitor. +# +# GIVEN a ServiceMonitor for Keycloak, +# WHEN Prometheus reconciles its scrape config +# -> THEN Prometheus carries a `serviceMonitor/keycloak` scrape pool in its active targets — the IdP +# is monitored (metrics/health), R-OBS-01. (The KeycloakDown / KeycloakDatabaseDown alerts in +# platform/monitoring/alerts.yaml consume this signal + kube-state-metrics.) +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: keycloak-monitored + labels: + conformance.home.k3s/R-OBS-01: "true" + annotations: + conformance.home.k3s/requirements: "R-OBS-01" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Prometheus discovers the Keycloak ServiceMonitor and carries it as an active scrape pool. +spec: + timeouts: + apply: 60s + assert: 3m + exec: 3m + steps: + - name: keycloak-servicemonitor-exists + try: + - assert: + resource: + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: keycloak + namespace: keycloak + - name: prometheus-carries-the-keycloak-scrape-pool + try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: keycloak-target-probe + spec: + backoffLimit: 3 + activeDeadlineSeconds: 150 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36.1 + command: + - sh + - -c + - | + set -u + prom="http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090" + for i in $(seq 1 12); do + out="$(wget -q -O - "$prom/api/v1/targets" 2>/dev/null || true)" + if echo "$out" | grep -q 'serviceMonitor/keycloak'; then + echo KEYCLOAK_TARGET_PRESENT_OK; exit 0 + fi + echo "poll $i: keycloak scrape pool not yet present"; sleep 10 + done + echo "FAIL: Prometheus has no serviceMonitor/keycloak scrape pool"; exit 1 + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: keycloak-target-probe + status: + succeeded: 1 + finally: + - script: + content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"' diff --git a/conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml b/conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml new file mode 100644 index 0000000..ac92c8c --- /dev/null +++ b/conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml @@ -0,0 +1,85 @@ +# ============================================================================= +# L2 BEHAVIORAL — native-OIDC SSO via the central IdP (R-IDP-01) [RUNNABLE] +# Spec: docs/15 identity-and-access; ADR-0014. Impl: Grafana as a Keycloak OIDC client +# (platform/monitoring/kube-prometheus-stack.yaml auth.generic_oauth) — the self-contained SSO proof. +# +# GIVEN Grafana is deployed and configured as an OIDC client of the Keycloak `homelab` realm, +# WHEN an unauthenticated browser starts the OIDC login (GET /login/generic_oauth) +# -> THEN Grafana redirects (302) to the CENTRAL IdP's authorize endpoint on +# keycloak.home.keithjackson.dev/realms/homelab — proving SSO is driven by the one IdP +# (R-IDP-01: central Identity Provider as the single user directory). +# +# Goes GREEN once the grafana-oidc client-secret hand-off is filled and Keycloak + Grafana are up. +# chainsaw v0.2.12: Job + status assert; no test-level finally. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: oidc-sso-grafana + labels: + conformance.home.k3s/R-IDP-01: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-01" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Grafana's OIDC login redirects to the central Keycloak realm — SSO is driven by the one IdP. +spec: + timeouts: + apply: 60s + assert: 3m + exec: 3m + steps: + - name: given-grafana-is-an-oidc-client + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kube-prometheus-stack-grafana + namespace: monitoring + status: + availableReplicas: 1 + - name: then-login-redirects-to-keycloak-realm + try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: grafana-oidc-probe + spec: + backoffLimit: 2 + activeDeadlineSeconds: 150 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36.1 + command: + - sh + - -c + - | + set -u + svc="http://kube-prometheus-stack-grafana.monitoring.svc.cluster.local" + out="$(wget -S --max-redirect=0 -O /dev/null "$svc/login/generic_oauth" 2>&1 || true)" + echo "$out" + if echo "$out" | grep -qi 'keycloak.home.keithjackson.dev/realms/homelab'; then + echo GRAFANA_OIDC_REDIRECT_OK + else + echo "FAIL: Grafana did not redirect to the Keycloak homelab realm"; exit 1 + fi + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: grafana-oidc-probe + status: + succeeded: 1 + finally: + - script: + content: 'echo "teardown: chainsaw deletes the per-test namespace (Job included)"' diff --git a/platform/keycloak/grafana-dashboard.yaml b/platform/keycloak/grafana-dashboard.yaml new file mode 100644 index 0000000..d1d701f --- /dev/null +++ b/platform/keycloak/grafana-dashboard.yaml @@ -0,0 +1,73 @@ +--- +# Keycloak health dashboard (R-OBS-03, docs/10 — "issues and health visible on dashboards"). A +# ConfigMap labelled grafana_dashboard is auto-discovered by the Grafana sidecar (searchNamespace: +# ALL, label grafana_dashboard — see platform/monitoring/kube-prometheus-stack.yaml) and provisioned +# into Grafana. Applied by the `platform` Kustomization (plain ConfigMap, no CRD → no split needed). +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-dashboard + namespace: keycloak + labels: + grafana_dashboard: "1" +data: + keycloak-health.json: | + { + "uid": "keycloak-health", + "title": "Keycloak — Identity / SSO health", + "tags": ["keycloak", "identity", "platform-health"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-6h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Keycloak pods ready", + "gridPos": {"h": 6, "w": 8, "x": 0, "y": 0}, + "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"keycloak\", pod=~\"keycloak-[0-9]+\", condition=\"true\"})", "legendFormat": "ready"} + ] + }, + { + "id": 2, + "type": "stat", + "title": "Keycloak DB (CNPG) pods ready", + "gridPos": {"h": 6, "w": 8, "x": 8, "y": 0}, + "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"keycloak\", pod=~\"keycloak-db-.*\", condition=\"true\"})", "legendFormat": "db ready"} + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Keycloak scrape target up", + "gridPos": {"h": 6, "w": 8, "x": 16, "y": 0}, + "targets": [ + {"expr": "up{namespace=\"keycloak\"}", "legendFormat": "{{pod}}"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "JVM heap used (Keycloak)", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "targets": [ + {"expr": "sum by (pod) (jvm_memory_used_bytes{namespace=\"keycloak\", area=\"heap\"})", "legendFormat": "{{pod}}"} + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "HTTP requests (Keycloak, rate)", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "targets": [ + {"expr": "sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[5m]))", "legendFormat": "req/s"} + ] + } + ] + } diff --git a/platform/keycloak/keycloak-realm-clients.sops.yaml.example b/platform/keycloak/keycloak-realm-clients.sops.yaml.example index a098e1f..ff0bbc5 100644 --- a/platform/keycloak/keycloak-realm-clients.sops.yaml.example +++ b/platform/keycloak/keycloak-realm-clients.sops.yaml.example @@ -5,7 +5,7 @@ # WHAT THIS IS: the OIDC client secrets for the two confidential clients the realm import # (keycloak-realm-import.yaml) provisions — grafana and oauth2-proxy. The realm import injects # these values via spec.placeholders ($(env:GRAFANA_CLIENT_SECRET) / $(env:OAUTH2_PROXY_CLIENT_SECRET)), -# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-SEC-01 / R-IDP-04). +# so the client secrets live ONLY here (SOPS-encrypted), never in the realm YAML (R-IDP-04; docs/07). # # HAND-OFF STEPS (operator): # (a) Generate two client secrets: diff --git a/platform/keycloak/keycloak-realm-import.yaml b/platform/keycloak/keycloak-realm-import.yaml index 026a4d0..894844f 100644 --- a/platform/keycloak/keycloak-realm-import.yaml +++ b/platform/keycloak/keycloak-realm-import.yaml @@ -4,7 +4,7 @@ # declaratively by the Keycloak Operator; CARRIED BY platform-config (dependsOn platform). # # Client secrets are NOT in this file: they are injected from the SOPS-encrypted keycloak-realm-clients -# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-SEC-01 / R-IDP-04). +# Secret via spec.placeholders ($(env:...)) — so secrets exist only encrypted (R-IDP-04; SOPS, docs/07). # Until that Secret is filled (operator hand-off, see keycloak-realm-clients.sops.yaml.example), this # import stays PENDING on the unresolved placeholder. apiVersion: k8s.keycloak.org/v2alpha1 diff --git a/platform/keycloak/kustomization.yaml b/platform/keycloak/kustomization.yaml index b5db95b..d07e761 100644 --- a/platform/keycloak/kustomization.yaml +++ b/platform/keycloak/kustomization.yaml @@ -26,3 +26,5 @@ resources: - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml # --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) --- - keycloak-admin-credentials.sops.yaml + # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) --- + - grafana-dashboard.yaml diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index 434e4ab..5ae3901 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -226,7 +226,7 @@ spec: severity: critical annotations: summary: "Egress VPN tunnel down" - description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (R-SEC-03 leak risk)." + description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)." runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down" # (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun # retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd @@ -241,7 +241,7 @@ spec: severity: warning annotations: summary: "Spike in policy-denied network flows" - description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (R-SEC-03)." + description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)." runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows" # ===================================================================================== From cb4fdf4a551fd2cb92594fe159b9820a8603f93e Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 21:10:39 -0500 Subject: [PATCH 3/3] =?UTF-8?q?keycloak:=20README=20=E2=80=94=20operator+C?= =?UTF-8?q?NPG=20design,=20runbook,=20hand-offs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/keycloak/README.md | 67 ++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/platform/keycloak/README.md b/platform/keycloak/README.md index 4602085..54aad4d 100644 --- a/platform/keycloak/README.md +++ b/platform/keycloak/README.md @@ -1,30 +1,65 @@ # platform/keycloak -**Keycloak** — the platform Identity Provider (SSO via OIDC + SAML). +**Keycloak** — the platform Identity Provider (SSO via OIDC + SAML), onboarded early (Phase 6 IdP). - **Governing ADR:** [ADR-0014 — IdP: Keycloak](../../../home-network-k3s/decisions/adr-0014-idp-keycloak.md) - **Spec:** [docs/15 — Identity & Access](../../../home-network-k3s/docs/15-identity-and-access.md) - **Serves:** R-IDP-01..06 - **Owner:** WS-SEC -## What this stub is +## Image / tooling strategy (F-KY1 landmine avoided) -Flux `HelmRelease` for the Keycloak instance with **PostgreSQL on Longhorn** (DR-critical). Replica -count is an overlay knob (`${KEYCLOAK_REPLICAS}`, R-ENV-02). Admin/DB credentials referenced **by -name**, **SOPS-encrypted** (WS-SEC). Realms/clients/groups are config-as-code (Phase 6). +The original stub used the **Bitnami Keycloak chart + bundled Bitnami PostgreSQL**. Bitnami's free +Docker Hub images were purged (DECISIONS-LOG **F-KY1**) → that path ImagePullBackOffs. Replaced with: -> The ADR allows the **Keycloak Operator OR Helm** for the instance; this stub uses Helm — swap to -> the Operator if WS-SEC prefers CR-driven config. +- **Keycloak: the official Keycloak Operator** (`quay.io/keycloak/keycloak-operator` + + `quay.io/keycloak/keycloak`, v26.0.6) driving a **`Keycloak` CR** + a **`KeycloakRealmImport` CR** for + config-as-code (ADR-0014 permits Operator OR Helm; Operator gives clean CR-driven realm reconcile). +- **Database: CloudNativePG** (`ghcr.io/cloudnative-pg`, see [`../cnpg`](../cnpg)) — a `Cluster` on the + 3-replica **`longhorn`** StorageClass (DR-critical, R-IDP-05), with declarative backup hooks. -## Contract (SSO/auth) +quay.io + ghcr.io are unaffected by the Bitnami purge. Operator manifests are pinned upstream release +files pulled as remote kustomize bases (the kubevirt-stub pattern); Renovate maintains the version. -The "SSO/auth contract" (`roadmap/workstreams.md`, WS-SEC): a ServiceDefinition's `auth` block maps to -a Keycloak client (native OIDC) or oauth2-proxy protection (forward-auth — see -[`../oauth2-proxy`](../oauth2-proxy)). Consumed by WS-APPS/WS-C2. +## Layout (controllers / configs split — F-MON1) -## TODO (later phases — onboarded EARLY in Phase 6) +| File | Applied by | What | +|---|---|---| +| `kustomization.yaml` (namespace, operator CRDs+Deployment, admin secret, dashboard) | **platform** | controllers | +| `keycloak-instance.yaml` (`Keycloak` CR, `${KEYCLOAK_REPLICAS}`) | **platform-config** | config | +| `postgres-cluster.yaml` (CNPG `Cluster` on longhorn) | **platform-config** | config | +| `keycloak-realm-import.yaml` (`KeycloakRealmImport`: realm/groups/clients/MFA) | **platform-config** | config | +| `httproute.yaml` (keycloak.home.keithjackson.dev) | **platform-config** | config | +| `servicemonitor.yaml` (Prometheus scrape) | **platform-config** | config | -- Realms/clients/groups as code (keycloak-config-cli / realm-import); MFA for privileged groups. -- **DR-critical:** off-cluster DB backup (velero/) + documented **break-glass** local-admin path. -- Exceptions that keep native auth (do not force SSO): Plex/Seerr (Plex auth), Minecraft - (Mojang/Microsoft), UniFi (native) — ADR-0014. +Realm config (groups `platform-admins`/`media-users`/`viewers`, OIDC clients `grafana`+`oauth2-proxy`, +brute-force ON, and a **conditional MFA browser flow that REQUIRES OTP for platform-admins**, R-IDP-04) +reconciles purely from git — no console clicking (R-IDP-03). + +## Admin credential + break-glass (R-IDP-04 / R-IDP-05) + +The provisioned `keycloak-admin-credentials.sops.yaml` (keys `admin-user`/`admin-password`) is injected +as the temporary **bootstrap/break-glass admin** via `KC_BOOTSTRAP_ADMIN_*` env (podTemplate escape +hatch) — this consumes the secret **as provisioned, without re-keying it** (the operator's native +`bootstrapAdmin` wants `username`/`password` keys, which would require editing the encrypted file). +The Keycloak bootstrap admin is the documented break-glass path so an IdP/SSO outage does not lock the +operator out (ADR-0014); Grafana likewise keeps its local admin login until SSO is proven. + +## Operator hand-offs (fill before / during deploy) + +1. **`keycloak-realm-clients.sops.yaml`** (keycloak ns) — from `.sops.yaml.example`. Two client + secrets (`grafana-client-secret`, `oauth2-proxy-client-secret`), `openssl rand -hex 32` each. The + realm import injects them into the Keycloak clients via `spec.placeholders`. Add to + `kustomization.yaml` once filled. The realm import stays PENDING until this exists. +2. **`../oauth2-proxy/oauth2-proxy-credentials.sops.yaml`** (oauth2-proxy ns) — `client-id: oauth2-proxy`, + `client-secret` = the SAME `oauth2-proxy-client-secret` above, `cookie-secret` = `openssl rand -base64 32`. +3. **`../monitoring/grafana-oidc-credentials.sops.yaml`** (monitoring ns) — `client-secret` = the SAME + `grafana-client-secret` above. + +(Cross-namespace Secrets can't be shared, so each client secret is stored twice — once for the realm +import in `keycloak`, once for the consuming app.) See the two `.sops.yaml.example` templates for the +exact keys + steps. No plaintext secret is ever committed. + +## Exceptions that keep native auth (do not force SSO) — ADR-0014 + +Plex/Seerr (Plex auth), Minecraft (Mojang/Microsoft), UniFi (native).