merge: R2 off-cluster backups + per-environment backup credentials

2026-09-07 01:55:14 -05:00
commit 3ca4ad6011
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: keycloak-db-backup-credentials-prod
+    namespace: keycloak
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:kQ5x5fXb6KdVaDsItGug2cpQMZw9tSvbsYDqTMxoP+A=,iv:dCTfwVxlpnjaMhGBIMa18d4yAiQ6SKhb4EMm1p1JyQ8=,tag:sNqMLs+rGQm/1MyV90InDQ==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:9lGvwvseAWK1bMXDXZJ+bjw4oHH/XVPi8+C8vfXGJ3jX8VW95rJrczPbEfV9rCudVIJFnxAdaQZZWaJCsC8fCQ==,iv:GOvPs2I77yiKh381bcv0kCwT5xbk8Fao6ctPCW2aCHg=,tag:hSjWjaZ+nEJwZyv3sIuq7A==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBN2NvVXJ5TW1mQlhYWnRC
+            UlVLVjFtNk1Wdkk1NzdicXBQVGNtc3JZclNBCjl3UlVTZmFJWGJGam54VytVR0lZ
+            djB3ZjBCOFFNOUFMZGxCcnZiZ3VDMUUKLS0tIFhDSWR3ZXZPOHFzVGNhZGhqdUQv
+            WmF0NlNIeTJCeVV0ZGkxSFJIL3BqNlEKntEpxCdpy7Xd16ZKe6skQYL6hCqYOAIN
+            NtwtQ9oF/8i4i0T7B8kcORB71cu8dPu6w6LtaSC+1qBV5TBSSH5U6Q==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-09-07T06:36:33Z"
+    mac: ENC[AES256_GCM,data:dcgB6z9BDzaft4do6f1SqjCqmQopc5mM0ckfU/HfE09eOSIRjpoZ+qMzJW8RIwUZNKrjEAwrrAEtrN67GRxL0jOEsw/6xNrWJRFv6KTeE3ECy5qMO4jLCQTeMvZuvfZE7iqwvHCCnjhkqkedEAD4Hwefq0c0EZ0ZxZK3z1tVdHk=,iv:Ue+ZJq9dJl9qbjJ1l8Q24VmsK+KW/uSRFu04xG6/tM8=,tag:MvJ6Ww3h3b/C2PqA/62abA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4
@@ -28,9 +28,17 @@ resources:
  - keycloak-admin-credentials.sops.yaml
  # --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders ---
  - keycloak-realm-clients.sops.yaml
-  # --- Scoped MinIO S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05) — SOPS-encrypted,
-  #     decrypted by the `platform` tier (which already has the decryption stanza). Lands in ns
-  #     keycloak; the keycloak-db Cluster (platform-config) reads it by name for WAL archiving. ---
+  # --- PER-ENVIRONMENT S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05), split
+  #     2026-09-07. SOPS-encrypted, decrypted by the `platform` tier (which already has the
+  #     decryption stanza). BOTH land in ns keycloak on BOTH clusters; the keycloak-db Cluster
+  #     (platform-config) selects ONE by name via
+  #     `keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
+  #     prod "-prod"). Two objects, so an R2 token rotation cannot silently break dev's MinIO
+  #     WAL archiving — and archive_mode=on makes a broken archive a disk-full clock, not a
+  #     degraded backup (see postgres-cluster.yaml).
+  #       dev  -> keycloak-db-backup-credentials       (MinIO, s3://cnpg)
+  #       prod -> keycloak-db-backup-credentials-prod  (Cloudflare R2, s3://k3s-prod-backup/barman)
  - keycloak-db-backup-credentials.sops.yaml
+  - keycloak-db-backup-credentials-prod.sops.yaml
  # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
  - grafana-dashboard.yaml
@@ -34,25 +34,43 @@ spec:
      memory: 256Mi
    limits:
      memory: 512Mi
-  # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
+  # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → object storage ---
  # Continuous WAL archiving + periodic base backups give proper Postgres PITR (superior to a Velero
-  # PVC snapshot for a live DB). destinationPath is the BUCKET ROOT; CNPG appends serverName (defaults
-  # to the Cluster name), so objects land under s3://cnpg/keycloak-db/. Path-style HTTP MinIO endpoint.
+  # PVC snapshot for a live DB). CNPG appends `serverName` (defaults to the Cluster name) to
+  # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/keycloak-db/ — dev
+  # s3://cnpg/keycloak-db/ on the on-Proxmox MinIO, prod s3://k3s-prod-backup/barman/keycloak-db/ on
+  # Cloudflare R2. Both endpointURL and destinationPath are cluster-vars knobs (R-ENV-02),
+  # substituted by the `platform-config` Kustomization (this file sits under platform/keycloak/ but
+  # platform-config is the only tier that lists it as a resource — see its kustomization.yaml).
  # Credentials: the scoped key from the in-namespace SOPS Secret keycloak-db-backup-credentials
-  # (a Pod only reads same-ns Secrets), kept in sync with platform/backup/minio-credentials.sops.yaml.
-  # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000) — prod
-  # MUST point at its own off-cluster target. Substituted by the `platform-config` Kustomization.
+  # (a Pod only reads same-ns Secrets).
+  #
+  # *** archive_mode IS ON, SO A FAILING ARCHIVE COMMAND IS A DISK-FULL CLOCK, NOT A DEGRADED
+  # BACKUP. *** Postgres will not recycle a WAL segment it has not successfully archived, so
+  # whenever these two values are wrong the pgdata PVC fills monotonically until the primary
+  # PANICs. This is not hypothetical: prod ran from bootstrap to 2026-09-07 with
+  # `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT` (barman-cloud-wal-archive exit 4 on every segment)
+  # and had burned ~579 MiB of a 5 GiB PVC on the way to taking SSO — and with it kubectl OIDC,
+  # Grafana, Headlamp, Gitea and forward-auth — down with it. Never leave a placeholder here, and
+  # treat `ContinuousArchiving=False` on this Cluster as an availability incident, not a backup one.
  backup:
    retentionPolicy: "30d"          # keep 30 days of base backups + WAL (PITR window); prunes older
    barmanObjectStore:
-      destinationPath: s3://cnpg
+      destinationPath: ${CNPG_BACKUP_PATH}
      endpointURL: ${S3_BACKUP_ENDPOINT}
+      # PER-ENVIRONMENT credential (2026-09-07): dev reads keycloak-db-backup-credentials (MinIO),
+      # prod reads keycloak-db-backup-credentials-prod (Cloudflare R2). Both Secrets are applied to
+      # both clusters by platform/keycloak/kustomization.yaml; ${BACKUP_CREDENTIALS_SUFFIX} (dev ""
+      # | prod "-prod") picks one. Before the split these were ONE shared Secret, so writing prod's
+      # R2 token into it would have left dev signing MinIO requests with an R2 key — a 403 on every
+      # archive command, which under archive_mode=on is the disk-full clock described above, not a
+      # missing backup.
      s3Credentials:
        accessKeyId:
-          name: keycloak-db-backup-credentials
+          name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
          key: ACCESS_KEY_ID
        secretAccessKey:
-          name: keycloak-db-backup-credentials
+          name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
          key: ACCESS_SECRET_KEY
      wal:
        compression: gzip