merge: R2 off-cluster backups + per-environment backup credentials

apps/wave-0/kustomization.yaml

@@ -11,6 +11,7 @@ resources:
 - jackett-oauth2-proxy-credentials.sops.yaml
 - jackett.yaml
 - nextcloud-db-app.sops.yaml
+- nextcloud-db-backup-credentials-prod.sops.yaml
 - nextcloud-db-backup-credentials.sops.yaml
 - nextcloud-db.yaml
 - pihole-oauth2-proxy-credentials.sops.yaml
@@ -19,5 +20,6 @@ resources:
 - qbittorrent-oauth2-proxy-credentials.sops.yaml
 - qbittorrent.yaml
 - semaphore-db-app.sops.yaml
+- semaphore-db-backup-credentials-prod.sops.yaml
 - semaphore-db-backup-credentials.sops.yaml
 - semaphore-db.yaml

apps/wave-0/nextcloud-db-backup-credentials-prod.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: nextcloud-db-backup-credentials-prod
+    namespace: nextcloud-db
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:4lx6et4HcXDegulyIoa7jZR/lVgapGPEkukajfCKjvk=,iv:p6V7tizbikk55tOvpPOsPHTe/p52QLX5A5xrwuB14rA=,tag:oNJuipT/Lb6zynJ2K2Q71A==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:V5bHPoPeBOVyLp1n1Jilyso4i5P2aPSH0hrryfweg2q/SctWZxRbLBQ/FLzOigbJsAhlwOGu069mQPma6xjfdw==,iv:8u4UOvI8dC/cm6mEWDlyN5E3SfRZmwKY9uZY/DjPEsQ=,tag:tw7tGs29TmUvMPBCtCszAA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiSUt5N2k2N2lDU2ZsMy9m
+            M1NFS0M2QjFsNFhnR0hCMzFyaTdiQWFZRFJZCmhBTWRXNmxLMGtsc3Rxd2FSMDZp
+            aktjcTE2c1hmU1EzM3RWNksyb3Q1YVUKLS0tIDBFS1I5aDBER2cvcWtLZ3BXTEp5
+            ZGI5ZmZIcStheUNQamxIVVVpc0ZMOHMKlE5aHewtHJpeU24F5sodc9CEm9KTjHQQ
+            8ThEQ/8vyazL9V4ojlICxSHU/SHkHpD1eBaTKcAQd/LZzujIcJ+QfA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-09-07T06:36:33Z"
+    mac: ENC[AES256_GCM,data:8awz1Jh8t4zjDMyd93/v+6on9iiIaSi01293A7uYhxj9L1+g63zeO+DPmSn6+KbZvHdqU2A7up09H1NzfXAEdGuvVbB36cdGq0zKank3PUi8c7IT6qNuWydxRAcqLnbbD+c30Z6GB+gAuZHstNOe4Rgc4aJvMN6FuP6x6Z0870c=,iv:KGGKJq2QA5xztvoHs/SViXW72DYwT16i+W+dJBUeIJQ=,tag:3UpYqx2+RIK0QdMH/BSIpA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

apps/wave-0/nextcloud-db.yaml

@@ -68,25 +68,38 @@ spec:
       memory: 256Mi
     limits:
       memory: 512Mi
-  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
+  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
   # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
-  # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
-  # (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP.
-  # Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials
-  # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
-  # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
-  # substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
+  # backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
+  # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/nextcloud-db/ — dev
+  # s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod
+  # s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and
+  # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
+  # Credentials: PER-ENVIRONMENT (2026-09-07). dev reads nextcloud-db-backup-credentials (the
+  # scoped MinIO key), prod reads nextcloud-db-backup-credentials-prod (the Cloudflare R2
+  # token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
+  # generated golden — render.py places the -prod sidecar in this DB's wave); the
+  # cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
+  # one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
+  # ONE shared Secret until the split, so giving prod its R2 token would have left dev
+  # signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
+  # disk-full clock below rather than a merely missing backup.
+  #
+  # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
+  # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
+  # the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
+  # endpoint. Treat ContinuousArchiving=False here as an availability incident.
   backup:
     retentionPolicy: "30d"
     barmanObjectStore:
-      destinationPath: s3://cnpg
+      destinationPath: ${CNPG_BACKUP_PATH}
       endpointURL: ${S3_BACKUP_ENDPOINT}
       s3Credentials:
         accessKeyId:
-          name: nextcloud-db-backup-credentials
+          name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
           key: ACCESS_KEY_ID
         secretAccessKey:
-          name: nextcloud-db-backup-credentials
+          name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
           key: ACCESS_SECRET_KEY
       wal:
         compression: gzip

apps/wave-0/semaphore-db-backup-credentials-prod.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: semaphore-db-backup-credentials-prod
+    namespace: semaphore-db
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:OhWEEYqrvmDB40NmiETgeUisBvaO9ocevCZAsurNijw=,iv:kEjOIg9NHbRYm+Ul1WrBAzb3gLbIv+ipiYCxFK0QoVc=,tag:KAB1JAIw0ATwbjDEKZVT9A==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:C0PNKv07LCYpFONLqd55sz+ioyU/i2YJUkbZ8OX/dCErM/DilqchyUAT5EqnT/r3YJmdaA1pX85S3fxlVVHkbg==,iv:4ha28b+xIebvX2JPITmtTovIe4vdmj2XfI3cgkUHH4c=,tag:xbMyflcHojytPzgaoOYcXQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZVTJ3cXR6cks3Y0RNS2hx
+            K3pjZEduYUxmL0V3SmFUcU9XbGVHSVFITWh3Cjl1eHlTLytoOW5PaVNFVkJoeTBo
+            T1FnRHNDVzJ6dWpuR09WaElvbWF4YU0KLS0tIEJoQ1VsM1cxRlg0R1dtNGdOenNn
+            aVNjMjVBTytLdHlFdmZCS2tPelo1Y2MKYt1G5t+sDiHgcg+Q9ECBzJn2gS/paSEC
+            7aoX60SBrPgHvnGls5lzc3Tin9nH4jWYl6Nu8DAvIE0YPXDFr5a0UA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-09-07T06:36:33Z"
+    mac: ENC[AES256_GCM,data:mhVP7tjdZK2FR2imvpJbTyT7BigvcIKEpLSnBS1oSITUirB8xZC7aHBfyFuIoKOrpOJgEe40nHUdDYCHRJtb4dLeVja8+7meVaQWzxC8BWVHsksW+73hsqhjAzlHeobx6Irbk4PAMqIY0/jnf4MHryjnRn3dUuwu9U1CqL30JbE=,iv:57oCcp6FW9EAIVKnYGtkaTrnbijyv54T+vmZxMbMqnw=,tag:agrRP/JLypU60KOeN0s/5w==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

apps/wave-0/semaphore-db.yaml

@@ -69,25 +69,38 @@ spec:
       memory: 256Mi
     limits:
       memory: 512Mi
-  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
+  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
   # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
-  # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
-  # (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP.
-  # Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials
-  # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
-  # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
-  # substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
+  # backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
+  # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/semaphore-db/ — dev
+  # s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod
+  # s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and
+  # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
+  # Credentials: PER-ENVIRONMENT (2026-09-07). dev reads semaphore-db-backup-credentials (the
+  # scoped MinIO key), prod reads semaphore-db-backup-credentials-prod (the Cloudflare R2
+  # token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
+  # generated golden — render.py places the -prod sidecar in this DB's wave); the
+  # cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
+  # one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
+  # ONE shared Secret until the split, so giving prod its R2 token would have left dev
+  # signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
+  # disk-full clock below rather than a merely missing backup.
+  #
+  # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
+  # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
+  # the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
+  # endpoint. Treat ContinuousArchiving=False here as an availability incident.
   backup:
     retentionPolicy: "30d"
     barmanObjectStore:
-      destinationPath: s3://cnpg
+      destinationPath: ${CNPG_BACKUP_PATH}
       endpointURL: ${S3_BACKUP_ENDPOINT}
       s3Credentials:
         accessKeyId:
-          name: semaphore-db-backup-credentials
+          name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
           key: ACCESS_KEY_ID
         secretAccessKey:
-          name: semaphore-db-backup-credentials
+          name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
           key: ACCESS_SECRET_KEY
       wal:
         compression: gzip

catalog/render.py

@@ -1321,6 +1321,11 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
       - `<svc>-db.yaml`                     -> wave(svc-db) (a strictly-lower wave; CNPG DB)
       - `<svc>-db-app.sops.yaml`            -> wave(svc-db) (CNPG initdb owner secret)
       - `<svc>-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret)
+      - `<svc>-db-backup-credentials-prod.sops.yaml` -> wave(svc-db) (the PROD half of that
+        secret: the two environments carry DIFFERENT object-store credentials, selected in
+        the Cluster CR by `${BACKUP_CREDENTIALS_SUFFIX}`. Both objects ship to both clusters,
+        so both must be placed in the DB wave — the Secret has to exist before the Cluster
+        that names it.)
     Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
     Only files that actually exist in `present_files` are placed, so the plan is robust to a
     service having no bootstrap secret or no DB. Deterministic (sorted).
@@ -1348,6 +1353,9 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
             add(wdb, f"{db}.yaml")
             add(wdb, f"{db}-app.sops.yaml")
             add(wdb, f"{db}-backup-credentials.sops.yaml")   # CNPG Barman S3 creds (R-STATE-03)
+            # PROD half of the same credential (2026-09-07 per-env split). Placed only when
+            # the file exists, so a DB that has not been split is unaffected.
+            add(wdb, f"{db}-backup-credentials-prod.sops.yaml")
 
     for unit, cfg in EXTRA_UNITS.items():
         anchor = cfg["anchor"]
@@ -1830,6 +1838,7 @@ def _selftest() -> int:
     present = {
         "a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
         "svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
+        "svc-db-backup-credentials-prod.sops.yaml",
         "gitea.yaml",
         "gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
     }
@@ -1839,8 +1848,10 @@ def _selftest() -> int:
     flat = {f: wv for wv, fs in plan.items() for f in fs}
     check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
     check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
-          and flat.get("svc-db-backup-credentials.sops.yaml") == 0,
-          "CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave")
+          and flat.get("svc-db-backup-credentials.sops.yaml") == 0
+          and flat.get("svc-db-backup-credentials-prod.sops.yaml") == 0,
+          "CNPG DB manifest + its owner secret + BOTH halves of its per-env Barman backup secret "
+          "must be in the DB's (lower) wave")
     check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
           "a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
     check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")

clusters/base/backup.yaml

@@ -34,6 +34,9 @@ spec:
     substituteFrom:
       - kind: ConfigMap
         name: cluster-vars
-  # SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per
-  # environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how
-  # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml.
+  # SOPS decryption for the `cloud` INI credentials — velero-cloud.sops.yaml (dev's scoped MinIO key)
+  # and velero-cloud-prod.sops.yaml (prod's Cloudflare R2 token) — is layered per environment by the
+  # overlay (clusters/dev and clusters/prod patch THIS Kustomization by name), identical to how
+  # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. Both Secrets are applied on both
+  # clusters; ${BACKUP_CREDENTIALS_SUFFIX} (substituted from cluster-vars above) decides which one the
+  # velero Deployment actually mounts. See clusters/dev/kustomization.yaml.

clusters/base/cluster-vars.yaml

@@ -96,9 +96,22 @@ data:
   #
   # Substitution coverage (verify before adding a knob here): every path that consumes these declares
   # postBuild.substituteFrom cluster-vars — platform (K3S_API_HOST), platform-config
-  # (CILIUM_LB_POOL_*, S3_BACKUP_ENDPOINT), crafty (CRAFTY_L4_VIP), apps-wave-0 (S3_BACKUP_ENDPOINT),
-  # velero (S3_BACKUP_ENDPOINT), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an UNSUBSTITUTED path would
-  # ship the literal string into the cluster.
+  # (CILIUM_LB_POOL_*, and keycloak-db's S3_BACKUP_ENDPOINT + CNPG_BACKUP_PATH: the Cluster lives at
+  # platform/keycloak/postgres-cluster.yaml but is rendered by platform-config, which is the ONLY
+  # tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db:
+  # S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH), velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION,
+  # VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an
+  # UNSUBSTITUTED path would ship the literal string into the cluster.
+  #
+  # ADDING A KNOB IS SAFE UNDER StrictPostBuildSubstitutions ONLY BECAUSE OF THE ROOT EDGE. An
+  # UNDEFINED variable is a HARD Kustomization failure, so a manifest that references a knob the
+  # live ConfigMap does not yet carry breaks its whole tier. `clusters/base/platform.yaml` gained
+  # `dependsOn: [flux-system]` for exactly this: dependsOn is REVISION-gated, so no substituting
+  # tier can reconcile revision N until the ROOT Kustomization has APPLIED revision N — i.e. until
+  # the new cluster-vars key EXISTS. Every substituting consumer chains transitively through
+  # `platform`, so one edge covers the graph, and a new knob plus its first reference may ship in
+  # ONE commit. (A hand-run `kubectl apply -k clusters/<env>` bypasses that ordering entirely: apply
+  # the overlay BEFORE the tier that references the knob, or the tier fails until it retries.)
 
   # k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
   #
@@ -150,9 +163,21 @@ data:
   # Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
   # Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
   # nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port.
-  # Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR.
+  # Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR —
+  # and, measured 2026-09-05 from a pod INSIDE prod, it CANNOT reach it anyway (prod is on the SOHO
+  # network 192.168.2.0/24, which is pfSense's WAN, while this MinIO sits inside pfSense's LAN:
+  # `curl http://192.168.1.9:9000/minio/health/live` timed out after 15s). Prod uses Cloudflare R2.
   S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"
 
+  # S3 region. Every S3 client REQUIRES one; MinIO ignores whatever it is; Cloudflare R2 demands
+  # exactly `auto` (its `head-bucket` reports the physical hint `WNAM`, which is NOT a usable region
+  # string). This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-
+  # accident for MinIO and wrong for anything else, which is why it is a knob now. The base default
+  # is dev's LIVE value, deliberately: dev's Velero has been signing requests with `us-east-1`
+  # against that MinIO since bootstrap, and this repo's rule is that base == the known-good dev
+  # value. Prod overrides to `auto`.
+  S3_BACKUP_REGION: "us-east-1"
+
   # --- storage (ADR-0003) ---
   LONGHORN_DEFAULT_REPLICA_COUNT: "3"       # default StorageClass replica count (3 on 3 nodes)
   # Longhorn's defaultDataPath (platform/longhorn/helmrelease.yaml): the on-NODE directory every
@@ -205,6 +230,99 @@ data:
   VELERO_BACKUP_SCHEDULE: "0 3 * * *"        # cron; overlays may differ
   VELERO_RETENTION: "168h"                   # dev short | prod long
 
+  # ==========================================================================================
+  # OBJECT-STORE TENANCY — the three knobs that keep two clusters and two backup tools from
+  # writing into each other's data. THESE ARE SAFETY KNOBS, not preferences.
+  # ==========================================================================================
+  # Nothing about the object layout was per-environment before this: Velero's bucket was the
+  # hardcoded literal `velero` with no prefix, and every CNPG Cluster used the hardcoded
+  # `s3://cnpg` with barman's `serverName` defaulting to the CLUSTER NAME — and BOTH environments
+  # have a keycloak-db, a nextcloud-db and a semaphore-db. Point two clusters at one bucket under
+  # that layout and:
+  #   (a) each cluster's Velero ADOPTS the other's backups and TTL-DELETES them on retention; and
+  #   (b) two different Postgres servers stream WAL into ONE Barman prefix, which does not merely
+  #       mix files — it CORRUPTS THE CHAIN and destroys PITR for both. That is the worse half:
+  #       dev's DR restore is PROVEN (R-STATE-03) and this would silently un-prove it.
+  #
+  # WHY THERE IS NO SINGLE `S3_BACKUP_BUCKET`. The two consumers take differently-SHAPED values:
+  # the Velero chart wants `bucket` and `prefix` as separate fields, while CNPG's
+  # barmanObjectStore.destinationPath is ONE full s3:// URL whose first path segment IS the bucket.
+  # CNPG's bucket therefore already lives inside CNPG_BACKUP_PATH; adding a second knob that had to
+  # hold the same bucket name a second time would be a rotation waiting to drift — the exact class
+  # of duplication this repo has been bitten by. Three knobs, each shaped for its own consumer, and
+  # no two of them can contradict each other.
+  #
+  # BASE = DEV'S LIVE LAYOUT, BYTE-FOR-BYTE, AND MUST STAY THAT WAY. Dev's existing backups are at
+  # `s3://velero/` (bucket root, no prefix) and `s3://cnpg/<clusterName>/` on that MinIO. Giving dev
+  # a prefix, or renaming its buckets, would ORPHAN the very backups its proven restore depends on —
+  # the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default` and its LONGHORN_DATA_PATH stays
+  # /var/lib/longhorn. Verified: with an EMPTY prefix the Velero chart omits `objectStorage.prefix`
+  # from the rendered BackupStorageLocation entirely (helm template, chart 12.1.0), so dev's live
+  # BSL is unchanged by this parameterization rather than merely equivalent to it.
+  #
+  # PROD, which has no history to preserve, gets ONE bucket with TWO disjoint top-level prefixes:
+  #
+  #     s3://k3s-prod-backup/velero/…                 Velero (kopia repo + backup metadata)
+  #     s3://k3s-prod-backup/barman/<clusterName>/…   CNPG WAL + base backups
+  #
+  # The operator created exactly one R2 bucket, so velero-vs-barman separation has to come from the
+  # prefix rather than the bucket. The BUCKET ROOT IS DELIBERATELY LEFT EMPTY: if a prefix ever
+  # renders wrong, its objects land somewhere obviously bogus instead of silently colliding with a
+  # real tenant's data. Dev↔prod isolation is doubly independent — different endpoint AND different
+  # bucket — so neither cluster can reach the other's objects even with a mis-substituted prefix.
+  VELERO_BACKUP_BUCKET: "velero"             # dev: the `velero` MinIO bucket | prod: k3s-prod-backup
+  VELERO_BACKUP_PREFIX: ""                   # dev: bucket root (live layout) | prod: "velero"
+  CNPG_BACKUP_PATH: "s3://cnpg"              # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman
+
+  # ------------------------------------------------------------------------------------------------
+  # WHICH CREDENTIAL SECRET THE BACKUP CONSUMERS READ (added 2026-09-07 — the per-env credential split)
+  # ------------------------------------------------------------------------------------------------
+  # The three knobs above say WHERE the backups go. This one says WHICH KEY signs the requests, and
+  # it exists because the endpoint moved per-environment while the credential did not.
+  #
+  # THE BUG IT FIXES. The four backup-credential Secrets — velero-cloud.sops.yaml and the three
+  # <db>-backup-credentials.sops.yaml — are SHARED BASE manifests: platform/ and apps/ are rendered
+  # by the same Flux tiers on both clusters, and cluster-vars substitution is the ONLY thing that
+  # differs (R-ENV-02). So writing prod's R2 token into those four files did not GIVE prod a
+  # credential, it REPLACED DEV'S: dev's Velero and all three dev CNPG Clusters would have started
+  # signing MinIO requests with an R2 key id. For Velero that is a broken BSL; for CNPG it is worse,
+  # because `archive_mode=on` means Postgres REFUSES TO RECYCLE a WAL segment it could not archive,
+  # so a 403 loop is a monotonically filling pgdata PVC ending in a PANIC — the same failure prod
+  # spent its whole bring-up in with a placeholder endpoint. One shared Secret also means rotating
+  # either environment's key silently breaks the other; that is the structural reason for the split,
+  # independent of dev's eventual decommission.
+  #
+  # THE MECHANISM. Each credential is now TWO Secret objects, both applied to BOTH clusters (they are
+  # base resources; nothing here forks a manifest per environment). The consumer names one:
+  #
+  #     velero:   credentials.existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}
+  #     CNPG:     s3Credentials.*.name:  <db>-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
+  #
+  # ONE knob, not four, ON PURPOSE. Four independent secret-name knobs could be updated three-
+  # quarters of the way, leaving one consumer pointed at the OTHER environment's key with nothing
+  # rendering wrong — and StrictPostBuildSubstitutions cannot catch that, because it only fails on an
+  # UNDEFINED variable, never on a defined-but-stale one. With a single suffix the four move together
+  # or not at all. The cost is that the rendered Secret name is not greppable from the consumer; the
+  # names are spelled out in full in each kustomization.yaml beside the resources.
+  #
+  # THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING. It renders the UNSUFFIXED names,
+  # which are the objects that already exist and that the live dev cluster is running. A default
+  # naming a Secret that does not exist would be worse than the shared-Secret state it replaces:
+  # Velero's pod would sit in ContainerCreating forever on a missing `existingSecret` volume, and a
+  # CNPG Cluster whose s3Credentials Secret is absent stops archiving WAL — i.e. the disk-full clock
+  # again — so an unresolvable default breaks the referencing cluster HARDER than a wrong key does.
+  # Empty is also the value dev must carry, so base and dev agree, exactly like VELERO_BACKUP_PREFIX.
+  #
+  # An EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions: strict mode fails on a variable that is
+  # not DEFINED, not on one defined as "". VELERO_BACKUP_PREFIX above has shipped empty since the
+  # gate was enabled and proves it. Both overlays state this knob explicitly — dev "" and prod
+  # "-prod" — so neither inherits it silently.
+  #
+  # ADDING A NEW BACKUP CREDENTIAL: create BOTH halves (`x-credentials.sops.yaml` and
+  # `x-credentials-prod.sops.yaml`), list both in the kustomization, and reference
+  # `x-credentials${BACKUP_CREDENTIALS_SUFFIX}`. Never a single shared Secret again.
+  BACKUP_CREDENTIALS_SUFFIX: ""              # dev: "" (MinIO creds) | prod: "-prod" (R2 creds)
+
   # --- placement ---
   PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os"   # base no-op selector; prod may pin roles
   PLATFORM_NODE_SELECTOR_VALUE: "linux"

clusters/dev/kustomization.yaml

@@ -106,8 +106,11 @@ patches:
           secretRef:
             name: sops-age
   # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml).
-  # It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS
-  # decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only).
+  # It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped
+  # MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption
+  # stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by
+  # BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is
+  # out-of-band (name only).
   - patch: |-
       apiVersion: kustomize.toolkit.fluxcd.io/v1
       kind: Kustomization
@@ -224,9 +227,56 @@ patches:
         value: "192.168.1.55"
       # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
       # barmanObjectStore and Velero's backupStorageLocation write to.
+      #
+      # DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07).
+      # Prod could not use it at all — it cannot even route to it — but dev's move is a separate,
+      # operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write
+      # a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window
+      # would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo
+      # (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod.
       - op: replace
         path: /data/S3_BACKUP_ENDPOINT
         value: "http://192.168.1.9:9000"
+      # DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same
+      # reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base
+      # defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones
+      # its proven restore depends on — without any manifest looking wrong.
+      #
+      # `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO
+      # ignores the value, but changing it is a needless risk against a live, working backup path.
+      # An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already
+      # are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified
+      # against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this.
+      - op: replace
+        path: /data/S3_BACKUP_REGION
+        value: "us-east-1"
+      - op: replace
+        path: /data/VELERO_BACKUP_BUCKET
+        value: "velero"
+      - op: replace
+        path: /data/VELERO_BACKUP_PREFIX
+        value: ""
+      - op: replace
+        path: /data/CNPG_BACKUP_PATH
+        value: "s3://cnpg"
+      # WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets
+      # (velero-cloud-credentials, <db>-backup-credentials) — the scoped MinIO keys dev's Velero and
+      # its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod
+      # renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead.
+      #
+      # RESTATED EXPLICITLY even though it equals the base default, for the same reason as
+      # EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED
+      # base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev
+      # signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on
+      # turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR
+      # restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential.
+      #
+      # DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator,
+      # 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING
+      # backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs.
+      - op: replace
+        path: /data/BACKUP_CREDENTIALS_SUFFIX
+        value: ""
       # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas.
       - op: replace
         path: /data/LONGHORN_DEFAULT_REPLICA_COUNT

clusters/prod/kustomization.yaml

@@ -392,10 +392,91 @@ patches:
       - op: replace
         path: /data/CRAFTY_L4_VIP
         value: "192.168.2.22"
-      # Off-cluster DR target. Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the
-      # network — a prod backup written there is not off-cluster DR and would mix two environments'
-      # PITR streams in one bucket. Operator supplies the prod endpoint (full URL incl. scheme+port).
+      # ==========================================================================================
+      # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
+      # ==========================================================================================
+      # Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the network and is NOT an
+      # option for prod, for two independent reasons:
+      #   1. UNREACHABLE. Prod is on the SOHO network 192.168.2.0/24 — pfSense's WAN — while that
+      #      MinIO is inside pfSense's LAN. Measured from a pod INSIDE prod (the workstation proves
+      #      nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` → connection
+      #      timed out after 15s. Reaching it would need a WAN→LAN rule that does not exist.
+      #   2. NOT OFF-CLUSTER DR. It runs on the same Proxmox host as dev's VMs, so it already shares
+      #      a physical failure domain with dev. R2 is genuinely off-site, and its free egress means
+      #      restore REHEARSALS cost nothing — R-STATE-03 has no financial disincentive to stay proven.
+      #
+      # THIS UNBLOCKS AN ACTIVE INCIDENT, not just a roadmap item. All three prod CNPG Clusters ship
+      # `archive_mode=on` with `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT`, so barman-cloud-wal-archive
+      # exits 4 on every segment and Postgres — correctly — REFUSES TO RECYCLE WAL. Verified live
+      # 2026-09-07: keycloak-db / nextcloud-db / semaphore-db all report
+      # ContinuousArchiving=False/ContinuousArchivingFailing, and keycloak-db's 5 GiB PVC is at
+      # 579 MiB used / 4360 MiB free and climbing monotonically. A disk-full PANIC there takes the
+      # cluster's SSO database down, and with it kubectl OIDC, Grafana, Headlamp, Gitea and
+      # forward-auth. Fixing the endpoint lets Postgres drain the whole backlog.
+      #
+      # ENDPOINT IS ACCOUNT-SCOPED, so it stays a prod-overlay value even though it is not a LAN
+      # address: this is the operator's Cloudflare account id, confirmed from inside a prod pod
+      # (`head-bucket` on k3s-prod-backup → BucketRegion WNAM; PUT and DELETE both succeed).
+      # It is NOT put in base, because base is dev's live MinIO and dev is deliberately STAYING
+      # there — moving dev to R2 would reset its PITR window and orphan the backups its PROVEN
+      # restore depends on. Two environments, two targets, isolated by URL *and* by bucket.
       - op: replace
         path: /data/S3_BACKUP_ENDPOINT
-        value: "TBD-PROD-S3-BACKUP-ENDPOINT"
+        value: "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com"
+      # R2 requires the region string to be exactly `auto`. (`head-bucket` answers
+      # `{"BucketRegion":"WNAM"}` — that is a PHYSICAL LOCATION HINT, not a signing region; using it
+      # here produces AuthorizationHeaderMalformed.) Dev keeps the base `us-east-1` its live MinIO
+      # has always been signed with.
+      - op: replace
+        path: /data/S3_BACKUP_REGION
+        value: "auto"
+      # ------------------------------------------------------------------------------------------
+      # OBJECT TENANCY INSIDE THE ONE BUCKET. The operator created a SINGLE R2 bucket, so velero and
+      # barman cannot be separated by bucket the way they are on dev's MinIO (`velero` and `cnpg`).
+      # They are separated by disjoint TOP-LEVEL PREFIXES instead:
+      #
+      #     s3://k3s-prod-backup/velero/…                 Velero: kopia repo + backup metadata
+      #     s3://k3s-prod-backup/barman/<clusterName>/…   CNPG: WAL + base backups
+      #
+      # The bucket ROOT is left empty on purpose — a misrendered prefix then lands somewhere
+      # obviously wrong instead of silently colliding with a live tenant. Velero's retention TTL
+      # deletes objects under its own prefix only, so it can never reap a WAL segment; and barman's
+      # per-cluster serverName subdirectory keeps the three Postgres servers' chains apart, which
+      # matters because interleaving two servers' WAL into one prefix corrupts the chain outright.
+      # Dev cannot be reached from any of these: different endpoint AND different bucket names.
+      # ------------------------------------------------------------------------------------------
+      - op: replace
+        path: /data/VELERO_BACKUP_BUCKET
+        value: "k3s-prod-backup"
+      - op: replace
+        path: /data/VELERO_BACKUP_PREFIX
+        value: "velero"
+      - op: replace
+        path: /data/CNPG_BACKUP_PATH
+        value: "s3://k3s-prod-backup/barman"
+      # ------------------------------------------------------------------------------------------
+      # WHICH CREDENTIAL SIGNS THE REQUESTS TO THAT BUCKET (per-env credential split, 2026-09-07).
+      # ------------------------------------------------------------------------------------------
+      # The knobs above moved the ENDPOINT and the OBJECT LAYOUT per environment. The credential did
+      # not move with them, and it could not: the four backup-credential Secrets are SHARED BASE
+      # manifests, applied to both clusters by the same Flux tiers. Writing the R2 token into them
+      # gave prod a working key by TAKING DEV'S AWAY — dev would have signed MinIO requests with an
+      # R2 key id, and under archive_mode=on that is a filling pgdata PVC, not a missing backup.
+      #
+      # So each credential is now two objects, and this suffix selects prod's:
+      #     velero-cloud-credentials-prod           the R2 token in the AWS INI `cloud` shape
+      #     <db>-backup-credentials-prod            the same token as ACCESS_KEY_ID / ACCESS_SECRET_KEY
+      # for keycloak-db, nextcloud-db and semaphore-db. Both halves ship to both clusters (they are
+      # base resources); ONLY this value decides which one each consumer reads. It is a SUFFIX rather
+      # than four secret-name knobs deliberately — four could be updated three-quarters of the way,
+      # leaving one consumer on dev's MinIO key with nothing rendering wrong, and strict-mode
+      # substitution cannot catch a defined-but-stale variable.
+      #
+      # THE R2 TOKEN IS THE ONE PROVEN FROM INSIDE PROD (chore/r2-credentials 0a6a862): head-bucket on
+      # k3s-prod-backup returns BucketRegion WNAM, and PUT and DELETE both succeed. Rotating it now
+      # touches ONLY the four `-prod` files; dev's MinIO keys are untouched objects. That isolation is
+      # the structural reason for the split, independent of dev's eventual decommission.
+      - op: replace
+        path: /data/BACKUP_CREDENTIALS_SUFFIX
+        value: "-prod"
       # TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist.

infra/PHASE1-STATUS.md

@@ -84,6 +84,9 @@ toolchain: yamllint, check-jsonschema, kyverno, chainsaw, yq) lives in the image
 infra/devcontainer/dev.sh build                    # once
 infra/devcontainer/dev.sh run make -C ci gate      # exactly what CI runs
 infra/devcontainer/dev.sh                          # interactive shell at /work
+
+infra/devcontainer/dev.sh run kubectl get nodes                # DEV (the default)
+CLUSTER=prod infra/devcontainer/dev.sh run kubectl get nodes   # PROD
 ```
 
 Unlike the libvirt era, **tofu and ansible now run fine inside the container** — the
@@ -95,10 +98,23 @@ which is what broke the old `dmacvicar/libvirt` provider in a container.
 file, and the sibling spec repo read-only at `/home-network-k3s` so the spec-immutability
 check runs instead of silently skipping.
 
-> **Known gap.** `dev.sh` defaults `KUBECONFIG` to `/work/infra/ansible/.cache/dev.kubeconfig`,
-> a libvirt-era path that no longer exists, and does not mount `~/.kube`. In-container `kubectl`
-> therefore needs an explicit `DEV_KUBECONFIG=<path to a break-glass cert kubeconfig>` — the OIDC
-> context cannot work there anyway (no browser). Not yet fixed.
+> **Closed (was a known gap): `dev.sh` can now target either cluster.** It used to hard-wire
+> `KUBECONFIG` to `dev.kubeconfig`, leaving an undocumented — and misleadingly named —
+> `DEV_KUBECONFIG=<prod path>` as the only route to prod. Select with **`CLUSTER=dev|prod`**;
+> **dev remains the default**, so every existing invocation means what it always meant. Inside the
+> container `KUBECONFIG` is a chain of a tiny per-invocation selector file over a merged config
+> carrying both clusters as contexts **`k3s-dev`** and **`k3s-prod`** (never `default`), regenerated
+> each run by [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
+> `infra/ansible/.cache/clusters.kubeconfig`. The merged file deliberately carries **no**
+> `current-context`: it is shared by every container, and stamping the current cluster into it made
+> concurrent `CLUSTER=dev` and `CLUSTER=prod` runs silently swap clusters mid-command. Naming matters: k3s writes *both* source kubeconfigs
+> with context, cluster and user all called `default`, and kubectl's merge keeps the first
+> duplicate — so a plain `KUBECONFIG=dev:prod` would silently resolve `default` to **dev** while you
+> believed you were on prod. Requesting a cluster whose kubeconfig is absent is a hard error, never
+> a fallback. `DEV_KUBECONFIG` still works and still wins, but is **superseded** — it is opaque
+> about its target and yields an ambiguous `default` context. `~/.kube` is still not mounted and the
+> OIDC context still cannot work in a container (no browser); use the break-glass cert kubeconfigs
+> above. Details: [`devcontainer/README.md`](devcontainer/README.md).
 
 ## Off-repo state you must not lose
 

platform/keycloak/keycloak-db-backup-credentials-prod.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: keycloak-db-backup-credentials-prod
+    namespace: keycloak
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:kQ5x5fXb6KdVaDsItGug2cpQMZw9tSvbsYDqTMxoP+A=,iv:dCTfwVxlpnjaMhGBIMa18d4yAiQ6SKhb4EMm1p1JyQ8=,tag:sNqMLs+rGQm/1MyV90InDQ==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:9lGvwvseAWK1bMXDXZJ+bjw4oHH/XVPi8+C8vfXGJ3jX8VW95rJrczPbEfV9rCudVIJFnxAdaQZZWaJCsC8fCQ==,iv:GOvPs2I77yiKh381bcv0kCwT5xbk8Fao6ctPCW2aCHg=,tag:hSjWjaZ+nEJwZyv3sIuq7A==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBN2NvVXJ5TW1mQlhYWnRC
+            UlVLVjFtNk1Wdkk1NzdicXBQVGNtc3JZclNBCjl3UlVTZmFJWGJGam54VytVR0lZ
+            djB3ZjBCOFFNOUFMZGxCcnZiZ3VDMUUKLS0tIFhDSWR3ZXZPOHFzVGNhZGhqdUQv
+            WmF0NlNIeTJCeVV0ZGkxSFJIL3BqNlEKntEpxCdpy7Xd16ZKe6skQYL6hCqYOAIN
+            NtwtQ9oF/8i4i0T7B8kcORB71cu8dPu6w6LtaSC+1qBV5TBSSH5U6Q==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-09-07T06:36:33Z"
+    mac: ENC[AES256_GCM,data:dcgB6z9BDzaft4do6f1SqjCqmQopc5mM0ckfU/HfE09eOSIRjpoZ+qMzJW8RIwUZNKrjEAwrrAEtrN67GRxL0jOEsw/6xNrWJRFv6KTeE3ECy5qMO4jLCQTeMvZuvfZE7iqwvHCCnjhkqkedEAD4Hwefq0c0EZ0ZxZK3z1tVdHk=,iv:Ue+ZJq9dJl9qbjJ1l8Q24VmsK+KW/uSRFu04xG6/tM8=,tag:MvJ6Ww3h3b/C2PqA/62abA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

platform/keycloak/kustomization.yaml

@@ -28,9 +28,17 @@ resources:
   - keycloak-admin-credentials.sops.yaml
   # --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders ---
   - keycloak-realm-clients.sops.yaml
-  # --- Scoped MinIO S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05) — SOPS-encrypted,
-  #     decrypted by the `platform` tier (which already has the decryption stanza). Lands in ns
-  #     keycloak; the keycloak-db Cluster (platform-config) reads it by name for WAL archiving. ---
+  # --- PER-ENVIRONMENT S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05), split
+  #     2026-09-07. SOPS-encrypted, decrypted by the `platform` tier (which already has the
+  #     decryption stanza). BOTH land in ns keycloak on BOTH clusters; the keycloak-db Cluster
+  #     (platform-config) selects ONE by name via
+  #     `keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
+  #     prod "-prod"). Two objects, so an R2 token rotation cannot silently break dev's MinIO
+  #     WAL archiving — and archive_mode=on makes a broken archive a disk-full clock, not a
+  #     degraded backup (see postgres-cluster.yaml).
+  #       dev  -> keycloak-db-backup-credentials       (MinIO, s3://cnpg)
+  #       prod -> keycloak-db-backup-credentials-prod  (Cloudflare R2, s3://k3s-prod-backup/barman)
   - keycloak-db-backup-credentials.sops.yaml
+  - keycloak-db-backup-credentials-prod.sops.yaml
   # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
   - grafana-dashboard.yaml

platform/keycloak/postgres-cluster.yaml

@@ -34,25 +34,43 @@ spec:
       memory: 256Mi
     limits:
       memory: 512Mi
-  # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
+  # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → object storage ---
   # Continuous WAL archiving + periodic base backups give proper Postgres PITR (superior to a Velero
-  # PVC snapshot for a live DB). destinationPath is the BUCKET ROOT; CNPG appends serverName (defaults
-  # to the Cluster name), so objects land under s3://cnpg/keycloak-db/. Path-style HTTP MinIO endpoint.
+  # PVC snapshot for a live DB). CNPG appends `serverName` (defaults to the Cluster name) to
+  # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/keycloak-db/ — dev
+  # s3://cnpg/keycloak-db/ on the on-Proxmox MinIO, prod s3://k3s-prod-backup/barman/keycloak-db/ on
+  # Cloudflare R2. Both endpointURL and destinationPath are cluster-vars knobs (R-ENV-02),
+  # substituted by the `platform-config` Kustomization (this file sits under platform/keycloak/ but
+  # platform-config is the only tier that lists it as a resource — see its kustomization.yaml).
   # Credentials: the scoped key from the in-namespace SOPS Secret keycloak-db-backup-credentials
-  # (a Pod only reads same-ns Secrets), kept in sync with platform/backup/minio-credentials.sops.yaml.
-  # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000) — prod
-  # MUST point at its own off-cluster target. Substituted by the `platform-config` Kustomization.
+  # (a Pod only reads same-ns Secrets).
+  #
+  # *** archive_mode IS ON, SO A FAILING ARCHIVE COMMAND IS A DISK-FULL CLOCK, NOT A DEGRADED
+  # BACKUP. *** Postgres will not recycle a WAL segment it has not successfully archived, so
+  # whenever these two values are wrong the pgdata PVC fills monotonically until the primary
+  # PANICs. This is not hypothetical: prod ran from bootstrap to 2026-09-07 with
+  # `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT` (barman-cloud-wal-archive exit 4 on every segment)
+  # and had burned ~579 MiB of a 5 GiB PVC on the way to taking SSO — and with it kubectl OIDC,
+  # Grafana, Headlamp, Gitea and forward-auth — down with it. Never leave a placeholder here, and
+  # treat `ContinuousArchiving=False` on this Cluster as an availability incident, not a backup one.
   backup:
     retentionPolicy: "30d"          # keep 30 days of base backups + WAL (PITR window); prunes older
     barmanObjectStore:
-      destinationPath: s3://cnpg
+      destinationPath: ${CNPG_BACKUP_PATH}
       endpointURL: ${S3_BACKUP_ENDPOINT}
+      # PER-ENVIRONMENT credential (2026-09-07): dev reads keycloak-db-backup-credentials (MinIO),
+      # prod reads keycloak-db-backup-credentials-prod (Cloudflare R2). Both Secrets are applied to
+      # both clusters by platform/keycloak/kustomization.yaml; ${BACKUP_CREDENTIALS_SUFFIX} (dev ""
+      # | prod "-prod") picks one. Before the split these were ONE shared Secret, so writing prod's
+      # R2 token into it would have left dev signing MinIO requests with an R2 key — a 403 on every
+      # archive command, which under archive_mode=on is the disk-full clock described above, not a
+      # missing backup.
       s3Credentials:
         accessKeyId:
-          name: keycloak-db-backup-credentials
+          name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
           key: ACCESS_KEY_ID
         secretAccessKey:
-          name: keycloak-db-backup-credentials
+          name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
           key: ACCESS_SECRET_KEY
       wal:
         compression: gzip

platform/velero/helmrelease.yaml

@@ -1,18 +1,29 @@
 ---
 # Velero — off-cluster backup/DR of cluster resources + PVC data (ADR-0003, R-STATE-03). LIVE.
 #
-# TARGET: the MinIO S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars; dev http://192.168.1.9:9000 on
-# Proxmox — plain HTTP, path-style), bucket
-# `velero`. region us-east-1 is a placeholder MinIO ignores (S3 clients still require one).
+# TARGET: the S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars) at
+# s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX}, in region ${S3_BACKUP_REGION}. Four knobs,
+# because the two environments' object stores differ in shape as well as address:
+#   dev   http://192.168.1.9:9000  bucket `velero`, NO prefix, region us-east-1  (on-Proxmox MinIO,
+#                                  plain HTTP, path-style; MinIO ignores the region)
+#   prod  https://<acct>.r2.cloudflarestorage.com  bucket `k3s-prod-backup`, prefix `velero`,
+#                                  region `auto`   (Cloudflare R2; prod cannot route to that MinIO,
+#                                  and R2 demands exactly `auto`)
+# Prod's bucket is SHARED with CNPG/barman (the operator created one bucket), so the `velero` prefix
+# is what keeps this controller's retention sweep away from the Postgres WAL stream. See the
+# "OBJECT-STORE TENANCY" block in clusters/base/cluster-vars.yaml.
 #
 # WHY THIS SHAPE:
 #  - provider aws + velero-plugin-for-aws (init container) drives the S3-compatible MinIO target.
 #  - File System Backup (kopia) via the node-agent DaemonSet — storage-agnostic PVC backup that does
 #    NOT depend on CSI VolumeSnapshots or a data-mover (the simplest path that still captures the
 #    Longhorn PVC *data*). snapshotsEnabled:false → no VolumeSnapshotLocation is created/needed.
-#  - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero) — the
-#    SCOPED MinIO key limited to the velero+cnpg buckets. Kept in sync with the staging source of
-#    truth platform/backup/minio-credentials.sops.yaml and the Proxmox host ansible-vault.
+#  - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero). ONE
+#    Secret serves both environments, and it now carries the Cloudflare R2 API token (written by
+#    platform/backup/set-r2-credentials.sh, 2026-09-07) rather than the scoped MinIO key. That is
+#    safe only because the R2 token is bucket-scoped to k3s-prod-backup, which dev never names:
+#    dev's Velero would fail loudly against its MinIO endpoint with an R2 key, it could not
+#    silently write somewhere unexpected. Rotating either target means re-running that script.
 #
 # VOLUME OPT-OUT (per the sizing decision) is implemented with TWO schedules (a single Backup has one
 # cluster-wide defaultVolumesToFsBackup, so per-namespace volume policy needs two): see `schedules`.
@@ -72,14 +83,45 @@ spec:
       backupStorageLocation:
         - name: default
           provider: aws
-          bucket: velero
+          # Bucket and prefix are BOTH knobs (cluster-vars), because the two environments' object
+          # stores are shaped differently. Dev's MinIO has a dedicated `velero` bucket and dev's
+          # backups sit at its ROOT, so dev renders bucket=velero with an EMPTY prefix — and the
+          # chart omits `objectStorage.prefix` entirely when it is empty (verified, chart 12.1.0),
+          # leaving dev's live BackupStorageLocation byte-unchanged. Prod's operator created ONE R2
+          # bucket shared with CNPG/barman, so prod renders bucket=k3s-prod-backup with prefix
+          # `velero` — a disjoint top-level prefix from barman's. Getting this wrong is not a
+          # cosmetic error: Velero ADOPTS every backup it finds under its location and TTL-DELETES
+          # it on retention, so a prefix that overlaps another tenant deletes that tenant's data.
+          bucket: ${VELERO_BACKUP_BUCKET}
+          prefix: "${VELERO_BACKUP_PREFIX}"
           default: true
           config:
-            region: us-east-1                 # placeholder; MinIO ignores it (S3 clients require one)
-            s3Url: ${S3_BACKUP_ENDPOINT}      # on-Proxmox MinIO, plain HTTP (cluster-vars)
-            s3ForcePathStyle: "true"          # MinIO is path-style
+            # Region is a knob, not a constant: S3 clients require SOME region, MinIO ignores
+            # whatever it is, and Cloudflare R2 requires exactly `auto`. Hardcoding `us-east-1`
+            # was correct-by-accident and broke the moment the target stopped being MinIO.
+            region: ${S3_BACKUP_REGION}
+            s3Url: ${S3_BACKUP_ENDPOINT}      # cluster-vars; dev MinIO (http) or prod R2 (https)
+            s3ForcePathStyle: "true"          # MinIO is path-style; R2 accepts path-style too
+            # *** REQUIRED FOR CLOUDFLARE R2 — DO NOT DROP THIS LINE. ***
+            # velero-plugin-for-aws v1.14.2 bundles an AWS SDK with flexible checksums ON by
+            # default, so it sends `x-amz-sdk-checksum-algorithm` / `x-amz-checksum-crc32` on every
+            # PUT. R2 REJECTS those headers with HTTP 400 InvalidArgument, which would fail every
+            # backup AND every kopia blob upload. The empty string is the documented opt-out, and
+            # it must be PRESENT-but-empty rather than absent — the plugin branches on the key
+            # existing, and the chart does emit an empty value (verified, chart 12.1.0). Harmless
+            # against MinIO, which ignores it, so it is stated unconditionally rather than as a
+            # fourth knob. If backups start failing with InvalidArgument after a plugin bump, look
+            # here first.
+            checksumAlgorithm: ""
     credentials:
-      existingSecret: velero-cloud-credentials   # SOPS Secret carrying the `cloud` INI key (ns velero)
+      # PER-ENVIRONMENT credential (2026-09-07). dev renders velero-cloud-credentials (the scoped
+      # MinIO key); prod renders velero-cloud-credentials-prod (the Cloudflare R2 token). BOTH
+      # Secrets are applied to BOTH clusters by platform/velero/kustomization.yaml — the suffix
+      # only chooses which one this Deployment mounts. cluster-vars: BACKUP_CREDENTIALS_SUFFIX,
+      # dev "" | prod "-prod". A name that resolves to a MISSING Secret does not degrade the
+      # backup, it wedges the pod in ContainerCreating (the volume never mounts), which is why
+      # the base default is the empty suffix — the unsuffixed Secret always exists.
+      existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}   # SOPS Secret carrying the `cloud` INI key (ns velero)
     # --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data
     #     by default (opt-out), but skip the two reconstructable heavy volumes' DATA while still
     #     capturing their Kubernetes resources. ---

platform/velero/kustomization.yaml

@@ -5,7 +5,19 @@ resources:
   - namespace.yaml
   - helmrepository.yaml
   - helmrelease.yaml
-  # Scoped MinIO S3 credential (the `cloud` INI key the AWS plugin reads) — SOPS-encrypted, decrypted
-  # per-env by the `velero` Flux Kustomization (clusters/dev). Kept in sync with the staging source of
-  # truth platform/backup/minio-credentials.sops.yaml + the Proxmox host ansible-vault.
+  # --- PER-ENVIRONMENT `cloud` INI credential (split 2026-09-07) -----------------------------------
+  # BOTH Secrets are applied to BOTH clusters; the HelmRelease picks ONE by name via
+  # `existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
+  # prod "-prod"). They are separate objects so rotating one environment's key can never touch the
+  # other's — the failure mode that made the split necessary (a single shared Secret carrying R2 keys
+  # would have pointed dev's Velero at MinIO with credentials MinIO has never issued).
+  #
+  # dev  -> velero-cloud-credentials       : the scoped MinIO key for http://192.168.1.9:9000.
+  #                                          Kept in sync with the staging source of truth
+  #                                          platform/backup/minio-credentials.sops.yaml + the
+  #                                          Proxmox host ansible-vault.
+  # prod -> velero-cloud-credentials-prod  : the Cloudflare R2 token for k3s-prod-backup.
+  #
+  # Both are decrypted by the `velero` Flux Kustomization's SOPS stanza (clusters/dev + clusters/prod).
   - velero-cloud.sops.yaml
+  - velero-cloud-prod.sops.yaml

platform/velero/values.yaml

@@ -2,8 +2,13 @@
 # doc pointer, kept for parity with the other components' layout).
 #
 # LIVE wiring (see helmrelease.yaml):
-#   - provider aws + velero-plugin-for-aws (init container) → MinIO S3 ${S3_BACKUP_ENDPOINT} (dev: http://192.168.1.9:9000)
-#     (path-style, bucket `velero`, region us-east-1 placeholder).
+#   - provider aws + velero-plugin-for-aws (init container) → S3 ${S3_BACKUP_ENDPOINT}, path-style,
+#     at s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX} in region ${S3_BACKUP_REGION}. Four
+#     cluster-vars knobs: dev = the on-Proxmox MinIO (http://192.168.1.9:9000, bucket `velero`, no
+#     prefix, us-east-1); prod = Cloudflare R2 (bucket `k3s-prod-backup`, prefix `velero`, `auto`),
+#     sharing one bucket with CNPG/barman under a disjoint prefix.
+#   - config.checksumAlgorithm: "" — MANDATORY for R2, which rejects the AWS SDK's
+#     x-amz-checksum-* headers with HTTP 400 InvalidArgument. Harmless against MinIO.
 #   - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover).
 #   - snapshotsEnabled:false → no VolumeSnapshotLocation.
 #   - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero).

platform/velero/velero-cloud-prod.sops.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: velero-cloud-credentials-prod
+    namespace: velero
+type: Opaque
+stringData:
+    cloud: ENC[AES256_GCM,data:Kq+hfYMq/NZJJqbK2zqwW97XKNW0LbT2Un20yc0B6evPXqp+aC0vV0odwos4XwV0puPIsEhU7IeD+JidVjU1eKv6Qax0rnoj90FVWihxWyLMNAMC+b8PxBnPhWdk6ZlrMKs13roGZwYh4bbrstnJamMqpD4oZpthEl/NEQNWVAb5ogJHsmPNiQOtHhtBWkTubfgPIw==,iv:gn8/4/XpUZ8c6+nlGe6SAODUyrhDqskQ+uVRx2YR3Qc=,tag:aTk3cRJTA0WjdIi1B9S0hg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwWmV5UGpmdnFKbEhjeHlZ
+            QUZCWm13ZlpVYU1URFErSmM5MEJ2WTRxZVFvCkZ5cEdJaXJ1NDNSaEF5d2lxdDQw
+            NGpzQm5ESGt6cUlIZVdHYTBjRGVwaVUKLS0tIHlpWkVacXdIZndBOXdlVk1LZzlN
+            NkZyRm1abk9CNlZRUERtbFM2RjVtS1UKWYtI7QZiTjPIHaLOOX9WvDK1cHwP2WfW
+            26WdNW90tTmt3cYdaFGnkpAVCMS3BRtdInirulfoGRgvffjWNBDVuQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-09-07T06:36:33Z"
+    mac: ENC[AES256_GCM,data:4v4L29KoIhQI94TU/s0u3HKR5C6mMc6mRWWsZzwIfqkl4fayRvLG67tbJ88vVkb6AXgqHoKXKlhbpn6hTJPBrL72f8+fii3CeLloVJyW2AuzHaHCSi92U04ywoSc45hClZrOLvUFf5KbtawPPKfP53oaNq+AVEZ1nqKd8ssUyBA=,iv:WVrdja00ptK/MmCC/Chu+Gq4ZYpBvpFRWW3lSo7Mon0=,tag:GzFfpkBlHu6Nc8ZdnCBuKQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4