diff --git a/apps/wave-0/nextcloud-db.yaml b/apps/wave-0/nextcloud-db.yaml index 5024610..754ba03 100644 --- a/apps/wave-0/nextcloud-db.yaml +++ b/apps/wave-0/nextcloud-db.yaml @@ -75,8 +75,15 @@ spec: # s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod # s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`. - # Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials - # (same-ns only). + # Credentials: PER-ENVIRONMENT (2026-09-07). dev reads nextcloud-db-backup-credentials (the + # scoped MinIO key), prod reads nextcloud-db-backup-credentials-prod (the Cloudflare R2 + # token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a + # generated golden — render.py places the -prod sidecar in this DB's wave); the + # cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which + # one this Cluster reads. Both are in-namespace, which is all a pod can read. They were + # ONE shared Secret until the split, so giving prod its R2 token would have left dev + # signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the + # disk-full clock below rather than a merely missing backup. # # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup: # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until @@ -89,10 +96,10 @@ spec: endpointURL: ${S3_BACKUP_ENDPOINT} s3Credentials: accessKeyId: - name: nextcloud-db-backup-credentials + name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX} key: ACCESS_KEY_ID secretAccessKey: - name: nextcloud-db-backup-credentials + name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX} key: ACCESS_SECRET_KEY wal: compression: gzip diff --git a/apps/wave-0/semaphore-db.yaml b/apps/wave-0/semaphore-db.yaml index cfef719..3991548 100644 --- a/apps/wave-0/semaphore-db.yaml +++ b/apps/wave-0/semaphore-db.yaml @@ -76,8 +76,15 @@ spec: # s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod # s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`. - # Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials - # (same-ns only). + # Credentials: PER-ENVIRONMENT (2026-09-07). dev reads semaphore-db-backup-credentials (the + # scoped MinIO key), prod reads semaphore-db-backup-credentials-prod (the Cloudflare R2 + # token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a + # generated golden — render.py places the -prod sidecar in this DB's wave); the + # cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which + # one this Cluster reads. Both are in-namespace, which is all a pod can read. They were + # ONE shared Secret until the split, so giving prod its R2 token would have left dev + # signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the + # disk-full clock below rather than a merely missing backup. # # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup: # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until @@ -90,10 +97,10 @@ spec: endpointURL: ${S3_BACKUP_ENDPOINT} s3Credentials: accessKeyId: - name: semaphore-db-backup-credentials + name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX} key: ACCESS_KEY_ID secretAccessKey: - name: semaphore-db-backup-credentials + name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX} key: ACCESS_SECRET_KEY wal: compression: gzip diff --git a/clusters/base/backup.yaml b/clusters/base/backup.yaml index 5e7fe6e..0cc0cca 100644 --- a/clusters/base/backup.yaml +++ b/clusters/base/backup.yaml @@ -34,6 +34,9 @@ spec: substituteFrom: - kind: ConfigMap name: cluster-vars - # SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per - # environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how - # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml. + # SOPS decryption for the `cloud` INI credentials — velero-cloud.sops.yaml (dev's scoped MinIO key) + # and velero-cloud-prod.sops.yaml (prod's Cloudflare R2 token) — is layered per environment by the + # overlay (clusters/dev and clusters/prod patch THIS Kustomization by name), identical to how + # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. Both Secrets are applied on both + # clusters; ${BACKUP_CREDENTIALS_SUFFIX} (substituted from cluster-vars above) decides which one the + # velero Deployment actually mounts. See clusters/dev/kustomization.yaml. diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index f82fa36..2762d67 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -274,6 +274,55 @@ data: VELERO_BACKUP_PREFIX: "" # dev: bucket root (live layout) | prod: "velero" CNPG_BACKUP_PATH: "s3://cnpg" # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman + # ------------------------------------------------------------------------------------------------ + # WHICH CREDENTIAL SECRET THE BACKUP CONSUMERS READ (added 2026-09-07 — the per-env credential split) + # ------------------------------------------------------------------------------------------------ + # The three knobs above say WHERE the backups go. This one says WHICH KEY signs the requests, and + # it exists because the endpoint moved per-environment while the credential did not. + # + # THE BUG IT FIXES. The four backup-credential Secrets — velero-cloud.sops.yaml and the three + # -backup-credentials.sops.yaml — are SHARED BASE manifests: platform/ and apps/ are rendered + # by the same Flux tiers on both clusters, and cluster-vars substitution is the ONLY thing that + # differs (R-ENV-02). So writing prod's R2 token into those four files did not GIVE prod a + # credential, it REPLACED DEV'S: dev's Velero and all three dev CNPG Clusters would have started + # signing MinIO requests with an R2 key id. For Velero that is a broken BSL; for CNPG it is worse, + # because `archive_mode=on` means Postgres REFUSES TO RECYCLE a WAL segment it could not archive, + # so a 403 loop is a monotonically filling pgdata PVC ending in a PANIC — the same failure prod + # spent its whole bring-up in with a placeholder endpoint. One shared Secret also means rotating + # either environment's key silently breaks the other; that is the structural reason for the split, + # independent of dev's eventual decommission. + # + # THE MECHANISM. Each credential is now TWO Secret objects, both applied to BOTH clusters (they are + # base resources; nothing here forks a manifest per environment). The consumer names one: + # + # velero: credentials.existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX} + # CNPG: s3Credentials.*.name: -backup-credentials${BACKUP_CREDENTIALS_SUFFIX} + # + # ONE knob, not four, ON PURPOSE. Four independent secret-name knobs could be updated three- + # quarters of the way, leaving one consumer pointed at the OTHER environment's key with nothing + # rendering wrong — and StrictPostBuildSubstitutions cannot catch that, because it only fails on an + # UNDEFINED variable, never on a defined-but-stale one. With a single suffix the four move together + # or not at all. The cost is that the rendered Secret name is not greppable from the consumer; the + # names are spelled out in full in each kustomization.yaml beside the resources. + # + # THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING. It renders the UNSUFFIXED names, + # which are the objects that already exist and that the live dev cluster is running. A default + # naming a Secret that does not exist would be worse than the shared-Secret state it replaces: + # Velero's pod would sit in ContainerCreating forever on a missing `existingSecret` volume, and a + # CNPG Cluster whose s3Credentials Secret is absent stops archiving WAL — i.e. the disk-full clock + # again — so an unresolvable default breaks the referencing cluster HARDER than a wrong key does. + # Empty is also the value dev must carry, so base and dev agree, exactly like VELERO_BACKUP_PREFIX. + # + # An EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions: strict mode fails on a variable that is + # not DEFINED, not on one defined as "". VELERO_BACKUP_PREFIX above has shipped empty since the + # gate was enabled and proves it. Both overlays state this knob explicitly — dev "" and prod + # "-prod" — so neither inherits it silently. + # + # ADDING A NEW BACKUP CREDENTIAL: create BOTH halves (`x-credentials.sops.yaml` and + # `x-credentials-prod.sops.yaml`), list both in the kustomization, and reference + # `x-credentials${BACKUP_CREDENTIALS_SUFFIX}`. Never a single shared Secret again. + BACKUP_CREDENTIALS_SUFFIX: "" # dev: "" (MinIO creds) | prod: "-prod" (R2 creds) + # --- placement --- PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os" # base no-op selector; prod may pin roles PLATFORM_NODE_SELECTOR_VALUE: "linux" diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index f5a48c3..cb2cb1a 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -106,8 +106,11 @@ patches: secretRef: name: sops-age # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml). - # It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS - # decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only). + # It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped + # MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption + # stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by + # BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is + # out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization @@ -256,6 +259,24 @@ patches: - op: replace path: /data/CNPG_BACKUP_PATH value: "s3://cnpg" + # WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets + # (velero-cloud-credentials, -backup-credentials) — the scoped MinIO keys dev's Velero and + # its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod + # renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead. + # + # RESTATED EXPLICITLY even though it equals the base default, for the same reason as + # EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED + # base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev + # signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on + # turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR + # restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential. + # + # DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator, + # 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING + # backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs. + - op: replace + path: /data/BACKUP_CREDENTIALS_SUFFIX + value: "" # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas. - op: replace path: /data/LONGHORN_DEFAULT_REPLICA_COUNT diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index 74df544..78c5da0 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -454,4 +454,29 @@ patches: - op: replace path: /data/CNPG_BACKUP_PATH value: "s3://k3s-prod-backup/barman" + # ------------------------------------------------------------------------------------------ + # WHICH CREDENTIAL SIGNS THE REQUESTS TO THAT BUCKET (per-env credential split, 2026-09-07). + # ------------------------------------------------------------------------------------------ + # The knobs above moved the ENDPOINT and the OBJECT LAYOUT per environment. The credential did + # not move with them, and it could not: the four backup-credential Secrets are SHARED BASE + # manifests, applied to both clusters by the same Flux tiers. Writing the R2 token into them + # gave prod a working key by TAKING DEV'S AWAY — dev would have signed MinIO requests with an + # R2 key id, and under archive_mode=on that is a filling pgdata PVC, not a missing backup. + # + # So each credential is now two objects, and this suffix selects prod's: + # velero-cloud-credentials-prod the R2 token in the AWS INI `cloud` shape + # -backup-credentials-prod the same token as ACCESS_KEY_ID / ACCESS_SECRET_KEY + # for keycloak-db, nextcloud-db and semaphore-db. Both halves ship to both clusters (they are + # base resources); ONLY this value decides which one each consumer reads. It is a SUFFIX rather + # than four secret-name knobs deliberately — four could be updated three-quarters of the way, + # leaving one consumer on dev's MinIO key with nothing rendering wrong, and strict-mode + # substitution cannot catch a defined-but-stale variable. + # + # THE R2 TOKEN IS THE ONE PROVEN FROM INSIDE PROD (chore/r2-credentials 0a6a862): head-bucket on + # k3s-prod-backup returns BucketRegion WNAM, and PUT and DELETE both succeed. Rotating it now + # touches ONLY the four `-prod` files; dev's MinIO keys are untouched objects. That isolation is + # the structural reason for the split, independent of dev's eventual decommission. + - op: replace + path: /data/BACKUP_CREDENTIALS_SUFFIX + value: "-prod" # TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist. diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml index 22a841c..bcfd330 100644 --- a/platform/keycloak/postgres-cluster.yaml +++ b/platform/keycloak/postgres-cluster.yaml @@ -58,12 +58,19 @@ spec: barmanObjectStore: destinationPath: ${CNPG_BACKUP_PATH} endpointURL: ${S3_BACKUP_ENDPOINT} + # PER-ENVIRONMENT credential (2026-09-07): dev reads keycloak-db-backup-credentials (MinIO), + # prod reads keycloak-db-backup-credentials-prod (Cloudflare R2). Both Secrets are applied to + # both clusters by platform/keycloak/kustomization.yaml; ${BACKUP_CREDENTIALS_SUFFIX} (dev "" + # | prod "-prod") picks one. Before the split these were ONE shared Secret, so writing prod's + # R2 token into it would have left dev signing MinIO requests with an R2 key — a 403 on every + # archive command, which under archive_mode=on is the disk-full clock described above, not a + # missing backup. s3Credentials: accessKeyId: - name: keycloak-db-backup-credentials + name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX} key: ACCESS_KEY_ID secretAccessKey: - name: keycloak-db-backup-credentials + name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX} key: ACCESS_SECRET_KEY wal: compression: gzip diff --git a/platform/velero/helmrelease.yaml b/platform/velero/helmrelease.yaml index 0391158..f41a0da 100644 --- a/platform/velero/helmrelease.yaml +++ b/platform/velero/helmrelease.yaml @@ -114,7 +114,14 @@ spec: # here first. checksumAlgorithm: "" credentials: - existingSecret: velero-cloud-credentials # SOPS Secret carrying the `cloud` INI key (ns velero) + # PER-ENVIRONMENT credential (2026-09-07). dev renders velero-cloud-credentials (the scoped + # MinIO key); prod renders velero-cloud-credentials-prod (the Cloudflare R2 token). BOTH + # Secrets are applied to BOTH clusters by platform/velero/kustomization.yaml — the suffix + # only chooses which one this Deployment mounts. cluster-vars: BACKUP_CREDENTIALS_SUFFIX, + # dev "" | prod "-prod". A name that resolves to a MISSING Secret does not degrade the + # backup, it wedges the pod in ContainerCreating (the volume never mounts), which is why + # the base default is the empty suffix — the unsuffixed Secret always exists. + existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX} # SOPS Secret carrying the `cloud` INI key (ns velero) # --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data # by default (opt-out), but skip the two reconstructable heavy volumes' DATA while still # capturing their Kubernetes resources. ---