diff --git a/apps/wave-0/nextcloud-db.yaml b/apps/wave-0/nextcloud-db.yaml index accab0f..5024610 100644 --- a/apps/wave-0/nextcloud-db.yaml +++ b/apps/wave-0/nextcloud-db.yaml @@ -68,18 +68,24 @@ spec: memory: 256Mi limits: memory: 512Mi - # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket --- + # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage --- # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's - # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName - # (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP. + # backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to + # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/nextcloud-db/ — dev + # s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod + # s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and + # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`. # Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials - # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml. - # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000), - # substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target. + # (same-ns only). + # + # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup: + # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until + # the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder + # endpoint. Treat ContinuousArchiving=False here as an availability incident. backup: retentionPolicy: "30d" barmanObjectStore: - destinationPath: s3://cnpg + destinationPath: ${CNPG_BACKUP_PATH} endpointURL: ${S3_BACKUP_ENDPOINT} s3Credentials: accessKeyId: diff --git a/apps/wave-0/semaphore-db.yaml b/apps/wave-0/semaphore-db.yaml index 7c8c3a5..cfef719 100644 --- a/apps/wave-0/semaphore-db.yaml +++ b/apps/wave-0/semaphore-db.yaml @@ -69,18 +69,24 @@ spec: memory: 256Mi limits: memory: 512Mi - # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket --- + # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage --- # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's - # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName - # (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP. + # backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to + # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/semaphore-db/ — dev + # s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod + # s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and + # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`. # Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials - # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml. - # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000), - # substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target. + # (same-ns only). + # + # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup: + # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until + # the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder + # endpoint. Treat ContinuousArchiving=False here as an availability incident. backup: retentionPolicy: "30d" barmanObjectStore: - destinationPath: s3://cnpg + destinationPath: ${CNPG_BACKUP_PATH} endpointURL: ${S3_BACKUP_ENDPOINT} s3Credentials: accessKeyId: diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index e49618c..f82fa36 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -96,9 +96,22 @@ data: # # Substitution coverage (verify before adding a knob here): every path that consumes these declares # postBuild.substituteFrom cluster-vars — platform (K3S_API_HOST), platform-config - # (CILIUM_LB_POOL_*, S3_BACKUP_ENDPOINT), crafty (CRAFTY_L4_VIP), apps-wave-0 (S3_BACKUP_ENDPOINT), - # velero (S3_BACKUP_ENDPOINT), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an UNSUBSTITUTED path would - # ship the literal string into the cluster. + # (CILIUM_LB_POOL_*, and keycloak-db's S3_BACKUP_ENDPOINT + CNPG_BACKUP_PATH: the Cluster lives at + # platform/keycloak/postgres-cluster.yaml but is rendered by platform-config, which is the ONLY + # tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db: + # S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH), velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION, + # VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an + # UNSUBSTITUTED path would ship the literal string into the cluster. + # + # ADDING A KNOB IS SAFE UNDER StrictPostBuildSubstitutions ONLY BECAUSE OF THE ROOT EDGE. An + # UNDEFINED variable is a HARD Kustomization failure, so a manifest that references a knob the + # live ConfigMap does not yet carry breaks its whole tier. `clusters/base/platform.yaml` gained + # `dependsOn: [flux-system]` for exactly this: dependsOn is REVISION-gated, so no substituting + # tier can reconcile revision N until the ROOT Kustomization has APPLIED revision N — i.e. until + # the new cluster-vars key EXISTS. Every substituting consumer chains transitively through + # `platform`, so one edge covers the graph, and a new knob plus its first reference may ship in + # ONE commit. (A hand-run `kubectl apply -k clusters/` bypasses that ordering entirely: apply + # the overlay BEFORE the tier that references the knob, or the tier fails until it retries.) # k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml). # @@ -150,9 +163,21 @@ data: # Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style. # Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db, # nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port. - # Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR. + # Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR — + # and, measured 2026-09-05 from a pod INSIDE prod, it CANNOT reach it anyway (prod is on the SOHO + # network 192.168.2.0/24, which is pfSense's WAN, while this MinIO sits inside pfSense's LAN: + # `curl http://192.168.1.9:9000/minio/health/live` timed out after 15s). Prod uses Cloudflare R2. S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000" + # S3 region. Every S3 client REQUIRES one; MinIO ignores whatever it is; Cloudflare R2 demands + # exactly `auto` (its `head-bucket` reports the physical hint `WNAM`, which is NOT a usable region + # string). This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by- + # accident for MinIO and wrong for anything else, which is why it is a knob now. The base default + # is dev's LIVE value, deliberately: dev's Velero has been signing requests with `us-east-1` + # against that MinIO since bootstrap, and this repo's rule is that base == the known-good dev + # value. Prod overrides to `auto`. + S3_BACKUP_REGION: "us-east-1" + # --- storage (ADR-0003) --- LONGHORN_DEFAULT_REPLICA_COUNT: "3" # default StorageClass replica count (3 on 3 nodes) # Longhorn's defaultDataPath (platform/longhorn/helmrelease.yaml): the on-NODE directory every @@ -205,6 +230,50 @@ data: VELERO_BACKUP_SCHEDULE: "0 3 * * *" # cron; overlays may differ VELERO_RETENTION: "168h" # dev short | prod long + # ========================================================================================== + # OBJECT-STORE TENANCY — the three knobs that keep two clusters and two backup tools from + # writing into each other's data. THESE ARE SAFETY KNOBS, not preferences. + # ========================================================================================== + # Nothing about the object layout was per-environment before this: Velero's bucket was the + # hardcoded literal `velero` with no prefix, and every CNPG Cluster used the hardcoded + # `s3://cnpg` with barman's `serverName` defaulting to the CLUSTER NAME — and BOTH environments + # have a keycloak-db, a nextcloud-db and a semaphore-db. Point two clusters at one bucket under + # that layout and: + # (a) each cluster's Velero ADOPTS the other's backups and TTL-DELETES them on retention; and + # (b) two different Postgres servers stream WAL into ONE Barman prefix, which does not merely + # mix files — it CORRUPTS THE CHAIN and destroys PITR for both. That is the worse half: + # dev's DR restore is PROVEN (R-STATE-03) and this would silently un-prove it. + # + # WHY THERE IS NO SINGLE `S3_BACKUP_BUCKET`. The two consumers take differently-SHAPED values: + # the Velero chart wants `bucket` and `prefix` as separate fields, while CNPG's + # barmanObjectStore.destinationPath is ONE full s3:// URL whose first path segment IS the bucket. + # CNPG's bucket therefore already lives inside CNPG_BACKUP_PATH; adding a second knob that had to + # hold the same bucket name a second time would be a rotation waiting to drift — the exact class + # of duplication this repo has been bitten by. Three knobs, each shaped for its own consumer, and + # no two of them can contradict each other. + # + # BASE = DEV'S LIVE LAYOUT, BYTE-FOR-BYTE, AND MUST STAY THAT WAY. Dev's existing backups are at + # `s3://velero/` (bucket root, no prefix) and `s3://cnpg//` on that MinIO. Giving dev + # a prefix, or renaming its buckets, would ORPHAN the very backups its proven restore depends on — + # the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default` and its LONGHORN_DATA_PATH stays + # /var/lib/longhorn. Verified: with an EMPTY prefix the Velero chart omits `objectStorage.prefix` + # from the rendered BackupStorageLocation entirely (helm template, chart 12.1.0), so dev's live + # BSL is unchanged by this parameterization rather than merely equivalent to it. + # + # PROD, which has no history to preserve, gets ONE bucket with TWO disjoint top-level prefixes: + # + # s3://k3s-prod-backup/velero/… Velero (kopia repo + backup metadata) + # s3://k3s-prod-backup/barman//… CNPG WAL + base backups + # + # The operator created exactly one R2 bucket, so velero-vs-barman separation has to come from the + # prefix rather than the bucket. The BUCKET ROOT IS DELIBERATELY LEFT EMPTY: if a prefix ever + # renders wrong, its objects land somewhere obviously bogus instead of silently colliding with a + # real tenant's data. Dev↔prod isolation is doubly independent — different endpoint AND different + # bucket — so neither cluster can reach the other's objects even with a mis-substituted prefix. + VELERO_BACKUP_BUCKET: "velero" # dev: the `velero` MinIO bucket | prod: k3s-prod-backup + VELERO_BACKUP_PREFIX: "" # dev: bucket root (live layout) | prod: "velero" + CNPG_BACKUP_PATH: "s3://cnpg" # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman + # --- placement --- PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os" # base no-op selector; prod may pin roles PLATFORM_NODE_SELECTOR_VALUE: "linux" diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 8c533ea..f5a48c3 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -224,9 +224,38 @@ patches: value: "192.168.1.55" # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG # barmanObjectStore and Velero's backupStorageLocation write to. + # + # DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07). + # Prod could not use it at all — it cannot even route to it — but dev's move is a separate, + # operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write + # a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window + # would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo + # (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod. - op: replace path: /data/S3_BACKUP_ENDPOINT value: "http://192.168.1.9:9000" + # DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same + # reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base + # defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones + # its proven restore depends on — without any manifest looking wrong. + # + # `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO + # ignores the value, but changing it is a needless risk against a live, working backup path. + # An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already + # are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified + # against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this. + - op: replace + path: /data/S3_BACKUP_REGION + value: "us-east-1" + - op: replace + path: /data/VELERO_BACKUP_BUCKET + value: "velero" + - op: replace + path: /data/VELERO_BACKUP_PREFIX + value: "" + - op: replace + path: /data/CNPG_BACKUP_PATH + value: "s3://cnpg" # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas. - op: replace path: /data/LONGHORN_DEFAULT_REPLICA_COUNT diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index da68488..74df544 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -392,10 +392,66 @@ patches: - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.2.22" - # Off-cluster DR target. Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the - # network — a prod backup written there is not off-cluster DR and would mix two environments' - # PITR streams in one bucket. Operator supplies the prod endpoint (full URL incl. scheme+port). + # ========================================================================================== + # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07). + # ========================================================================================== + # Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the network and is NOT an + # option for prod, for two independent reasons: + # 1. UNREACHABLE. Prod is on the SOHO network 192.168.2.0/24 — pfSense's WAN — while that + # MinIO is inside pfSense's LAN. Measured from a pod INSIDE prod (the workstation proves + # nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` → connection + # timed out after 15s. Reaching it would need a WAN→LAN rule that does not exist. + # 2. NOT OFF-CLUSTER DR. It runs on the same Proxmox host as dev's VMs, so it already shares + # a physical failure domain with dev. R2 is genuinely off-site, and its free egress means + # restore REHEARSALS cost nothing — R-STATE-03 has no financial disincentive to stay proven. + # + # THIS UNBLOCKS AN ACTIVE INCIDENT, not just a roadmap item. All three prod CNPG Clusters ship + # `archive_mode=on` with `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT`, so barman-cloud-wal-archive + # exits 4 on every segment and Postgres — correctly — REFUSES TO RECYCLE WAL. Verified live + # 2026-09-07: keycloak-db / nextcloud-db / semaphore-db all report + # ContinuousArchiving=False/ContinuousArchivingFailing, and keycloak-db's 5 GiB PVC is at + # 579 MiB used / 4360 MiB free and climbing monotonically. A disk-full PANIC there takes the + # cluster's SSO database down, and with it kubectl OIDC, Grafana, Headlamp, Gitea and + # forward-auth. Fixing the endpoint lets Postgres drain the whole backlog. + # + # ENDPOINT IS ACCOUNT-SCOPED, so it stays a prod-overlay value even though it is not a LAN + # address: this is the operator's Cloudflare account id, confirmed from inside a prod pod + # (`head-bucket` on k3s-prod-backup → BucketRegion WNAM; PUT and DELETE both succeed). + # It is NOT put in base, because base is dev's live MinIO and dev is deliberately STAYING + # there — moving dev to R2 would reset its PITR window and orphan the backups its PROVEN + # restore depends on. Two environments, two targets, isolated by URL *and* by bucket. - op: replace path: /data/S3_BACKUP_ENDPOINT - value: "TBD-PROD-S3-BACKUP-ENDPOINT" + value: "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com" + # R2 requires the region string to be exactly `auto`. (`head-bucket` answers + # `{"BucketRegion":"WNAM"}` — that is a PHYSICAL LOCATION HINT, not a signing region; using it + # here produces AuthorizationHeaderMalformed.) Dev keeps the base `us-east-1` its live MinIO + # has always been signed with. + - op: replace + path: /data/S3_BACKUP_REGION + value: "auto" + # ------------------------------------------------------------------------------------------ + # OBJECT TENANCY INSIDE THE ONE BUCKET. The operator created a SINGLE R2 bucket, so velero and + # barman cannot be separated by bucket the way they are on dev's MinIO (`velero` and `cnpg`). + # They are separated by disjoint TOP-LEVEL PREFIXES instead: + # + # s3://k3s-prod-backup/velero/… Velero: kopia repo + backup metadata + # s3://k3s-prod-backup/barman//… CNPG: WAL + base backups + # + # The bucket ROOT is left empty on purpose — a misrendered prefix then lands somewhere + # obviously wrong instead of silently colliding with a live tenant. Velero's retention TTL + # deletes objects under its own prefix only, so it can never reap a WAL segment; and barman's + # per-cluster serverName subdirectory keeps the three Postgres servers' chains apart, which + # matters because interleaving two servers' WAL into one prefix corrupts the chain outright. + # Dev cannot be reached from any of these: different endpoint AND different bucket names. + # ------------------------------------------------------------------------------------------ + - op: replace + path: /data/VELERO_BACKUP_BUCKET + value: "k3s-prod-backup" + - op: replace + path: /data/VELERO_BACKUP_PREFIX + value: "velero" + - op: replace + path: /data/CNPG_BACKUP_PATH + value: "s3://k3s-prod-backup/barman" # TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist. diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml index 4a71b21..22a841c 100644 --- a/platform/keycloak/postgres-cluster.yaml +++ b/platform/keycloak/postgres-cluster.yaml @@ -34,18 +34,29 @@ spec: memory: 256Mi limits: memory: 512Mi - # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket --- + # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → object storage --- # Continuous WAL archiving + periodic base backups give proper Postgres PITR (superior to a Velero - # PVC snapshot for a live DB). destinationPath is the BUCKET ROOT; CNPG appends serverName (defaults - # to the Cluster name), so objects land under s3://cnpg/keycloak-db/. Path-style HTTP MinIO endpoint. + # PVC snapshot for a live DB). CNPG appends `serverName` (defaults to the Cluster name) to + # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/keycloak-db/ — dev + # s3://cnpg/keycloak-db/ on the on-Proxmox MinIO, prod s3://k3s-prod-backup/barman/keycloak-db/ on + # Cloudflare R2. Both endpointURL and destinationPath are cluster-vars knobs (R-ENV-02), + # substituted by the `platform-config` Kustomization (this file sits under platform/keycloak/ but + # platform-config is the only tier that lists it as a resource — see its kustomization.yaml). # Credentials: the scoped key from the in-namespace SOPS Secret keycloak-db-backup-credentials - # (a Pod only reads same-ns Secrets), kept in sync with platform/backup/minio-credentials.sops.yaml. - # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000) — prod - # MUST point at its own off-cluster target. Substituted by the `platform-config` Kustomization. + # (a Pod only reads same-ns Secrets). + # + # *** archive_mode IS ON, SO A FAILING ARCHIVE COMMAND IS A DISK-FULL CLOCK, NOT A DEGRADED + # BACKUP. *** Postgres will not recycle a WAL segment it has not successfully archived, so + # whenever these two values are wrong the pgdata PVC fills monotonically until the primary + # PANICs. This is not hypothetical: prod ran from bootstrap to 2026-09-07 with + # `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT` (barman-cloud-wal-archive exit 4 on every segment) + # and had burned ~579 MiB of a 5 GiB PVC on the way to taking SSO — and with it kubectl OIDC, + # Grafana, Headlamp, Gitea and forward-auth — down with it. Never leave a placeholder here, and + # treat `ContinuousArchiving=False` on this Cluster as an availability incident, not a backup one. backup: retentionPolicy: "30d" # keep 30 days of base backups + WAL (PITR window); prunes older barmanObjectStore: - destinationPath: s3://cnpg + destinationPath: ${CNPG_BACKUP_PATH} endpointURL: ${S3_BACKUP_ENDPOINT} s3Credentials: accessKeyId: diff --git a/platform/velero/helmrelease.yaml b/platform/velero/helmrelease.yaml index b1fb799..0391158 100644 --- a/platform/velero/helmrelease.yaml +++ b/platform/velero/helmrelease.yaml @@ -1,18 +1,29 @@ --- # Velero — off-cluster backup/DR of cluster resources + PVC data (ADR-0003, R-STATE-03). LIVE. # -# TARGET: the MinIO S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars; dev http://192.168.1.9:9000 on -# Proxmox — plain HTTP, path-style), bucket -# `velero`. region us-east-1 is a placeholder MinIO ignores (S3 clients still require one). +# TARGET: the S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars) at +# s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX}, in region ${S3_BACKUP_REGION}. Four knobs, +# because the two environments' object stores differ in shape as well as address: +# dev http://192.168.1.9:9000 bucket `velero`, NO prefix, region us-east-1 (on-Proxmox MinIO, +# plain HTTP, path-style; MinIO ignores the region) +# prod https://.r2.cloudflarestorage.com bucket `k3s-prod-backup`, prefix `velero`, +# region `auto` (Cloudflare R2; prod cannot route to that MinIO, +# and R2 demands exactly `auto`) +# Prod's bucket is SHARED with CNPG/barman (the operator created one bucket), so the `velero` prefix +# is what keeps this controller's retention sweep away from the Postgres WAL stream. See the +# "OBJECT-STORE TENANCY" block in clusters/base/cluster-vars.yaml. # # WHY THIS SHAPE: # - provider aws + velero-plugin-for-aws (init container) drives the S3-compatible MinIO target. # - File System Backup (kopia) via the node-agent DaemonSet — storage-agnostic PVC backup that does # NOT depend on CSI VolumeSnapshots or a data-mover (the simplest path that still captures the # Longhorn PVC *data*). snapshotsEnabled:false → no VolumeSnapshotLocation is created/needed. -# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero) — the -# SCOPED MinIO key limited to the velero+cnpg buckets. Kept in sync with the staging source of -# truth platform/backup/minio-credentials.sops.yaml and the Proxmox host ansible-vault. +# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero). ONE +# Secret serves both environments, and it now carries the Cloudflare R2 API token (written by +# platform/backup/set-r2-credentials.sh, 2026-09-07) rather than the scoped MinIO key. That is +# safe only because the R2 token is bucket-scoped to k3s-prod-backup, which dev never names: +# dev's Velero would fail loudly against its MinIO endpoint with an R2 key, it could not +# silently write somewhere unexpected. Rotating either target means re-running that script. # # VOLUME OPT-OUT (per the sizing decision) is implemented with TWO schedules (a single Backup has one # cluster-wide defaultVolumesToFsBackup, so per-namespace volume policy needs two): see `schedules`. @@ -72,12 +83,36 @@ spec: backupStorageLocation: - name: default provider: aws - bucket: velero + # Bucket and prefix are BOTH knobs (cluster-vars), because the two environments' object + # stores are shaped differently. Dev's MinIO has a dedicated `velero` bucket and dev's + # backups sit at its ROOT, so dev renders bucket=velero with an EMPTY prefix — and the + # chart omits `objectStorage.prefix` entirely when it is empty (verified, chart 12.1.0), + # leaving dev's live BackupStorageLocation byte-unchanged. Prod's operator created ONE R2 + # bucket shared with CNPG/barman, so prod renders bucket=k3s-prod-backup with prefix + # `velero` — a disjoint top-level prefix from barman's. Getting this wrong is not a + # cosmetic error: Velero ADOPTS every backup it finds under its location and TTL-DELETES + # it on retention, so a prefix that overlaps another tenant deletes that tenant's data. + bucket: ${VELERO_BACKUP_BUCKET} + prefix: "${VELERO_BACKUP_PREFIX}" default: true config: - region: us-east-1 # placeholder; MinIO ignores it (S3 clients require one) - s3Url: ${S3_BACKUP_ENDPOINT} # on-Proxmox MinIO, plain HTTP (cluster-vars) - s3ForcePathStyle: "true" # MinIO is path-style + # Region is a knob, not a constant: S3 clients require SOME region, MinIO ignores + # whatever it is, and Cloudflare R2 requires exactly `auto`. Hardcoding `us-east-1` + # was correct-by-accident and broke the moment the target stopped being MinIO. + region: ${S3_BACKUP_REGION} + s3Url: ${S3_BACKUP_ENDPOINT} # cluster-vars; dev MinIO (http) or prod R2 (https) + s3ForcePathStyle: "true" # MinIO is path-style; R2 accepts path-style too + # *** REQUIRED FOR CLOUDFLARE R2 — DO NOT DROP THIS LINE. *** + # velero-plugin-for-aws v1.14.2 bundles an AWS SDK with flexible checksums ON by + # default, so it sends `x-amz-sdk-checksum-algorithm` / `x-amz-checksum-crc32` on every + # PUT. R2 REJECTS those headers with HTTP 400 InvalidArgument, which would fail every + # backup AND every kopia blob upload. The empty string is the documented opt-out, and + # it must be PRESENT-but-empty rather than absent — the plugin branches on the key + # existing, and the chart does emit an empty value (verified, chart 12.1.0). Harmless + # against MinIO, which ignores it, so it is stated unconditionally rather than as a + # fourth knob. If backups start failing with InvalidArgument after a plugin bump, look + # here first. + checksumAlgorithm: "" credentials: existingSecret: velero-cloud-credentials # SOPS Secret carrying the `cloud` INI key (ns velero) # --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data diff --git a/platform/velero/values.yaml b/platform/velero/values.yaml index 5a3ae5a..e73db5e 100644 --- a/platform/velero/values.yaml +++ b/platform/velero/values.yaml @@ -2,8 +2,13 @@ # doc pointer, kept for parity with the other components' layout). # # LIVE wiring (see helmrelease.yaml): -# - provider aws + velero-plugin-for-aws (init container) → MinIO S3 ${S3_BACKUP_ENDPOINT} (dev: http://192.168.1.9:9000) -# (path-style, bucket `velero`, region us-east-1 placeholder). +# - provider aws + velero-plugin-for-aws (init container) → S3 ${S3_BACKUP_ENDPOINT}, path-style, +# at s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX} in region ${S3_BACKUP_REGION}. Four +# cluster-vars knobs: dev = the on-Proxmox MinIO (http://192.168.1.9:9000, bucket `velero`, no +# prefix, us-east-1); prod = Cloudflare R2 (bucket `k3s-prod-backup`, prefix `velero`, `auto`), +# sharing one bucket with CNPG/barman under a disjoint prefix. +# - config.checksumAlgorithm: "" — MANDATORY for R2, which rejects the AWS SDK's +# x-amz-checksum-* headers with HTTP 400 InvalidArgument. Harmless against MinIO. # - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover). # - snapshotsEnabled:false → no VolumeSnapshotLocation. # - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero).