diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index c8b8f37..807dd60 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -154,11 +154,29 @@ data: UNIFI_L4_VIP: "192.168.1.54" # UniFi controller VM: inform/STUN/adoption (apps/vms/unifi) CRAFTY_L4_VIP: "192.168.1.55" # Crafty: Minecraft Java (TCP) + Bedrock (UDP) game ports - # Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style. - # Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db, - # nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port. - # Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR. - S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000" + # Off-cluster S3 backup target (R-STATE-03, R-IDP-05). Consumed by BOTH backup paths: every CNPG + # Cluster's barmanObjectStore.endpointURL (keycloak-db, nextcloud-db, semaphore-db) and Velero's + # backupStorageLocation s3Url. Full URL including scheme (and port, if not the scheme default). + # + # OPERATOR DECISION 2026-09-05: **Cloudflare R2**, replacing the on-Proxmox MinIO — "for the time + # being until a better on-prem backup solution is created". Two reasons beyond cost: + # 1. The k3s prod cluster is on the SOHO network (pfSense's WAN) and CANNOT reach the MinIO inside + # pfSense's LAN — measured, from a pod: connection timed out. R2 needs no firewall change. + # 2. That MinIO runs on the SAME Proxmox host as dev's VMs, so dev's "off-cluster" backup shared a + # physical failure domain with dev. One host loss took the cluster AND its backups. R2 is + # genuinely off-site. + # Measured to justify the cost: dev's whole estate is 4.0 GiB Velero (6,264 objects, 7d retention) + + # 211 MiB CNPG (1,755 objects) — pennies per month, and R2 egress is free, so restore REHEARSALS + # stay free and R-STATE-03 has no cost disincentive to stay proven. + # + # The MinIO copy is NOT deleted — it stays as a historical/local copy (3-2-1). Restoring from it + # means pointing this knob back at http://192.168.1.9:9000 temporarily. + S3_BACKUP_ENDPOINT: "https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com" + + # S3 region. Required by every S3 client, IGNORED by MinIO, and must be exactly `auto` for R2. + # This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-accident for + # MinIO and wrong for R2, which is why it is a knob now. + S3_BACKUP_REGION: "auto" # --- storage (ADR-0003) --- LONGHORN_DEFAULT_REPLICA_COUNT: "3" # default StorageClass replica count (3 on 3 nodes) @@ -227,12 +245,23 @@ data: # interleaving WAL into one Barman stream CORRUPTS THE CHAIN and destroys PITR — and # dev's DR restore is PROVEN (R-STATE-03). This is the worst of the two. # - # THE BASE VALUES ARE DEV'S CURRENT LAYOUT ON PURPOSE, and must stay that way: dev's existing - # backups live at the bucket root / at `s3://cnpg`. Giving dev a prefix would ORPHAN the very - # backups its proven restore depends on — the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays - # `default`. Prod, which has no history to preserve, takes the namespaced paths. - VELERO_BACKUP_PREFIX: "" # dev: bucket root (existing layout) | prod: "prod" - CNPG_BACKUP_PATH: "s3://cnpg" # dev: bucket root (existing layout) | prod: "s3://cnpg/prod" + # Both environments are now NAMESPACED SYMMETRICALLY, which was not possible an hour ago and is + # worth explaining, because the earlier reasoning was the opposite: + # + # While the target was the MinIO, dev's base value HAD to be the bucket root — its existing + # backups lived there and giving dev a prefix would have ORPHANED the very backups its proven + # restore depends on (the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default`). + # + # Moving to R2 starts BOTH clusters on an EMPTY bucket, so there is no history to orphan and no + # reason to keep the asymmetry. `dev` and `prod` is clearer, greppable, and leaves the bucket root + # free — so an unset/misrendered prefix lands somewhere obviously wrong instead of silently + # colliding with a real environment's backups. + # + # *** THESE TWO VALUES AND S3_BACKUP_ENDPOINT MUST CHANGE TOGETHER. *** Setting a `dev` prefix while + # the endpoint is still the MinIO would orphan dev's existing MinIO backups for no benefit. They are + # one change; do not split the commit. + VELERO_BACKUP_PREFIX: "dev" # dev: "dev" | prod: "prod" + CNPG_BACKUP_PATH: "s3://cnpg/dev" # dev: s3://cnpg/dev | prod: s3://cnpg/prod # --- placement --- PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os" # base no-op selector; prod may pin roles diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 71f7533..ac4632c 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -266,11 +266,20 @@ patches: - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.1.55" - # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG - # barmanObjectStore and Velero's backupStorageLocation write to. - - op: replace - path: /data/S3_BACKUP_ENDPOINT - value: "http://192.168.1.9:9000" + # OFF-CLUSTER BACKUP TARGET — no op here any more, deliberately. + # + # This used to pin the on-Proxmox MinIO (http://192.168.1.9:9000). As of 2026-09-05 the operator + # moved backups to Cloudflare R2 "for the time being until a better on-prem backup solution is + # created", and R2 is a single environment-INDEPENDENT endpoint: both clusters write to the same + # buckets and are isolated by VELERO_BACKUP_PREFIX / CNPG_BACKUP_PATH (dev vs prod), not by URL. + # Under R-ENV-02 a value that does not differ per environment belongs in base and must NOT be + # restated here — two copies of one endpoint is just a rotation waiting to drift. + # + # *** MERGING THIS RESETS DEV'S PITR WINDOW. *** Dev's R2 prefix starts EMPTY, so Velero writes a + # fresh kopia repository and CNPG takes new base backups; the 30-day Barman window rebuilds from + # zero. The existing MinIO copy is NOT deleted and stays restorable — point S3_BACKUP_ENDPOINT + # back at http://192.168.1.9:9000 (and VELERO_BACKUP_PREFIX back to "") to reach it. Do not merge + # this while dev's proven restore is the only recovery path you are relying on. # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas. - op: replace path: /data/LONGHORN_DEFAULT_REPLICA_COUNT diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index f93420d..3abe7a6 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -384,38 +384,36 @@ patches: path: /data/CRAFTY_L4_VIP value: "192.168.2.22" # ------------------------------------------------------------------------------------------ - # OFF-CLUSTER DR TARGET — operator ruling 2026-09-05: "the S3 backup location is the same as - # Dev." Prod therefore points at the on-Proxmox MinIO, and the TBD placeholder is retired. - # Stated EXPLICITLY rather than omitting the op: an omitted op would inherit the same value - # silently, and a shared backup target is exactly the kind of decision that must be visible - # in the overlay that owns it. - # ------------------------------------------------------------------------------------------ - # *** NOT YET REACHABLE — velero and the CNPG backups CANNOT be enabled on prod today. *** - # Measured 2026-09-05 from a throwaway pod INSIDE the prod cluster (not from the workstation, - # which proves nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` - # → connection timed out after 15s. EXPECTED TOPOLOGY: prod is NOT behind pfSense — it is on the - # SOHO network 192.168.2.0/24, which is pfSense's WAN, while MinIO sits inside pfSense's LAN - # 192.168.1.0/24. Reaching it means crossing INTO pfSense from its WAN, which needs a rule - # MinIO. Until that routing exists, Velero and CNPG would install cleanly and fail every - # backup at RUNTIME — a tier that reports healthy while no backup exists. - # Re-test with: bootstrap/standup.sh ... --s3-endpoint http://192.168.1.9:9000 s3check + # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05). # - # SHARING THE ENDPOINT IS SAFE ONLY BECAUSE OF THE TWO PATH KNOBS BELOW. Nothing about the - # paths was per-environment before this change: Velero used one unprefixed bucket, and every - # CNPG Cluster used `s3://cnpg` with serverName defaulting to the CLUSTER NAME — and both - # environments have keycloak-db / nextcloud-db / semaphore-db. Enabling prod against dev's - # MinIO without these knobs would (a) let prod adopt and TTL-delete dev's Velero backups and - # (b) interleave two Postgres servers' WAL into one Barman stream, CORRUPTING dev's chain and - # destroying the PITR that R-STATE-03 proved. See clusters/base/cluster-vars.yaml. - - op: replace - path: /data/S3_BACKUP_ENDPOINT - value: "http://192.168.1.9:9000" - # Prod's Velero objects nest under `prod/`; dev keeps the bucket root where its backups already - # are (renaming dev's path would orphan the backups its proven restore depends on). + # THERE IS DELIBERATELY NO `S3_BACKUP_ENDPOINT` OP HERE, and that is a reversal of the rule + # stated at the top of this overlay ("an OMITTED op means prod silently inherits a dev + # address"). That rule exists because base defaults are dev's LAN addresses. R2 is not one: it + # is a single shared, environment-independent endpoint, so under R-ENV-02 it belongs in base and + # overriding it with an identical value would only create two places to change on a rotation — + # the exact drift this repo has been bitten by before. Isolation does NOT come from the + # endpoint; it comes from the two path knobs below. + # + # WHY R2 AND NOT THE MinIO the operator first named: prod is on the SOHO network + # 192.168.2.0/24, which is pfSense's WAN, while the MinIO sits inside pfSense's LAN + # 192.168.1.0/24. Measured from a pod INSIDE prod (the workstation proves nothing about pod + # egress): `curl http://192.168.1.9:9000/minio/health/live` → connection timed out after 15s. + # Reaching it would need a WAN→LAN rule that does not exist. R2 needs no firewall change, and + # unlike that MinIO — which runs on the same Proxmox host as dev's VMs — it is genuinely + # off-site. Verify prod can reach the new endpoint before resuming velero: + # bootstrap/standup.sh ... --s3-endpoint s3check + # + # THE PATH KNOBS ARE WHAT MAKE ONE SHARED BUCKET SAFE. Nothing about the paths was + # per-environment before 2026-09-05: Velero used one unprefixed bucket, and every CNPG Cluster + # used `s3://cnpg` with serverName defaulting to the CLUSTER NAME — and both environments have + # keycloak-db / nextcloud-db / semaphore-db. Without these knobs prod would (a) adopt and + # TTL-delete dev's Velero backups and (b) interleave two Postgres servers' WAL into ONE Barman + # stream, corrupting the chain and destroying PITR for both. See clusters/base/cluster-vars.yaml. + # ------------------------------------------------------------------------------------------ - op: replace path: /data/VELERO_BACKUP_PREFIX value: "prod" - # Prod's Barman objects nest under s3://cnpg/prod//; dev keeps s3://cnpg//. + # Barman objects land at s3://cnpg/prod//; dev's at s3://cnpg/dev//. - op: replace path: /data/CNPG_BACKUP_PATH value: "s3://cnpg/prod" diff --git a/platform/velero/helmrelease.yaml b/platform/velero/helmrelease.yaml index 477b5a6..8f5d2c9 100644 --- a/platform/velero/helmrelease.yaml +++ b/platform/velero/helmrelease.yaml @@ -81,9 +81,20 @@ spec: prefix: ${VELERO_BACKUP_PREFIX} default: true config: - region: us-east-1 # placeholder; MinIO ignores it (S3 clients require one) - s3Url: ${S3_BACKUP_ENDPOINT} # on-Proxmox MinIO, plain HTTP (cluster-vars) - s3ForcePathStyle: "true" # MinIO is path-style + # Region is a knob, not a constant: S3 clients require SOME region, MinIO ignores whatever + # it is, and Cloudflare R2 requires exactly `auto`. Hardcoding `us-east-1` silently broke + # the moment the target stopped being MinIO. + region: ${S3_BACKUP_REGION} + s3Url: ${S3_BACKUP_ENDPOINT} # cluster-vars; R2 (https) or a MinIO/versitygw (http) + s3ForcePathStyle: "true" # MinIO is path-style; R2 accepts path-style too + # *** REQUIRED FOR CLOUDFLARE R2 — DO NOT DROP THIS LINE. *** + # velero-plugin-for-aws v1.14+ bundles an AWS SDK that sends `x-amz-sdk-checksum-algorithm` + # / `x-amz-checksum-crc32` on every PUT. R2 REJECTS those headers (HTTP 400 + # "InvalidArgument"), so every backup and every kopia blob upload fails. Setting this to the + # empty string disables the SDK's flexible-checksum behaviour. It is harmless against MinIO, + # which simply ignores it — so it is stated unconditionally rather than per-environment. + # If backups start failing with InvalidArgument after a plugin bump, look here first. + checksumAlgorithm: "" credentials: existingSecret: velero-cloud-credentials # SOPS Secret carrying the `cloud` INI key (ns velero) # --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data