From d1248e6778d830623da54e9153c0227dbc353ede Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Fri, 28 Aug 2026 22:37:17 -0500 Subject: [PATCH] feat(backup): wire Velero + CNPG Barman to the live MinIO S3 target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author the GitOps side of off-cluster backups against the on-Proxmox MinIO (http://192.168.1.9:9000, path-style; buckets velero + cnpg). Velero (R-STATE-03): - HelmRelease (chart 8.x / Velero 1.15) + velero-plugin-for-aws init container + node-agent kopia File System Backup (deployNodeAgent, uploaderType=kopia, snapshotsEnabled=false). BSL default -> bucket velero, s3ForcePathStyle. - Two daily Schedules: `daily` backs up the whole cluster's resources+PVC data (defaultVolumesToFsBackup opt-out) EXCEPT monitoring+unifi; `daily-resources-only` captures monitoring+unifi RESOURCES only (Prometheus TSDB, Loki 72h logs, and the ~7.4GiB UniFi VM disk are reconstructable — excluded from volume backup). - velero-cloud-credentials SOPS Secret (cloud INI key, ns velero). - Own non-gating Flux tier clusters/base/backup.yaml (dependsOn platform-config, wait:false) + clusters/dev SOPS decryption patch — mirrors headlamp. CNPG native Barman PITR (R-IDP-05 / R-STATE-03): - keycloak-db, nextcloud-db, semaphore-db gain spec.backup.barmanObjectStore (s3://cnpg/, endpointURL MinIO, gzip WAL+data, 30d retention) + a daily ScheduledBackup (6-field cron, immediate) each. - Per-namespace scoped-key SOPS Secrets (-backup-credentials), same-ns only. - render.py: emit-waves now recognizes -db-backup-credentials.sops.yaml as a DB-wave sidecar (+ self-test); wave-0 kustomization regenerated. Creds mirror platform/backup/minio-credentials.sops.yaml (scoped key k3s-backup), SOPS round-trip verified with the cluster age key. gate: green. Co-Authored-By: Claude Opus 4.8 --- apps/wave-0/kustomization.yaml | 2 + .../nextcloud-db-backup-credentials.sops.yaml | 29 +++++ apps/wave-0/nextcloud-db.yaml | 42 ++++++- .../semaphore-db-backup-credentials.sops.yaml | 29 +++++ apps/wave-0/semaphore-db.yaml | 42 ++++++- catalog/render.py | 10 +- clusters/base/backup.yaml | 39 +++++++ clusters/base/kustomization.yaml | 4 + clusters/dev/kustomization.yaml | 14 +++ .../keycloak-db-backup-credentials.sops.yaml | 29 +++++ platform/keycloak/kustomization.yaml | 4 + platform/keycloak/postgres-cluster.yaml | 45 +++++++- platform/velero/README.md | 52 +++++++-- platform/velero/helmrelease.yaml | 105 +++++++++++++++--- platform/velero/kustomization.yaml | 4 + platform/velero/values.yaml | 17 ++- platform/velero/velero-cloud.sops.yaml | 28 +++++ 17 files changed, 446 insertions(+), 49 deletions(-) create mode 100644 apps/wave-0/nextcloud-db-backup-credentials.sops.yaml create mode 100644 apps/wave-0/semaphore-db-backup-credentials.sops.yaml create mode 100644 clusters/base/backup.yaml create mode 100644 platform/keycloak/keycloak-db-backup-credentials.sops.yaml create mode 100644 platform/velero/velero-cloud.sops.yaml diff --git a/apps/wave-0/kustomization.yaml b/apps/wave-0/kustomization.yaml index b3d539a..3a76461 100644 --- a/apps/wave-0/kustomization.yaml +++ b/apps/wave-0/kustomization.yaml @@ -10,9 +10,11 @@ resources: - gitea.yaml - jackett.yaml - nextcloud-db-app.sops.yaml +- nextcloud-db-backup-credentials.sops.yaml - nextcloud-db.yaml - pihole.yaml - plex.yaml - qbittorrent.yaml - semaphore-db-app.sops.yaml +- semaphore-db-backup-credentials.sops.yaml - semaphore-db.yaml diff --git a/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml b/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml new file mode 100644 index 0000000..3cef69f --- /dev/null +++ b/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nextcloud-db-backup-credentials + namespace: nextcloud-db +type: Opaque +stringData: + ACCESS_KEY_ID: ENC[AES256_GCM,data:gS533S1h2f5uWQ==,iv:XmcNCYdlkAIewn7Br8Kk4CHcZWbMQxJE0Mlge9/oGRY=,tag:+krC/C+EMQl5pYgwkuut1g==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:NTcBY05zUy8ngV3phNF18FBHRCEKQGu2KQ2yQGu4YiuJasllznZ+YvCBn/3i0Z7/,iv:uXyenHmNqBRb5CC/ELgVbFXV/W59ZPOrSeGwIAi8Hz0=,tag:9ZRcKL0ijOPBi4QUbDZyoQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFMlBnNy94T2tlUUtaYTc0 + Q0FKcDB3K2xPY2QvRWRyWTlTMnhWK0NlekFRCnpacHkwNHVONmRlR0xYNWQyVUhk + eUdVNTNVQjRsTm40cHBRWVFvbkJ0eTAKLS0tIGVJaE5ja0F1ejJOUkZKOWlXbXpD + ODAyK2g4bGlXeXV0Q1FMbXNiSEpUYncKvwviYgI/XjCBJG5kpS8tOHLTLsnXv1bl + b4E4rigNesmreSYxAyhGw9bgQNY6ba/F3Ecexrdx+iV/tYYc/VoDKA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-08-29T03:31:48Z" + mac: ENC[AES256_GCM,data:ks4i7zjrJQajKdncnnV5dD4Ap1Q1xdKyiQAiqhao2USXu9S+4flX2kzc9lv8lFG/dTiWvLu6Tqgmj46ssiqtd0E8DnrASNEuKz/Xlw1iCka2008C5BhE/k8oy84kXXDgEuo7GiYdhJU/mAaAnY4p10ZQkvTKddQMmSO65YRwRMo=,iv:skZoS+bhAOnLlY5R5OfZvFV8yCEiEKtLac4mVbdmZ2s=,tag:6T1JIroe5hI8RXWAyzEKLw==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.4 diff --git a/apps/wave-0/nextcloud-db.yaml b/apps/wave-0/nextcloud-db.yaml index edb8fb1..c61f487 100644 --- a/apps/wave-0/nextcloud-db.yaml +++ b/apps/wave-0/nextcloud-db.yaml @@ -68,7 +68,41 @@ spec: memory: 256Mi limits: memory: 512Mi - # --- Off-cluster DR backup (R-STATE-03) — OPERATOR HAND-OFF (OQ-BKP, shared with Velero, Phase 4) --- - # When the off-cluster backup target is chosen, add a Barman object store + a ScheduledBackup here - # (same hand-off as keycloak-db). Until then the DB is protected by 3-replica Longhorn + (Phase-4) - # Velero PVC backups of the namespace; the SD declares backup.policy daily-offcluster as the intent. + # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket --- + # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's + # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName + # (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP. + # Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials + # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml. + backup: + retentionPolicy: "30d" + barmanObjectStore: + destinationPath: s3://cnpg + endpointURL: http://192.168.1.9:9000 + s3Credentials: + accessKeyId: + name: nextcloud-db-backup-credentials + key: ACCESS_KEY_ID + secretAccessKey: + name: nextcloud-db-backup-credentials + key: ACCESS_SECRET_KEY + wal: + compression: gzip + maxParallel: 2 + data: + compression: gzip +--- +# Daily base backup of nextcloud-db (WAL archived continuously by the barmanObjectStore above). CNPG +# cron is 6-field (sec min hour dom mon dow); immediate:true seeds the first base backup. 02:00 slot. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: nextcloud-db-daily + namespace: nextcloud-db +spec: + immediate: true + schedule: "0 0 2 * * *" + backupOwnerReference: self + cluster: + name: nextcloud-db + method: barmanObjectStore diff --git a/apps/wave-0/semaphore-db-backup-credentials.sops.yaml b/apps/wave-0/semaphore-db-backup-credentials.sops.yaml new file mode 100644 index 0000000..d7a4574 --- /dev/null +++ b/apps/wave-0/semaphore-db-backup-credentials.sops.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Secret +metadata: + name: semaphore-db-backup-credentials + namespace: semaphore-db +type: Opaque +stringData: + ACCESS_KEY_ID: ENC[AES256_GCM,data:jmhu9U7Lyo0ugA==,iv:EyU67JLLO10za1tM/peju9tvBBINJ7ZDn6F9Zu0g6H8=,tag:9M/adv8Gdu3XNBJ/G4xfeA==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:f9qW7BX5OQ2L7FqZ6pr6lqdEhTy+7xabLSMTnHZ/UYj1RnQ9asL+/HNkEWsMatJC,iv:hS2Q5l4EsvMBop/sBnld83tm2MhsTTI5N72BLGLMdtk=,tag:MwfC205a/djUA6HNqobcUw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwU3VmQmFzQzUzTGFSVkxq + enlnVmRhTSt6V2xRMnF5ZXdpWlpsVDZZckg0CnY1eWRVNTVHMDhZaVpiczhLNGdW + Rjg3RWp4MWtUdnhjd0ZBUTZmWVY1WHMKLS0tIFlxYk1VcjBCR2Npbi93YklzUGF0 + cGpweDl4VjNsNHM0Sm8zVDBWcGFJanMKpBBV9cnSZRVmIGjEi9YBMlbm4df253cS + osMRBCKizDIz1Q0Awq952y/L97ljUnSE6JWnlKTriHoGtLDct4mg3Q== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-08-29T03:31:48Z" + mac: ENC[AES256_GCM,data:uqtBwfubrUqaRN5r5qfMlITBvKxfbdyxZGDcZgyWtmDvX1OulKBvuOYBZaDW1MdIrq++nbKhocQK2PkuAmqZI3NbfUZ5mrGWsIvD3mImrfyiOnASvnCMwIlMTHlcJbJTIY+8JLmzbr+2TPbZ7zVechR4EoX9tEykY+GZml7cTic=,iv:7qoBRE+JlmArMejf++orrccAPDOpJsDp4vA+UHYCkoA=,tag:IwE7rZCXB+UhiP0yWWIFGA==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.4 diff --git a/apps/wave-0/semaphore-db.yaml b/apps/wave-0/semaphore-db.yaml index 5ae919e..cdd5884 100644 --- a/apps/wave-0/semaphore-db.yaml +++ b/apps/wave-0/semaphore-db.yaml @@ -69,7 +69,41 @@ spec: memory: 256Mi limits: memory: 512Mi - # --- Off-cluster DR backup (R-STATE-03) — OPERATOR HAND-OFF (OQ-BKP, shared with Velero, Phase 4) --- - # When the off-cluster backup target is chosen, add a Barman object store + a ScheduledBackup here - # (same hand-off as keycloak-db / nextcloud-db). Until then the DB is protected by 3-replica Longhorn - # + (Phase-4) Velero PVC backups of the namespace; the SD declares backup.policy daily-offcluster. + # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket --- + # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's + # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName + # (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP. + # Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials + # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml. + backup: + retentionPolicy: "30d" + barmanObjectStore: + destinationPath: s3://cnpg + endpointURL: http://192.168.1.9:9000 + s3Credentials: + accessKeyId: + name: semaphore-db-backup-credentials + key: ACCESS_KEY_ID + secretAccessKey: + name: semaphore-db-backup-credentials + key: ACCESS_SECRET_KEY + wal: + compression: gzip + maxParallel: 2 + data: + compression: gzip +--- +# Daily base backup of semaphore-db (WAL archived continuously by the barmanObjectStore above). CNPG +# cron is 6-field (sec min hour dom mon dow); immediate:true seeds the first base backup. 04:00 slot. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: semaphore-db-daily + namespace: semaphore-db +spec: + immediate: true + schedule: "0 0 4 * * *" + backupOwnerReference: self + cluster: + name: semaphore-db + method: barmanObjectStore diff --git a/catalog/render.py b/catalog/render.py index d74c0b8..dee946b 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -1038,6 +1038,7 @@ def plan_apps_waves(sds: list, present_files: set) -> dict: - `-bootstrap-secret.sops.yaml` -> wave(svc) (consumed by svc's env, same apply) - `-db.yaml` -> wave(svc-db) (a strictly-lower wave; CNPG DB) - `-db-app.sops.yaml` -> wave(svc-db) (CNPG initdb owner secret) + - `-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret) Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset. Only files that actually exist in `present_files` are placed, so the plan is robust to a service having no bootstrap secret or no DB. Deterministic (sorted). @@ -1060,6 +1061,7 @@ def plan_apps_waves(sds: list, present_files: set) -> dict: wdb = waves[db] add(wdb, f"{db}.yaml") add(wdb, f"{db}-app.sops.yaml") + add(wdb, f"{db}-backup-credentials.sops.yaml") # CNPG Barman S3 creds (R-STATE-03) for unit, cfg in EXTRA_UNITS.items(): anchor = cfg["anchor"] @@ -1453,7 +1455,8 @@ def _selftest() -> int: # EXTRA_UNITS anchor strictly above; a file absent from the tree is never placed. present = { "a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml", - "svc-db.yaml", "svc-db-app.sops.yaml", "gitea.yaml", + "svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml", + "gitea.yaml", "gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml", } plan_sds = wsds + [{"name": "gitea", "source": {"image": "x:1"}}, @@ -1461,8 +1464,9 @@ def _selftest() -> int: plan = plan_apps_waves(plan_sds, present) flat = {f: wv for wv, fs in plan.items() for f in fs} check(flat.get("a.yaml") == 0, "leaf app must land in wave 0") - check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0, - "CNPG DB manifest + its owner secret must be in the DB's (lower) wave") + check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0 + and flat.get("svc-db-backup-credentials.sops.yaml") == 0, + "CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave") check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1, "a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB") check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0") diff --git a/clusters/base/backup.yaml b/clusters/base/backup.yaml new file mode 100644 index 0000000..5e7fe6e --- /dev/null +++ b/clusters/base/backup.yaml @@ -0,0 +1,39 @@ +--- +# Flux Kustomization: Velero backup controller — reconciles ./platform/velero. +# +# OWN NON-GATING TIER (dependsOn platform-config, wait:false) — the same rationale as headlamp/ +# oauth2-proxy/crafty. Velero is a BACKUP CONTROLLER; it must NEVER gate the platform graph. Its +# HelmRelease (chart + AWS plugin init-container + node-agent DaemonSet) can legitimately be +# not-Ready during a cold start (image pulls, the node-agent rolling out on every node, the BSL +# doing its first reachability probe against MinIO). If it sat in the gating `platform` tier, that +# transient unhealth would stall platform-config + every apps wave/vm on "dependency not ready" — +# exactly the failure headlamp already encodes. Nothing in the graph dependsOn backup, and wait:false +# guarantees its health can NEVER gate anything even if a future resource is (mistakenly) pointed at it. +# +# It reconciles AFTER platform-config only so the storage/CRD surface it backs up already exists; the +# backup path itself (Velero pods → http://192.168.1.9:9000) is independent of the rest of the graph. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero + namespace: flux-system +spec: + interval: 10m + retryInterval: 2m + timeout: 5m + prune: true + wait: false + dependsOn: + - name: platform-config + sourceRef: + kind: GitRepository + name: flux-system + path: ./platform/velero + # cluster-vars feeds the backup cadence/retention knobs (${VELERO_BACKUP_SCHEDULE}, ${VELERO_RETENTION}). + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-vars + # SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per + # environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how + # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml. diff --git a/clusters/base/kustomization.yaml b/clusters/base/kustomization.yaml index df4ee11..85aa5a6 100644 --- a/clusters/base/kustomization.yaml +++ b/clusters/base/kustomization.yaml @@ -34,4 +34,8 @@ resources: # reverse-proxy sidecar (Option B). POST-platform tier (dependsOn platform-config, wait:false); same # non-gating leaf-UI rationale as headlamp/homepage/oauth2-proxy (a UI must never gate the platform). - crafty.yaml # Flux Kustomization -> ./platform/crafty (dependsOn platform-config) + # Velero backup controller — POST-platform tier (dependsOn platform-config, wait:false). A backup + # controller must NEVER gate the platform chain; same non-gating rationale as headlamp/oauth2-proxy. + # Carries velero-cloud.sops.yaml, so clusters/dev adds a SOPS decryption patch for it (by name). + - backup.yaml # Flux Kustomization -> ./platform/velero (dependsOn platform-config) - cluster-vars.yaml # overlay knob surface (base defaults) diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 74a172c..56b94ba 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -105,6 +105,20 @@ patches: provider: sops secretRef: name: sops-age + # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml). + # It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS + # decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only). + - patch: |- + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: velero + namespace: flux-system + spec: + decryption: + provider: sops + secretRef: + name: sops-age - patch: |- apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository diff --git a/platform/keycloak/keycloak-db-backup-credentials.sops.yaml b/platform/keycloak/keycloak-db-backup-credentials.sops.yaml new file mode 100644 index 0000000..67c5bf0 --- /dev/null +++ b/platform/keycloak/keycloak-db-backup-credentials.sops.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Secret +metadata: + name: keycloak-db-backup-credentials + namespace: keycloak +type: Opaque +stringData: + ACCESS_KEY_ID: ENC[AES256_GCM,data:N7iknaokuZ/tLg==,iv:yaGfsH5vplrNRqGnnpi8lYBGaIPDgTuRr0N2CWXXhh8=,tag:uLS1vvtUFGQu+Lb4b04pPg==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:lV9r1ZXtsSwfaGO42x6hH3zE0cvBWSjcN1jzcvfRzpKdXa45eY68YA0tKdSC+0oy,iv:xs/uQJq+NkasynNRtgq+mGz8bFzgw8ZkuWI/oKR5b2g=,tag:SU/XmxCQGCrkcEsFTHbg9A==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQVpJa1FWSDZudXhxQmJC + ZWlHVlorQk8vOWQxVzVLc3ZpTTVQQlBpbUNjClVwc3gvaVpyQzc5YVdYclp4YlVW + bWFpcFg3bDRvM1ROV0ZFSUZSU1NoeTAKLS0tIEpNSWlzRUM5VnpVY1cyRlY0T2xF + SjNFTy9FVm5GOFZ6WmgrV04ybGQxQjgKSnB2se7Ar1+piB0fn0wKSaOa3MLpetNa + gKPHJKP9OR5+KhlTnspBlxX9Un0WZcvZOQqAdewqlDklv4ukO7PgdQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-08-29T03:31:48Z" + mac: ENC[AES256_GCM,data:sxAENlR+qyBt2cBVG54hLbThXhzfNcKe8k1Ju3NmqWRyYLp/o7WHRwqydWHNYeY0YJJyuv73o1dDkVEPZEI2pxIlhaSwxhx+sDLaJnrYXq3YesbaT7jXkBrzz2MG1xjzV4nXgVYveM4rwCAWLij8qvIsbPMPyCHvaUjG/g0KDQ0=,iv:DoZgcHFsQUzTrStZfYr2bfQR6FGe5whNRF+lpzrXzKE=,tag:vGkrue7N0wjlDFZmQCqVQg==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.4 diff --git a/platform/keycloak/kustomization.yaml b/platform/keycloak/kustomization.yaml index 4bfcae6..20c3bba 100644 --- a/platform/keycloak/kustomization.yaml +++ b/platform/keycloak/kustomization.yaml @@ -28,5 +28,9 @@ resources: - keycloak-admin-credentials.sops.yaml # --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders --- - keycloak-realm-clients.sops.yaml + # --- Scoped MinIO S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05) — SOPS-encrypted, + # decrypted by the `platform` tier (which already has the decryption stanza). Lands in ns + # keycloak; the keycloak-db Cluster (platform-config) reads it by name for WAL archiving. --- + - keycloak-db-backup-credentials.sops.yaml # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) --- - grafana-dashboard.yaml diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml index f156ef7..4a662ff 100644 --- a/platform/keycloak/postgres-cluster.yaml +++ b/platform/keycloak/postgres-cluster.yaml @@ -34,8 +34,43 @@ spec: memory: 256Mi limits: memory: 512Mi - # --- Off-cluster DR backup (R-IDP-05) — OPERATOR HAND-OFF (OQ-BKP, shared with Velero, Phase 4) --- - # When the off-cluster backup target is chosen, add a Barman object store + a ScheduledBackup: - # spec.backup.barmanObjectStore = { destinationPath, endpointURL, s3Credentials {…SOPS…} } - # + a postgresql.cnpg.io/v1 ScheduledBackup (cron) referencing this Cluster. - # Until then the DB is protected by 3-replica Longhorn + (Phase-4) Velero PVC backups of the namespace. + # --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket --- + # Continuous WAL archiving + periodic base backups give proper Postgres PITR (superior to a Velero + # PVC snapshot for a live DB). destinationPath is the BUCKET ROOT; CNPG appends serverName (defaults + # to the Cluster name), so objects land under s3://cnpg/keycloak-db/. Path-style HTTP MinIO endpoint. + # Credentials: the scoped key from the in-namespace SOPS Secret keycloak-db-backup-credentials + # (a Pod only reads same-ns Secrets), kept in sync with platform/backup/minio-credentials.sops.yaml. + backup: + retentionPolicy: "30d" # keep 30 days of base backups + WAL (PITR window); prunes older + barmanObjectStore: + destinationPath: s3://cnpg + endpointURL: http://192.168.1.9:9000 + s3Credentials: + accessKeyId: + name: keycloak-db-backup-credentials + key: ACCESS_KEY_ID + secretAccessKey: + name: keycloak-db-backup-credentials + key: ACCESS_SECRET_KEY + wal: + compression: gzip + maxParallel: 2 + data: + compression: gzip +--- +# Daily base backup of keycloak-db (WAL is archived continuously by the barmanObjectStore above). +# CNPG cron is 6-field (sec min hour dom mon dow). immediate:true takes a first base backup as soon +# as the schedule is created so the PITR chain has a starting point. Staggered 01:00 (nextcloud 02:00, +# semaphore 04:00, Velero 03:00) to avoid concurrent base backups hammering MinIO. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: keycloak-db-daily + namespace: keycloak +spec: + immediate: true + schedule: "0 0 1 * * *" + backupOwnerReference: self + cluster: + name: keycloak-db + method: barmanObjectStore diff --git a/platform/velero/README.md b/platform/velero/README.md index b2136bb..2b62a16 100644 --- a/platform/velero/README.md +++ b/platform/velero/README.md @@ -1,22 +1,52 @@ # platform/velero -**Velero** — off-cluster backup and DR of volumes + Kubernetes resource state. +**Velero** — off-cluster backup and DR of Kubernetes resource state + PVC data. - **Governing ADR:** [ADR-0003 — Longhorn (+ bulk tier + Velero)](../../../home-network-k3s/decisions/adr-0003-storage-longhorn.md) - **Spec:** [docs/04 — Storage & State](../../../home-network-k3s/docs/04-storage-and-state.md) - **Serves:** R-STATE-03 - **Owner:** WS-STORAGE -## What this stub is +## What this is (LIVE) -Flux `HelmRelease` with a placeholder off-cluster backup target (S3-compatible/NFS) and scheduled -backups whose cadence/retention are overlay knobs (`${VELERO_*}`, R-ENV-02). Backup credentials are -referenced **by name** (`velero-backup-credentials`), **SOPS-encrypted** (WS-SEC). Works with Longhorn -volume backups. +Flux `HelmRelease` (Velero chart 8.x / Velero 1.15.x) wired to the on-Proxmox **MinIO** S3 target: -## TODO (later phases) +- **Endpoint:** `http://192.168.1.9:9000` (plain HTTP, **path-style**, region `us-east-1` placeholder) +- **BucketStorageLocation `default`:** provider `aws`, bucket `velero`, `s3ForcePathStyle=true`. +- **AWS plugin** (`velero/velero-plugin-for-aws`) loaded as an init container. +- **File System Backup (kopia):** `deployNodeAgent=true`, `uploaderType=kopia`, `snapshotsEnabled=false` + — storage-agnostic PVC backup, **no** CSI snapshot / data-mover. +- **Credentials:** `velero-cloud-credentials` — a SOPS Secret (`velero-cloud.sops.yaml`) holding the + `cloud` INI key, the **scoped** MinIO key limited to the `velero`+`cnpg` buckets. Kept in sync with + the staging source of truth `platform/backup/minio-credentials.sops.yaml` and the Proxmox host + ansible-vault. -- **OQ-2 (human):** concrete off-cluster target (endpoint, bucket, resilience) is a prerequisite. -- **Phase 4 (WS-STORAGE):** wire the target + Longhorn backups; prove a **DR drill** — restore a - namespace + volumes from off-cluster backup (R-STATE-03, Phase-4 exit gate). -- **DR-critical services** (Gitea, Keycloak, Nextcloud) get off-cluster backup + mirror per docs/05. +Reconciled by its **own non-gating** Flux Kustomization `clusters/base/backup.yaml` +(`dependsOn: platform-config`, `wait:false`) — a backup controller must never gate the platform graph. + +## Backup scope (what's included vs excluded) + +Two daily `Schedule`s (cadence `${VELERO_BACKUP_SCHEDULE}`, retention `${VELERO_RETENTION}`): + +| Schedule | Namespaces | Volumes (kopia FSB) | +|---|---|---| +| `daily` | ALL **except** `monitoring`, `unifi` | **all** PVC data (`defaultVolumesToFsBackup:true`, opt-out) | +| `daily-resources-only` | `monitoring`, `unifi` | **none** (`defaultVolumesToFsBackup:false`) — resources only | + +**Excluded volume data (reconstructable, per the sizing decision):** +- `monitoring` — Prometheus TSDB PVC (ephemeral) + Loki 72h-log PVC. Grafana/Alertmanager are + `emptyDir` (no PVC); Grafana dashboards are GitOps-provisioned. +- `unifi` — the ~7.4 GiB KubeVirt VM disk (huge; UniFi is protected by its own config export). + +Their **Kubernetes resources** are still captured (by `daily-resources-only`) to aid DR. + +**Included (the irreplaceable ~6 GiB):** every other namespace's PVC data via kopia — Nextcloud data, +Gitea, Plex/*arr config, PiHole, and the CNPG Postgres PVCs. (CNPG DBs additionally have native Barman +PITR to the `cnpg` bucket — see the DB manifests.) + +## Post-deploy (PM) + +- `velero backup-location get` → `default` Available. +- `velero backup create --from-schedule daily` → objects land under `s3://velero/`. +- DR drill (R-STATE-03, Phase-4 gate): `velero restore create --from-backup ` into a scratch + namespace and verify PVC data restores via the node-agent. diff --git a/platform/velero/helmrelease.yaml b/platform/velero/helmrelease.yaml index 90e8951..deb6501 100644 --- a/platform/velero/helmrelease.yaml +++ b/platform/velero/helmrelease.yaml @@ -1,5 +1,23 @@ --- -# Velero — off-cluster backup/DR of volumes + resource state (ADR-0003, R-STATE-03). Phase 0 STUB. +# Velero — off-cluster backup/DR of cluster resources + PVC data (ADR-0003, R-STATE-03). LIVE. +# +# TARGET: the on-Proxmox MinIO S3 endpoint http://192.168.1.9:9000 (plain HTTP, path-style), bucket +# `velero`. region us-east-1 is a placeholder MinIO ignores (S3 clients still require one). +# +# WHY THIS SHAPE: +# - provider aws + velero-plugin-for-aws (init container) drives the S3-compatible MinIO target. +# - File System Backup (kopia) via the node-agent DaemonSet — storage-agnostic PVC backup that does +# NOT depend on CSI VolumeSnapshots or a data-mover (the simplest path that still captures the +# Longhorn PVC *data*). snapshotsEnabled:false → no VolumeSnapshotLocation is created/needed. +# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero) — the +# SCOPED MinIO key limited to the velero+cnpg buckets. Kept in sync with the staging source of +# truth platform/backup/minio-credentials.sops.yaml and the Proxmox host ansible-vault. +# +# VOLUME OPT-OUT (per the sizing decision) is implemented with TWO schedules (a single Backup has one +# cluster-wide defaultVolumesToFsBackup, so per-namespace volume policy needs two): see `schedules`. +# +# NON-GATING TIER: reconciled by clusters/base/backup.yaml (dependsOn platform-config, wait:false). +# A backup controller must NEVER gate the platform graph (mirror headlamp/oauth2-proxy). apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -10,7 +28,7 @@ spec: chart: spec: chart: velero - version: ">=8.0.0 <9.0.0" # TODO(WS-STORAGE): pin at bring-up; Renovate maintains + version: ">=8.0.0 <9.0.0" # Velero v1.15.x line. TODO(WS-STORAGE): pin at bring-up; Renovate maintains sourceRef: kind: HelmRepository name: vmware-tanzu @@ -22,25 +40,80 @@ spec: remediation: retries: 3 values: - # Off-cluster backup target — S3-compatible/NFS (ADR-0003). STUB: concrete target + creds are - # OQ-2 / prerequisites. Credentials referenced BY NAME only, SOPS-encrypted (WS-SEC). + # --- AWS object-store plugin (BSL provider=aws → MinIO S3-compatible), loaded as an init container. + # Image minor MUST track the Velero minor (chart 8.x = Velero 1.15 → plugin v1.11.x). --- + initContainers: + - name: velero-plugin-for-aws + image: velero/velero-plugin-for-aws:v1.11.1 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /target + name: plugins + # --- File System Backup (kopia): the node-agent DaemonSet mounts each pod's volumes and copies + # the data into S3. Storage-agnostic; no CSI-snapshot / data-mover wiring. --- + deployNodeAgent: true + snapshotsEnabled: false # no VolumeSnapshotLocation — kopia FSB is the volume-data path configuration: + uploaderType: kopia backupStorageLocation: - name: default - provider: aws # TODO(WS-STORAGE): S3-compatible endpoint per OQ-2 - bucket: REPLACE-ME + provider: aws + bucket: velero + default: true config: - region: REPLACE-ME - s3Url: REPLACE-ME - s3ForcePathStyle: "true" - volumeSnapshotLocation: - - name: default - provider: REPLACE-ME # Longhorn CSI snapshots / Velero fs-backup + region: us-east-1 # placeholder; MinIO ignores it (S3 clients require one) + s3Url: http://192.168.1.9:9000 # on-Proxmox MinIO, plain HTTP + s3ForcePathStyle: "true" # MinIO is path-style credentials: - existingSecret: velero-backup-credentials # SOPS-encrypted (WS-SEC) — name only + existingSecret: velero-cloud-credentials # SOPS Secret carrying the `cloud` INI key (ns velero) + # --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data + # by default (opt-out), but skip the two reconstructable heavy volumes' DATA while still + # capturing their Kubernetes resources. --- schedules: + # (1) The whole cluster EXCEPT monitoring+unifi: all resources + all pod-volume data via kopia. + # defaultVolumesToFsBackup:true is the OPT-OUT model — nothing with real data is forgotten. daily: - schedule: "${VELERO_BACKUP_SCHEDULE}" # overlay knob (cron) + disabled: false + schedule: "${VELERO_BACKUP_SCHEDULE}" + useOwnerReferencesInBackup: false template: - ttl: "${VELERO_RETENTION}" # overlay knob (dev short / prod long) - # TODO(WS-STORAGE, Phase 4): tested restore path (DR drill) is an acceptance gate (R-STATE-03). + ttl: "${VELERO_RETENTION}" + storageLocation: default + snapshotVolumes: false + defaultVolumesToFsBackup: true + excludedNamespaces: + - monitoring # volume DATA excluded here; resources captured by schedule (2) + - unifi + # (2) monitoring + unifi: RESOURCES ONLY (defaultVolumesToFsBackup:false → no volume data). Their + # PVCs are reconstructable and excluded from data backup per the sizing decision: + # - monitoring: the Prometheus TSDB PVC (ephemeral/reconstructable) and the Loki 72h-log PVC + # (Grafana/Alertmanager are emptyDir — no PVC). Grafana dashboards are GitOps-provisioned. + # - unifi: the ~7.4 GiB KubeVirt VM disk (huge + reconstructable; UniFi is protected by its + # own config export, not this disk). + # Backing up their K8s objects (VM/DataVolume/Services/HelmRelease-owned CRs) still aids DR. + daily-resources-only: + disabled: false + schedule: "${VELERO_BACKUP_SCHEDULE}" + useOwnerReferencesInBackup: false + template: + ttl: "${VELERO_RETENTION}" + storageLocation: default + snapshotVolumes: false + defaultVolumesToFsBackup: false + includedNamespaces: + - monitoring + - unifi + # Modest home-dev footprint (mirror the other platform components). + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + nodeAgent: + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi diff --git a/platform/velero/kustomization.yaml b/platform/velero/kustomization.yaml index 5704992..bfd99ae 100644 --- a/platform/velero/kustomization.yaml +++ b/platform/velero/kustomization.yaml @@ -5,3 +5,7 @@ resources: - namespace.yaml - helmrepository.yaml - helmrelease.yaml + # Scoped MinIO S3 credential (the `cloud` INI key the AWS plugin reads) — SOPS-encrypted, decrypted + # per-env by the `velero` Flux Kustomization (clusters/dev). Kept in sync with the staging source of + # truth platform/backup/minio-credentials.sops.yaml + the Proxmox host ansible-vault. + - velero-cloud.sops.yaml diff --git a/platform/velero/values.yaml b/platform/velero/values.yaml index be3c7ba..55bfe67 100644 --- a/platform/velero/values.yaml +++ b/platform/velero/values.yaml @@ -1,7 +1,12 @@ -# Velero Helm values — PLACEHOLDER (Phase 0). Authoritative values inline in helmrelease.yaml. +# Velero Helm values — the AUTHORITATIVE values live inline in helmrelease.yaml (this file is a +# doc pointer, kept for parity with the other components' layout). # -# backupStorageLocation: off-cluster S3-compatible/NFS (ADR-0003; concrete target = OQ-2) -# volumeSnapshotLocation: Longhorn CSI snapshots / Velero fs-backup -# credentials.existingSecret: velero-backup-credentials (SOPS-encrypted, WS-SEC — name only) -# schedules.daily: ${VELERO_BACKUP_SCHEDULE} / ttl ${VELERO_RETENTION} (overlay knobs) -# DR drill (tested restore) is a Phase-4 acceptance gate (R-STATE-03). +# LIVE wiring (see helmrelease.yaml): +# - provider aws + velero-plugin-for-aws (init container) → MinIO S3 http://192.168.1.9:9000 +# (path-style, bucket `velero`, region us-east-1 placeholder). +# - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover). +# - snapshotsEnabled:false → no VolumeSnapshotLocation. +# - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero). +# - two daily Schedules (${VELERO_BACKUP_SCHEDULE} / ttl ${VELERO_RETENTION}): the whole cluster +# via FSB, with monitoring+unifi backed up RESOURCES-ONLY (their volume data is reconstructable). +# DR drill (tested restore) remains a Phase-4 acceptance gate (R-STATE-03) — the PM runs it post-deploy. diff --git a/platform/velero/velero-cloud.sops.yaml b/platform/velero/velero-cloud.sops.yaml new file mode 100644 index 0000000..b3142be --- /dev/null +++ b/platform/velero/velero-cloud.sops.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Secret +metadata: + name: velero-cloud-credentials + namespace: velero +type: Opaque +stringData: + cloud: ENC[AES256_GCM,data:ZnGAkgO3sOXfaL+1QpK6jVixBmR/1iQhhipo+dWcmJBGbxgxFv37L4Y2gmLG0a58G3oghIYWOX++uc/mrH4ZYlLCjARHHBjHCtp4GouXUADwxj2jlVo40bk5GCpsvn3ge8zLdl8dyBF1ZAx66dQ=,iv:FgQOJqJxgchnM7cvl9NeW1o3wf2cos9jl1AR8vnVN9I=,tag:AqFE1a+KAQs5JUlY1FV5wQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjVDBaTTlhS2hqY1JHVDNO + M3pRUEJQdy9iVEpQSUYrSlpaU2RFbGhWQTJVCjZhN2h3Sk1RVzNWOW5iemFYUnRh + d1pHTjNFNm1jWGVIVGhXMERZcXNMOHcKLS0tIGJLaFY4Unc1akJkMTkycU1JT3FS + dzRUSUZjQ2JpcytuaXF0ZWFlQWZCSUkK+ig7ypo/llaP0Xyd0CaCgHMW4Z05pWMC + Xh6jDlRCswRGsLIWLK/nRvQRIArmd4ge1P0WVu+DQAXGL8OtQJ8cZA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-08-29T03:31:48Z" + mac: ENC[AES256_GCM,data:KtqFiWAYimikxjnUl+oFxSR8tyMr5UwRIlAYte4ZtekLCCJYZ1M389HckAdkgp+r+R/481rghhI4MswtokV30WH3t+nkVVyPB92qRGaHFpdwq/248dXj7WUizyRUoW453+/w/EJhnZzBaN8l2XOAVFj8mM3m/fezzyk82odnsPQ=,iv:EWixHOIwDhYq/BxVRjS1te8k02g6gMtwpmXcPnac1PE=,tag:OtedEDOPxE9qG7r0IK/MIg==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.4