merge: R2 off-cluster backups + per-environment backup credentials
This commit is contained in:
jakman85 committed
commit
3ca4ad6011
18 files changed
+574
-67
No files matched your search
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: keycloak-db-backup-credentials-prod
|
||||
namespace: keycloak
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:kQ5x5fXb6KdVaDsItGug2cpQMZw9tSvbsYDqTMxoP+A=,iv:dCTfwVxlpnjaMhGBIMa18d4yAiQ6SKhb4EMm1p1JyQ8=,tag:sNqMLs+rGQm/1MyV90InDQ==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:9lGvwvseAWK1bMXDXZJ+bjw4oHH/XVPi8+C8vfXGJ3jX8VW95rJrczPbEfV9rCudVIJFnxAdaQZZWaJCsC8fCQ==,iv:GOvPs2I77yiKh381bcv0kCwT5xbk8Fao6ctPCW2aCHg=,tag:hSjWjaZ+nEJwZyv3sIuq7A==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBN2NvVXJ5TW1mQlhYWnRC
|
||||
UlVLVjFtNk1Wdkk1NzdicXBQVGNtc3JZclNBCjl3UlVTZmFJWGJGam54VytVR0lZ
|
||||
djB3ZjBCOFFNOUFMZGxCcnZiZ3VDMUUKLS0tIFhDSWR3ZXZPOHFzVGNhZGhqdUQv
|
||||
WmF0NlNIeTJCeVV0ZGkxSFJIL3BqNlEKntEpxCdpy7Xd16ZKe6skQYL6hCqYOAIN
|
||||
NtwtQ9oF/8i4i0T7B8kcORB71cu8dPu6w6LtaSC+1qBV5TBSSH5U6Q==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:dcgB6z9BDzaft4do6f1SqjCqmQopc5mM0ckfU/HfE09eOSIRjpoZ+qMzJW8RIwUZNKrjEAwrrAEtrN67GRxL0jOEsw/6xNrWJRFv6KTeE3ECy5qMO4jLCQTeMvZuvfZE7iqwvHCCnjhkqkedEAD4Hwefq0c0EZ0ZxZK3z1tVdHk=,iv:Ue+ZJq9dJl9qbjJ1l8Q24VmsK+KW/uSRFu04xG6/tM8=,tag:MvJ6Ww3h3b/C2PqA/62abA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -28,9 +28,17 @@ resources:
|
||||
- keycloak-admin-credentials.sops.yaml
|
||||
# --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders ---
|
||||
- keycloak-realm-clients.sops.yaml
|
||||
# --- Scoped MinIO S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05) — SOPS-encrypted,
|
||||
# decrypted by the `platform` tier (which already has the decryption stanza). Lands in ns
|
||||
# keycloak; the keycloak-db Cluster (platform-config) reads it by name for WAL archiving. ---
|
||||
# --- PER-ENVIRONMENT S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05), split
|
||||
# 2026-09-07. SOPS-encrypted, decrypted by the `platform` tier (which already has the
|
||||
# decryption stanza). BOTH land in ns keycloak on BOTH clusters; the keycloak-db Cluster
|
||||
# (platform-config) selects ONE by name via
|
||||
# `keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
|
||||
# prod "-prod"). Two objects, so an R2 token rotation cannot silently break dev's MinIO
|
||||
# WAL archiving — and archive_mode=on makes a broken archive a disk-full clock, not a
|
||||
# degraded backup (see postgres-cluster.yaml).
|
||||
# dev -> keycloak-db-backup-credentials (MinIO, s3://cnpg)
|
||||
# prod -> keycloak-db-backup-credentials-prod (Cloudflare R2, s3://k3s-prod-backup/barman)
|
||||
- keycloak-db-backup-credentials.sops.yaml
|
||||
- keycloak-db-backup-credentials-prod.sops.yaml
|
||||
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
||||
- grafana-dashboard.yaml
|
||||
@@ -34,25 +34,43 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → object storage ---
|
||||
# Continuous WAL archiving + periodic base backups give proper Postgres PITR (superior to a Velero
|
||||
# PVC snapshot for a live DB). destinationPath is the BUCKET ROOT; CNPG appends serverName (defaults
|
||||
# to the Cluster name), so objects land under s3://cnpg/keycloak-db/. Path-style HTTP MinIO endpoint.
|
||||
# PVC snapshot for a live DB). CNPG appends `serverName` (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/keycloak-db/ — dev
|
||||
# s3://cnpg/keycloak-db/ on the on-Proxmox MinIO, prod s3://k3s-prod-backup/barman/keycloak-db/ on
|
||||
# Cloudflare R2. Both endpointURL and destinationPath are cluster-vars knobs (R-ENV-02),
|
||||
# substituted by the `platform-config` Kustomization (this file sits under platform/keycloak/ but
|
||||
# platform-config is the only tier that lists it as a resource — see its kustomization.yaml).
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret keycloak-db-backup-credentials
|
||||
# (a Pod only reads same-ns Secrets), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000) — prod
|
||||
# MUST point at its own off-cluster target. Substituted by the `platform-config` Kustomization.
|
||||
# (a Pod only reads same-ns Secrets).
|
||||
#
|
||||
# *** archive_mode IS ON, SO A FAILING ARCHIVE COMMAND IS A DISK-FULL CLOCK, NOT A DEGRADED
|
||||
# BACKUP. *** Postgres will not recycle a WAL segment it has not successfully archived, so
|
||||
# whenever these two values are wrong the pgdata PVC fills monotonically until the primary
|
||||
# PANICs. This is not hypothetical: prod ran from bootstrap to 2026-09-07 with
|
||||
# `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT` (barman-cloud-wal-archive exit 4 on every segment)
|
||||
# and had burned ~579 MiB of a 5 GiB PVC on the way to taking SSO — and with it kubectl OIDC,
|
||||
# Grafana, Headlamp, Gitea and forward-auth — down with it. Never leave a placeholder here, and
|
||||
# treat `ContinuousArchiving=False` on this Cluster as an availability incident, not a backup one.
|
||||
backup:
|
||||
retentionPolicy: "30d" # keep 30 days of base backups + WAL (PITR window); prunes older
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
# PER-ENVIRONMENT credential (2026-09-07): dev reads keycloak-db-backup-credentials (MinIO),
|
||||
# prod reads keycloak-db-backup-credentials-prod (Cloudflare R2). Both Secrets are applied to
|
||||
# both clusters by platform/keycloak/kustomization.yaml; ${BACKUP_CREDENTIALS_SUFFIX} (dev ""
|
||||
# | prod "-prod") picks one. Before the split these were ONE shared Secret, so writing prod's
|
||||
# R2 token into it would have left dev signing MinIO requests with an R2 key — a 403 on every
|
||||
# archive command, which under archive_mode=on is the disk-full clock described above, not a
|
||||
# missing backup.
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: keycloak-db-backup-credentials
|
||||
name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: keycloak-db-backup-credentials
|
||||
name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
@@ -1,18 +1,29 @@
|
||||
---
|
||||
# Velero — off-cluster backup/DR of cluster resources + PVC data (ADR-0003, R-STATE-03). LIVE.
|
||||
#
|
||||
# TARGET: the MinIO S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars; dev http://192.168.1.9:9000 on
|
||||
# Proxmox — plain HTTP, path-style), bucket
|
||||
# `velero`. region us-east-1 is a placeholder MinIO ignores (S3 clients still require one).
|
||||
# TARGET: the S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars) at
|
||||
# s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX}, in region ${S3_BACKUP_REGION}. Four knobs,
|
||||
# because the two environments' object stores differ in shape as well as address:
|
||||
# dev http://192.168.1.9:9000 bucket `velero`, NO prefix, region us-east-1 (on-Proxmox MinIO,
|
||||
# plain HTTP, path-style; MinIO ignores the region)
|
||||
# prod https://<acct>.r2.cloudflarestorage.com bucket `k3s-prod-backup`, prefix `velero`,
|
||||
# region `auto` (Cloudflare R2; prod cannot route to that MinIO,
|
||||
# and R2 demands exactly `auto`)
|
||||
# Prod's bucket is SHARED with CNPG/barman (the operator created one bucket), so the `velero` prefix
|
||||
# is what keeps this controller's retention sweep away from the Postgres WAL stream. See the
|
||||
# "OBJECT-STORE TENANCY" block in clusters/base/cluster-vars.yaml.
|
||||
#
|
||||
# WHY THIS SHAPE:
|
||||
# - provider aws + velero-plugin-for-aws (init container) drives the S3-compatible MinIO target.
|
||||
# - File System Backup (kopia) via the node-agent DaemonSet — storage-agnostic PVC backup that does
|
||||
# NOT depend on CSI VolumeSnapshots or a data-mover (the simplest path that still captures the
|
||||
# Longhorn PVC *data*). snapshotsEnabled:false → no VolumeSnapshotLocation is created/needed.
|
||||
# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero) — the
|
||||
# SCOPED MinIO key limited to the velero+cnpg buckets. Kept in sync with the staging source of
|
||||
# truth platform/backup/minio-credentials.sops.yaml and the Proxmox host ansible-vault.
|
||||
# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero). ONE
|
||||
# Secret serves both environments, and it now carries the Cloudflare R2 API token (written by
|
||||
# platform/backup/set-r2-credentials.sh, 2026-09-07) rather than the scoped MinIO key. That is
|
||||
# safe only because the R2 token is bucket-scoped to k3s-prod-backup, which dev never names:
|
||||
# dev's Velero would fail loudly against its MinIO endpoint with an R2 key, it could not
|
||||
# silently write somewhere unexpected. Rotating either target means re-running that script.
|
||||
#
|
||||
# VOLUME OPT-OUT (per the sizing decision) is implemented with TWO schedules (a single Backup has one
|
||||
# cluster-wide defaultVolumesToFsBackup, so per-namespace volume policy needs two): see `schedules`.
|
||||
@@ -72,14 +83,45 @@ spec:
|
||||
backupStorageLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
bucket: velero
|
||||
# Bucket and prefix are BOTH knobs (cluster-vars), because the two environments' object
|
||||
# stores are shaped differently. Dev's MinIO has a dedicated `velero` bucket and dev's
|
||||
# backups sit at its ROOT, so dev renders bucket=velero with an EMPTY prefix — and the
|
||||
# chart omits `objectStorage.prefix` entirely when it is empty (verified, chart 12.1.0),
|
||||
# leaving dev's live BackupStorageLocation byte-unchanged. Prod's operator created ONE R2
|
||||
# bucket shared with CNPG/barman, so prod renders bucket=k3s-prod-backup with prefix
|
||||
# `velero` — a disjoint top-level prefix from barman's. Getting this wrong is not a
|
||||
# cosmetic error: Velero ADOPTS every backup it finds under its location and TTL-DELETES
|
||||
# it on retention, so a prefix that overlaps another tenant deletes that tenant's data.
|
||||
bucket: ${VELERO_BACKUP_BUCKET}
|
||||
prefix: "${VELERO_BACKUP_PREFIX}"
|
||||
default: true
|
||||
config:
|
||||
region: us-east-1 # placeholder; MinIO ignores it (S3 clients require one)
|
||||
s3Url: ${S3_BACKUP_ENDPOINT} # on-Proxmox MinIO, plain HTTP (cluster-vars)
|
||||
s3ForcePathStyle: "true" # MinIO is path-style
|
||||
# Region is a knob, not a constant: S3 clients require SOME region, MinIO ignores
|
||||
# whatever it is, and Cloudflare R2 requires exactly `auto`. Hardcoding `us-east-1`
|
||||
# was correct-by-accident and broke the moment the target stopped being MinIO.
|
||||
region: ${S3_BACKUP_REGION}
|
||||
s3Url: ${S3_BACKUP_ENDPOINT} # cluster-vars; dev MinIO (http) or prod R2 (https)
|
||||
s3ForcePathStyle: "true" # MinIO is path-style; R2 accepts path-style too
|
||||
# *** REQUIRED FOR CLOUDFLARE R2 — DO NOT DROP THIS LINE. ***
|
||||
# velero-plugin-for-aws v1.14.2 bundles an AWS SDK with flexible checksums ON by
|
||||
# default, so it sends `x-amz-sdk-checksum-algorithm` / `x-amz-checksum-crc32` on every
|
||||
# PUT. R2 REJECTS those headers with HTTP 400 InvalidArgument, which would fail every
|
||||
# backup AND every kopia blob upload. The empty string is the documented opt-out, and
|
||||
# it must be PRESENT-but-empty rather than absent — the plugin branches on the key
|
||||
# existing, and the chart does emit an empty value (verified, chart 12.1.0). Harmless
|
||||
# against MinIO, which ignores it, so it is stated unconditionally rather than as a
|
||||
# fourth knob. If backups start failing with InvalidArgument after a plugin bump, look
|
||||
# here first.
|
||||
checksumAlgorithm: ""
|
||||
credentials:
|
||||
existingSecret: velero-cloud-credentials # SOPS Secret carrying the `cloud` INI key (ns velero)
|
||||
# PER-ENVIRONMENT credential (2026-09-07). dev renders velero-cloud-credentials (the scoped
|
||||
# MinIO key); prod renders velero-cloud-credentials-prod (the Cloudflare R2 token). BOTH
|
||||
# Secrets are applied to BOTH clusters by platform/velero/kustomization.yaml — the suffix
|
||||
# only chooses which one this Deployment mounts. cluster-vars: BACKUP_CREDENTIALS_SUFFIX,
|
||||
# dev "" | prod "-prod". A name that resolves to a MISSING Secret does not degrade the
|
||||
# backup, it wedges the pod in ContainerCreating (the volume never mounts), which is why
|
||||
# the base default is the empty suffix — the unsuffixed Secret always exists.
|
||||
existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX} # SOPS Secret carrying the `cloud` INI key (ns velero)
|
||||
# --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data
|
||||
# by default (opt-out), but skip the two reconstructable heavy volumes' DATA while still
|
||||
# capturing their Kubernetes resources. ---
|
||||
|
||||
@@ -5,7 +5,19 @@ resources:
|
||||
- namespace.yaml
|
||||
- helmrepository.yaml
|
||||
- helmrelease.yaml
|
||||
# Scoped MinIO S3 credential (the `cloud` INI key the AWS plugin reads) — SOPS-encrypted, decrypted
|
||||
# per-env by the `velero` Flux Kustomization (clusters/dev). Kept in sync with the staging source of
|
||||
# truth platform/backup/minio-credentials.sops.yaml + the Proxmox host ansible-vault.
|
||||
# --- PER-ENVIRONMENT `cloud` INI credential (split 2026-09-07) -----------------------------------
|
||||
# BOTH Secrets are applied to BOTH clusters; the HelmRelease picks ONE by name via
|
||||
# `existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
|
||||
# prod "-prod"). They are separate objects so rotating one environment's key can never touch the
|
||||
# other's — the failure mode that made the split necessary (a single shared Secret carrying R2 keys
|
||||
# would have pointed dev's Velero at MinIO with credentials MinIO has never issued).
|
||||
#
|
||||
# dev -> velero-cloud-credentials : the scoped MinIO key for http://192.168.1.9:9000.
|
||||
# Kept in sync with the staging source of truth
|
||||
# platform/backup/minio-credentials.sops.yaml + the
|
||||
# Proxmox host ansible-vault.
|
||||
# prod -> velero-cloud-credentials-prod : the Cloudflare R2 token for k3s-prod-backup.
|
||||
#
|
||||
# Both are decrypted by the `velero` Flux Kustomization's SOPS stanza (clusters/dev + clusters/prod).
|
||||
- velero-cloud.sops.yaml
|
||||
- velero-cloud-prod.sops.yaml
|
||||
@@ -2,8 +2,13 @@
|
||||
# doc pointer, kept for parity with the other components' layout).
|
||||
#
|
||||
# LIVE wiring (see helmrelease.yaml):
|
||||
# - provider aws + velero-plugin-for-aws (init container) → MinIO S3 ${S3_BACKUP_ENDPOINT} (dev: http://192.168.1.9:9000)
|
||||
# (path-style, bucket `velero`, region us-east-1 placeholder).
|
||||
# - provider aws + velero-plugin-for-aws (init container) → S3 ${S3_BACKUP_ENDPOINT}, path-style,
|
||||
# at s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX} in region ${S3_BACKUP_REGION}. Four
|
||||
# cluster-vars knobs: dev = the on-Proxmox MinIO (http://192.168.1.9:9000, bucket `velero`, no
|
||||
# prefix, us-east-1); prod = Cloudflare R2 (bucket `k3s-prod-backup`, prefix `velero`, `auto`),
|
||||
# sharing one bucket with CNPG/barman under a disjoint prefix.
|
||||
# - config.checksumAlgorithm: "" — MANDATORY for R2, which rejects the AWS SDK's
|
||||
# x-amz-checksum-* headers with HTTP 400 InvalidArgument. Harmless against MinIO.
|
||||
# - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover).
|
||||
# - snapshotsEnabled:false → no VolumeSnapshotLocation.
|
||||
# - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero).
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: velero-cloud-credentials-prod
|
||||
namespace: velero
|
||||
type: Opaque
|
||||
stringData:
|
||||
cloud: ENC[AES256_GCM,data:Kq+hfYMq/NZJJqbK2zqwW97XKNW0LbT2Un20yc0B6evPXqp+aC0vV0odwos4XwV0puPIsEhU7IeD+JidVjU1eKv6Qax0rnoj90FVWihxWyLMNAMC+b8PxBnPhWdk6ZlrMKs13roGZwYh4bbrstnJamMqpD4oZpthEl/NEQNWVAb5ogJHsmPNiQOtHhtBWkTubfgPIw==,iv:gn8/4/XpUZ8c6+nlGe6SAODUyrhDqskQ+uVRx2YR3Qc=,tag:aTk3cRJTA0WjdIi1B9S0hg==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwWmV5UGpmdnFKbEhjeHlZ
|
||||
QUZCWm13ZlpVYU1URFErSmM5MEJ2WTRxZVFvCkZ5cEdJaXJ1NDNSaEF5d2lxdDQw
|
||||
NGpzQm5ESGt6cUlIZVdHYTBjRGVwaVUKLS0tIHlpWkVacXdIZndBOXdlVk1LZzlN
|
||||
NkZyRm1abk9CNlZRUERtbFM2RjVtS1UKWYtI7QZiTjPIHaLOOX9WvDK1cHwP2WfW
|
||||
26WdNW90tTmt3cYdaFGnkpAVCMS3BRtdInirulfoGRgvffjWNBDVuQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:4v4L29KoIhQI94TU/s0u3HKR5C6mMc6mRWWsZzwIfqkl4fayRvLG67tbJ88vVkb6AXgqHoKXKlhbpn6hTJPBrL72f8+fii3CeLloVJyW2AuzHaHCSi92U04ywoSc45hClZrOLvUFf5KbtawPPKfP53oaNq+AVEZ1nqKd8ssUyBA=,iv:WVrdja00ptK/MmCC/Chu+Gq4ZYpBvpFRWW3lSo7Mon0=,tag:GzFfpkBlHu6Nc8ZdnCBuKQ==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
Reference in new issue
Block a user