Compare commits
6
Commits
dc2d1ce17e
...
3ca4ad6011
| Author | SHA1 | Date | |
|---|---|---|---|
 jakman85
|
3ca4ad6011 | ||
 jakman85andClaude Opus 5
|
708a05dfc1 | ||
|
jakman85andClaude Opus 5
|
161430d915 | ||
|
jakman85andClaude Opus 5
|
e7c4a95c2d | ||
|
jakman85andClaude Opus 5
|
0a0527e7db | ||
|
jakman85andClaude Opus 5
|
9739e9be91 |
No files matched your search
@@ -11,6 +11,7 @@ resources:
|
||||
- jackett-oauth2-proxy-credentials.sops.yaml
|
||||
- jackett.yaml
|
||||
- nextcloud-db-app.sops.yaml
|
||||
- nextcloud-db-backup-credentials-prod.sops.yaml
|
||||
- nextcloud-db-backup-credentials.sops.yaml
|
||||
- nextcloud-db.yaml
|
||||
- pihole-oauth2-proxy-credentials.sops.yaml
|
||||
@@ -19,5 +20,6 @@ resources:
|
||||
- qbittorrent-oauth2-proxy-credentials.sops.yaml
|
||||
- qbittorrent.yaml
|
||||
- semaphore-db-app.sops.yaml
|
||||
- semaphore-db-backup-credentials-prod.sops.yaml
|
||||
- semaphore-db-backup-credentials.sops.yaml
|
||||
- semaphore-db.yaml
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: nextcloud-db-backup-credentials-prod
|
||||
namespace: nextcloud-db
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:4lx6et4HcXDegulyIoa7jZR/lVgapGPEkukajfCKjvk=,iv:p6V7tizbikk55tOvpPOsPHTe/p52QLX5A5xrwuB14rA=,tag:oNJuipT/Lb6zynJ2K2Q71A==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:V5bHPoPeBOVyLp1n1Jilyso4i5P2aPSH0hrryfweg2q/SctWZxRbLBQ/FLzOigbJsAhlwOGu069mQPma6xjfdw==,iv:8u4UOvI8dC/cm6mEWDlyN5E3SfRZmwKY9uZY/DjPEsQ=,tag:tw7tGs29TmUvMPBCtCszAA==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiSUt5N2k2N2lDU2ZsMy9m
|
||||
M1NFS0M2QjFsNFhnR0hCMzFyaTdiQWFZRFJZCmhBTWRXNmxLMGtsc3Rxd2FSMDZp
|
||||
aktjcTE2c1hmU1EzM3RWNksyb3Q1YVUKLS0tIDBFS1I5aDBER2cvcWtLZ3BXTEp5
|
||||
ZGI5ZmZIcStheUNQamxIVVVpc0ZMOHMKlE5aHewtHJpeU24F5sodc9CEm9KTjHQQ
|
||||
8ThEQ/8vyazL9V4ojlICxSHU/SHkHpD1eBaTKcAQd/LZzujIcJ+QfA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:8awz1Jh8t4zjDMyd93/v+6on9iiIaSi01293A7uYhxj9L1+g63zeO+DPmSn6+KbZvHdqU2A7up09H1NzfXAEdGuvVbB36cdGq0zKank3PUi8c7IT6qNuWydxRAcqLnbbD+c30Z6GB+gAuZHstNOe4Rgc4aJvMN6FuP6x6Z0870c=,iv:KGGKJq2QA5xztvoHs/SViXW72DYwT16i+W+dJBUeIJQ=,tag:3UpYqx2+RIK0QdMH/BSIpA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -68,25 +68,38 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
|
||||
# Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
|
||||
# backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
|
||||
# (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP.
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials
|
||||
# (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
|
||||
# substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
|
||||
# backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/nextcloud-db/ — dev
|
||||
# s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod
|
||||
# s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and
|
||||
# destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
|
||||
# Credentials: PER-ENVIRONMENT (2026-09-07). dev reads nextcloud-db-backup-credentials (the
|
||||
# scoped MinIO key), prod reads nextcloud-db-backup-credentials-prod (the Cloudflare R2
|
||||
# token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
|
||||
# generated golden — render.py places the -prod sidecar in this DB's wave); the
|
||||
# cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
|
||||
# one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
|
||||
# ONE shared Secret until the split, so giving prod its R2 token would have left dev
|
||||
# signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
|
||||
# disk-full clock below rather than a merely missing backup.
|
||||
#
|
||||
# archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
|
||||
# Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
|
||||
# the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
|
||||
# endpoint. Treat ContinuousArchiving=False here as an availability incident.
|
||||
backup:
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: nextcloud-db-backup-credentials
|
||||
name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: nextcloud-db-backup-credentials
|
||||
name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: semaphore-db-backup-credentials-prod
|
||||
namespace: semaphore-db
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:OhWEEYqrvmDB40NmiETgeUisBvaO9ocevCZAsurNijw=,iv:kEjOIg9NHbRYm+Ul1WrBAzb3gLbIv+ipiYCxFK0QoVc=,tag:KAB1JAIw0ATwbjDEKZVT9A==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:C0PNKv07LCYpFONLqd55sz+ioyU/i2YJUkbZ8OX/dCErM/DilqchyUAT5EqnT/r3YJmdaA1pX85S3fxlVVHkbg==,iv:4ha28b+xIebvX2JPITmtTovIe4vdmj2XfI3cgkUHH4c=,tag:xbMyflcHojytPzgaoOYcXQ==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZVTJ3cXR6cks3Y0RNS2hx
|
||||
K3pjZEduYUxmL0V3SmFUcU9XbGVHSVFITWh3Cjl1eHlTLytoOW5PaVNFVkJoeTBo
|
||||
T1FnRHNDVzJ6dWpuR09WaElvbWF4YU0KLS0tIEJoQ1VsM1cxRlg0R1dtNGdOenNn
|
||||
aVNjMjVBTytLdHlFdmZCS2tPelo1Y2MKYt1G5t+sDiHgcg+Q9ECBzJn2gS/paSEC
|
||||
7aoX60SBrPgHvnGls5lzc3Tin9nH4jWYl6Nu8DAvIE0YPXDFr5a0UA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:mhVP7tjdZK2FR2imvpJbTyT7BigvcIKEpLSnBS1oSITUirB8xZC7aHBfyFuIoKOrpOJgEe40nHUdDYCHRJtb4dLeVja8+7meVaQWzxC8BWVHsksW+73hsqhjAzlHeobx6Irbk4PAMqIY0/jnf4MHryjnRn3dUuwu9U1CqL30JbE=,iv:57oCcp6FW9EAIVKnYGtkaTrnbijyv54T+vmZxMbMqnw=,tag:agrRP/JLypU60KOeN0s/5w==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -69,25 +69,38 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
|
||||
# Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
|
||||
# backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
|
||||
# (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP.
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials
|
||||
# (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
|
||||
# substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
|
||||
# backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/semaphore-db/ — dev
|
||||
# s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod
|
||||
# s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and
|
||||
# destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
|
||||
# Credentials: PER-ENVIRONMENT (2026-09-07). dev reads semaphore-db-backup-credentials (the
|
||||
# scoped MinIO key), prod reads semaphore-db-backup-credentials-prod (the Cloudflare R2
|
||||
# token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
|
||||
# generated golden — render.py places the -prod sidecar in this DB's wave); the
|
||||
# cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
|
||||
# one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
|
||||
# ONE shared Secret until the split, so giving prod its R2 token would have left dev
|
||||
# signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
|
||||
# disk-full clock below rather than a merely missing backup.
|
||||
#
|
||||
# archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
|
||||
# Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
|
||||
# the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
|
||||
# endpoint. Treat ContinuousArchiving=False here as an availability incident.
|
||||
backup:
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: semaphore-db-backup-credentials
|
||||
name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: semaphore-db-backup-credentials
|
||||
name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
+13
-2
@@ -1321,6 +1321,11 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
|
||||
- `<svc>-db.yaml` -> wave(svc-db) (a strictly-lower wave; CNPG DB)
|
||||
- `<svc>-db-app.sops.yaml` -> wave(svc-db) (CNPG initdb owner secret)
|
||||
- `<svc>-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret)
|
||||
- `<svc>-db-backup-credentials-prod.sops.yaml` -> wave(svc-db) (the PROD half of that
|
||||
secret: the two environments carry DIFFERENT object-store credentials, selected in
|
||||
the Cluster CR by `${BACKUP_CREDENTIALS_SUFFIX}`. Both objects ship to both clusters,
|
||||
so both must be placed in the DB wave — the Secret has to exist before the Cluster
|
||||
that names it.)
|
||||
Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
|
||||
Only files that actually exist in `present_files` are placed, so the plan is robust to a
|
||||
service having no bootstrap secret or no DB. Deterministic (sorted).
|
||||
@@ -1348,6 +1353,9 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
|
||||
add(wdb, f"{db}.yaml")
|
||||
add(wdb, f"{db}-app.sops.yaml")
|
||||
add(wdb, f"{db}-backup-credentials.sops.yaml") # CNPG Barman S3 creds (R-STATE-03)
|
||||
# PROD half of the same credential (2026-09-07 per-env split). Placed only when
|
||||
# the file exists, so a DB that has not been split is unaffected.
|
||||
add(wdb, f"{db}-backup-credentials-prod.sops.yaml")
|
||||
|
||||
for unit, cfg in EXTRA_UNITS.items():
|
||||
anchor = cfg["anchor"]
|
||||
@@ -1830,6 +1838,7 @@ def _selftest() -> int:
|
||||
present = {
|
||||
"a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
|
||||
"svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
|
||||
"svc-db-backup-credentials-prod.sops.yaml",
|
||||
"gitea.yaml",
|
||||
"gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
|
||||
}
|
||||
@@ -1839,8 +1848,10 @@ def _selftest() -> int:
|
||||
flat = {f: wv for wv, fs in plan.items() for f in fs}
|
||||
check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
|
||||
check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
|
||||
and flat.get("svc-db-backup-credentials.sops.yaml") == 0,
|
||||
"CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave")
|
||||
and flat.get("svc-db-backup-credentials.sops.yaml") == 0
|
||||
and flat.get("svc-db-backup-credentials-prod.sops.yaml") == 0,
|
||||
"CNPG DB manifest + its owner secret + BOTH halves of its per-env Barman backup secret "
|
||||
"must be in the DB's (lower) wave")
|
||||
check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
|
||||
"a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
|
||||
check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")
|
||||
|
||||
@@ -34,6 +34,9 @@ spec:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per
|
||||
# environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how
|
||||
# headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml.
|
||||
# SOPS decryption for the `cloud` INI credentials — velero-cloud.sops.yaml (dev's scoped MinIO key)
|
||||
# and velero-cloud-prod.sops.yaml (prod's Cloudflare R2 token) — is layered per environment by the
|
||||
# overlay (clusters/dev and clusters/prod patch THIS Kustomization by name), identical to how
|
||||
# headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. Both Secrets are applied on both
|
||||
# clusters; ${BACKUP_CREDENTIALS_SUFFIX} (substituted from cluster-vars above) decides which one the
|
||||
# velero Deployment actually mounts. See clusters/dev/kustomization.yaml.
|
||||
@@ -96,9 +96,22 @@ data:
|
||||
#
|
||||
# Substitution coverage (verify before adding a knob here): every path that consumes these declares
|
||||
# postBuild.substituteFrom cluster-vars — platform (K3S_API_HOST), platform-config
|
||||
# (CILIUM_LB_POOL_*, S3_BACKUP_ENDPOINT), crafty (CRAFTY_L4_VIP), apps-wave-0 (S3_BACKUP_ENDPOINT),
|
||||
# velero (S3_BACKUP_ENDPOINT), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an UNSUBSTITUTED path would
|
||||
# ship the literal string into the cluster.
|
||||
# (CILIUM_LB_POOL_*, and keycloak-db's S3_BACKUP_ENDPOINT + CNPG_BACKUP_PATH: the Cluster lives at
|
||||
# platform/keycloak/postgres-cluster.yaml but is rendered by platform-config, which is the ONLY
|
||||
# tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db:
|
||||
# S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH), velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION,
|
||||
# VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an
|
||||
# UNSUBSTITUTED path would ship the literal string into the cluster.
|
||||
#
|
||||
# ADDING A KNOB IS SAFE UNDER StrictPostBuildSubstitutions ONLY BECAUSE OF THE ROOT EDGE. An
|
||||
# UNDEFINED variable is a HARD Kustomization failure, so a manifest that references a knob the
|
||||
# live ConfigMap does not yet carry breaks its whole tier. `clusters/base/platform.yaml` gained
|
||||
# `dependsOn: [flux-system]` for exactly this: dependsOn is REVISION-gated, so no substituting
|
||||
# tier can reconcile revision N until the ROOT Kustomization has APPLIED revision N — i.e. until
|
||||
# the new cluster-vars key EXISTS. Every substituting consumer chains transitively through
|
||||
# `platform`, so one edge covers the graph, and a new knob plus its first reference may ship in
|
||||
# ONE commit. (A hand-run `kubectl apply -k clusters/<env>` bypasses that ordering entirely: apply
|
||||
# the overlay BEFORE the tier that references the knob, or the tier fails until it retries.)
|
||||
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
|
||||
#
|
||||
@@ -150,9 +163,21 @@ data:
|
||||
# Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
|
||||
# Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
|
||||
# nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port.
|
||||
# Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR.
|
||||
# Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR —
|
||||
# and, measured 2026-09-05 from a pod INSIDE prod, it CANNOT reach it anyway (prod is on the SOHO
|
||||
# network 192.168.2.0/24, which is pfSense's WAN, while this MinIO sits inside pfSense's LAN:
|
||||
# `curl http://192.168.1.9:9000/minio/health/live` timed out after 15s). Prod uses Cloudflare R2.
|
||||
S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"
|
||||
|
||||
# S3 region. Every S3 client REQUIRES one; MinIO ignores whatever it is; Cloudflare R2 demands
|
||||
# exactly `auto` (its `head-bucket` reports the physical hint `WNAM`, which is NOT a usable region
|
||||
# string). This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-
|
||||
# accident for MinIO and wrong for anything else, which is why it is a knob now. The base default
|
||||
# is dev's LIVE value, deliberately: dev's Velero has been signing requests with `us-east-1`
|
||||
# against that MinIO since bootstrap, and this repo's rule is that base == the known-good dev
|
||||
# value. Prod overrides to `auto`.
|
||||
S3_BACKUP_REGION: "us-east-1"
|
||||
|
||||
# --- storage (ADR-0003) ---
|
||||
LONGHORN_DEFAULT_REPLICA_COUNT: "3" # default StorageClass replica count (3 on 3 nodes)
|
||||
# Longhorn's defaultDataPath (platform/longhorn/helmrelease.yaml): the on-NODE directory every
|
||||
@@ -205,6 +230,99 @@ data:
|
||||
VELERO_BACKUP_SCHEDULE: "0 3 * * *" # cron; overlays may differ
|
||||
VELERO_RETENTION: "168h" # dev short | prod long
|
||||
|
||||
# ==========================================================================================
|
||||
# OBJECT-STORE TENANCY — the three knobs that keep two clusters and two backup tools from
|
||||
# writing into each other's data. THESE ARE SAFETY KNOBS, not preferences.
|
||||
# ==========================================================================================
|
||||
# Nothing about the object layout was per-environment before this: Velero's bucket was the
|
||||
# hardcoded literal `velero` with no prefix, and every CNPG Cluster used the hardcoded
|
||||
# `s3://cnpg` with barman's `serverName` defaulting to the CLUSTER NAME — and BOTH environments
|
||||
# have a keycloak-db, a nextcloud-db and a semaphore-db. Point two clusters at one bucket under
|
||||
# that layout and:
|
||||
# (a) each cluster's Velero ADOPTS the other's backups and TTL-DELETES them on retention; and
|
||||
# (b) two different Postgres servers stream WAL into ONE Barman prefix, which does not merely
|
||||
# mix files — it CORRUPTS THE CHAIN and destroys PITR for both. That is the worse half:
|
||||
# dev's DR restore is PROVEN (R-STATE-03) and this would silently un-prove it.
|
||||
#
|
||||
# WHY THERE IS NO SINGLE `S3_BACKUP_BUCKET`. The two consumers take differently-SHAPED values:
|
||||
# the Velero chart wants `bucket` and `prefix` as separate fields, while CNPG's
|
||||
# barmanObjectStore.destinationPath is ONE full s3:// URL whose first path segment IS the bucket.
|
||||
# CNPG's bucket therefore already lives inside CNPG_BACKUP_PATH; adding a second knob that had to
|
||||
# hold the same bucket name a second time would be a rotation waiting to drift — the exact class
|
||||
# of duplication this repo has been bitten by. Three knobs, each shaped for its own consumer, and
|
||||
# no two of them can contradict each other.
|
||||
#
|
||||
# BASE = DEV'S LIVE LAYOUT, BYTE-FOR-BYTE, AND MUST STAY THAT WAY. Dev's existing backups are at
|
||||
# `s3://velero/` (bucket root, no prefix) and `s3://cnpg/<clusterName>/` on that MinIO. Giving dev
|
||||
# a prefix, or renaming its buckets, would ORPHAN the very backups its proven restore depends on —
|
||||
# the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default` and its LONGHORN_DATA_PATH stays
|
||||
# /var/lib/longhorn. Verified: with an EMPTY prefix the Velero chart omits `objectStorage.prefix`
|
||||
# from the rendered BackupStorageLocation entirely (helm template, chart 12.1.0), so dev's live
|
||||
# BSL is unchanged by this parameterization rather than merely equivalent to it.
|
||||
#
|
||||
# PROD, which has no history to preserve, gets ONE bucket with TWO disjoint top-level prefixes:
|
||||
#
|
||||
# s3://k3s-prod-backup/velero/… Velero (kopia repo + backup metadata)
|
||||
# s3://k3s-prod-backup/barman/<clusterName>/… CNPG WAL + base backups
|
||||
#
|
||||
# The operator created exactly one R2 bucket, so velero-vs-barman separation has to come from the
|
||||
# prefix rather than the bucket. The BUCKET ROOT IS DELIBERATELY LEFT EMPTY: if a prefix ever
|
||||
# renders wrong, its objects land somewhere obviously bogus instead of silently colliding with a
|
||||
# real tenant's data. Dev↔prod isolation is doubly independent — different endpoint AND different
|
||||
# bucket — so neither cluster can reach the other's objects even with a mis-substituted prefix.
|
||||
VELERO_BACKUP_BUCKET: "velero" # dev: the `velero` MinIO bucket | prod: k3s-prod-backup
|
||||
VELERO_BACKUP_PREFIX: "" # dev: bucket root (live layout) | prod: "velero"
|
||||
CNPG_BACKUP_PATH: "s3://cnpg" # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# WHICH CREDENTIAL SECRET THE BACKUP CONSUMERS READ (added 2026-09-07 — the per-env credential split)
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# The three knobs above say WHERE the backups go. This one says WHICH KEY signs the requests, and
|
||||
# it exists because the endpoint moved per-environment while the credential did not.
|
||||
#
|
||||
# THE BUG IT FIXES. The four backup-credential Secrets — velero-cloud.sops.yaml and the three
|
||||
# <db>-backup-credentials.sops.yaml — are SHARED BASE manifests: platform/ and apps/ are rendered
|
||||
# by the same Flux tiers on both clusters, and cluster-vars substitution is the ONLY thing that
|
||||
# differs (R-ENV-02). So writing prod's R2 token into those four files did not GIVE prod a
|
||||
# credential, it REPLACED DEV'S: dev's Velero and all three dev CNPG Clusters would have started
|
||||
# signing MinIO requests with an R2 key id. For Velero that is a broken BSL; for CNPG it is worse,
|
||||
# because `archive_mode=on` means Postgres REFUSES TO RECYCLE a WAL segment it could not archive,
|
||||
# so a 403 loop is a monotonically filling pgdata PVC ending in a PANIC — the same failure prod
|
||||
# spent its whole bring-up in with a placeholder endpoint. One shared Secret also means rotating
|
||||
# either environment's key silently breaks the other; that is the structural reason for the split,
|
||||
# independent of dev's eventual decommission.
|
||||
#
|
||||
# THE MECHANISM. Each credential is now TWO Secret objects, both applied to BOTH clusters (they are
|
||||
# base resources; nothing here forks a manifest per environment). The consumer names one:
|
||||
#
|
||||
# velero: credentials.existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
# CNPG: s3Credentials.*.name: <db>-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
#
|
||||
# ONE knob, not four, ON PURPOSE. Four independent secret-name knobs could be updated three-
|
||||
# quarters of the way, leaving one consumer pointed at the OTHER environment's key with nothing
|
||||
# rendering wrong — and StrictPostBuildSubstitutions cannot catch that, because it only fails on an
|
||||
# UNDEFINED variable, never on a defined-but-stale one. With a single suffix the four move together
|
||||
# or not at all. The cost is that the rendered Secret name is not greppable from the consumer; the
|
||||
# names are spelled out in full in each kustomization.yaml beside the resources.
|
||||
#
|
||||
# THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING. It renders the UNSUFFIXED names,
|
||||
# which are the objects that already exist and that the live dev cluster is running. A default
|
||||
# naming a Secret that does not exist would be worse than the shared-Secret state it replaces:
|
||||
# Velero's pod would sit in ContainerCreating forever on a missing `existingSecret` volume, and a
|
||||
# CNPG Cluster whose s3Credentials Secret is absent stops archiving WAL — i.e. the disk-full clock
|
||||
# again — so an unresolvable default breaks the referencing cluster HARDER than a wrong key does.
|
||||
# Empty is also the value dev must carry, so base and dev agree, exactly like VELERO_BACKUP_PREFIX.
|
||||
#
|
||||
# An EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions: strict mode fails on a variable that is
|
||||
# not DEFINED, not on one defined as "". VELERO_BACKUP_PREFIX above has shipped empty since the
|
||||
# gate was enabled and proves it. Both overlays state this knob explicitly — dev "" and prod
|
||||
# "-prod" — so neither inherits it silently.
|
||||
#
|
||||
# ADDING A NEW BACKUP CREDENTIAL: create BOTH halves (`x-credentials.sops.yaml` and
|
||||
# `x-credentials-prod.sops.yaml`), list both in the kustomization, and reference
|
||||
# `x-credentials${BACKUP_CREDENTIALS_SUFFIX}`. Never a single shared Secret again.
|
||||
BACKUP_CREDENTIALS_SUFFIX: "" # dev: "" (MinIO creds) | prod: "-prod" (R2 creds)
|
||||
|
||||
# --- placement ---
|
||||
PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os" # base no-op selector; prod may pin roles
|
||||
PLATFORM_NODE_SELECTOR_VALUE: "linux"
|
||||
@@ -106,8 +106,11 @@ patches:
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml).
|
||||
# It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS
|
||||
# decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only).
|
||||
# It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped
|
||||
# MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption
|
||||
# stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by
|
||||
# BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is
|
||||
# out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
@@ -224,9 +227,56 @@ patches:
|
||||
value: "192.168.1.55"
|
||||
# Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
|
||||
# barmanObjectStore and Velero's backupStorageLocation write to.
|
||||
#
|
||||
# DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07).
|
||||
# Prod could not use it at all — it cannot even route to it — but dev's move is a separate,
|
||||
# operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write
|
||||
# a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window
|
||||
# would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo
|
||||
# (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_ENDPOINT
|
||||
value: "http://192.168.1.9:9000"
|
||||
# DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same
|
||||
# reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base
|
||||
# defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones
|
||||
# its proven restore depends on — without any manifest looking wrong.
|
||||
#
|
||||
# `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO
|
||||
# ignores the value, but changing it is a needless risk against a live, working backup path.
|
||||
# An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already
|
||||
# are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified
|
||||
# against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_REGION
|
||||
value: "us-east-1"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_BUCKET
|
||||
value: "velero"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_PREFIX
|
||||
value: ""
|
||||
- op: replace
|
||||
path: /data/CNPG_BACKUP_PATH
|
||||
value: "s3://cnpg"
|
||||
# WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets
|
||||
# (velero-cloud-credentials, <db>-backup-credentials) — the scoped MinIO keys dev's Velero and
|
||||
# its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod
|
||||
# renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead.
|
||||
#
|
||||
# RESTATED EXPLICITLY even though it equals the base default, for the same reason as
|
||||
# EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED
|
||||
# base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev
|
||||
# signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on
|
||||
# turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR
|
||||
# restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential.
|
||||
#
|
||||
# DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator,
|
||||
# 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING
|
||||
# backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs.
|
||||
- op: replace
|
||||
path: /data/BACKUP_CREDENTIALS_SUFFIX
|
||||
value: ""
|
||||
# dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas.
|
||||
- op: replace
|
||||
path: /data/LONGHORN_DEFAULT_REPLICA_COUNT
|
||||
|
||||
@@ -392,10 +392,91 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CRAFTY_L4_VIP
|
||||
value: "192.168.2.22"
|
||||
# Off-cluster DR target. Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the
|
||||
# network — a prod backup written there is not off-cluster DR and would mix two environments'
|
||||
# PITR streams in one bucket. Operator supplies the prod endpoint (full URL incl. scheme+port).
|
||||
# ==========================================================================================
|
||||
# OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
|
||||
# ==========================================================================================
|
||||
# Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the network and is NOT an
|
||||
# option for prod, for two independent reasons:
|
||||
# 1. UNREACHABLE. Prod is on the SOHO network 192.168.2.0/24 — pfSense's WAN — while that
|
||||
# MinIO is inside pfSense's LAN. Measured from a pod INSIDE prod (the workstation proves
|
||||
# nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` → connection
|
||||
# timed out after 15s. Reaching it would need a WAN→LAN rule that does not exist.
|
||||
# 2. NOT OFF-CLUSTER DR. It runs on the same Proxmox host as dev's VMs, so it already shares
|
||||
# a physical failure domain with dev. R2 is genuinely off-site, and its free egress means
|
||||
# restore REHEARSALS cost nothing — R-STATE-03 has no financial disincentive to stay proven.
|
||||
#
|
||||
# THIS UNBLOCKS AN ACTIVE INCIDENT, not just a roadmap item. All three prod CNPG Clusters ship
|
||||
# `archive_mode=on` with `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT`, so barman-cloud-wal-archive
|
||||
# exits 4 on every segment and Postgres — correctly — REFUSES TO RECYCLE WAL. Verified live
|
||||
# 2026-09-07: keycloak-db / nextcloud-db / semaphore-db all report
|
||||
# ContinuousArchiving=False/ContinuousArchivingFailing, and keycloak-db's 5 GiB PVC is at
|
||||
# 579 MiB used / 4360 MiB free and climbing monotonically. A disk-full PANIC there takes the
|
||||
# cluster's SSO database down, and with it kubectl OIDC, Grafana, Headlamp, Gitea and
|
||||
# forward-auth. Fixing the endpoint lets Postgres drain the whole backlog.
|
||||
#
|
||||
# ENDPOINT IS ACCOUNT-SCOPED, so it stays a prod-overlay value even though it is not a LAN
|
||||
# address: this is the operator's Cloudflare account id, confirmed from inside a prod pod
|
||||
# (`head-bucket` on k3s-prod-backup → BucketRegion WNAM; PUT and DELETE both succeed).
|
||||
# It is NOT put in base, because base is dev's live MinIO and dev is deliberately STAYING
|
||||
# there — moving dev to R2 would reset its PITR window and orphan the backups its PROVEN
|
||||
# restore depends on. Two environments, two targets, isolated by URL *and* by bucket.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_ENDPOINT
|
||||
value: "TBD-PROD-S3-BACKUP-ENDPOINT"
|
||||
value: "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com"
|
||||
# R2 requires the region string to be exactly `auto`. (`head-bucket` answers
|
||||
# `{"BucketRegion":"WNAM"}` — that is a PHYSICAL LOCATION HINT, not a signing region; using it
|
||||
# here produces AuthorizationHeaderMalformed.) Dev keeps the base `us-east-1` its live MinIO
|
||||
# has always been signed with.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_REGION
|
||||
value: "auto"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# OBJECT TENANCY INSIDE THE ONE BUCKET. The operator created a SINGLE R2 bucket, so velero and
|
||||
# barman cannot be separated by bucket the way they are on dev's MinIO (`velero` and `cnpg`).
|
||||
# They are separated by disjoint TOP-LEVEL PREFIXES instead:
|
||||
#
|
||||
# s3://k3s-prod-backup/velero/… Velero: kopia repo + backup metadata
|
||||
# s3://k3s-prod-backup/barman/<clusterName>/… CNPG: WAL + base backups
|
||||
#
|
||||
# The bucket ROOT is left empty on purpose — a misrendered prefix then lands somewhere
|
||||
# obviously wrong instead of silently colliding with a live tenant. Velero's retention TTL
|
||||
# deletes objects under its own prefix only, so it can never reap a WAL segment; and barman's
|
||||
# per-cluster serverName subdirectory keeps the three Postgres servers' chains apart, which
|
||||
# matters because interleaving two servers' WAL into one prefix corrupts the chain outright.
|
||||
# Dev cannot be reached from any of these: different endpoint AND different bucket names.
|
||||
# ------------------------------------------------------------------------------------------
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_BUCKET
|
||||
value: "k3s-prod-backup"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_PREFIX
|
||||
value: "velero"
|
||||
- op: replace
|
||||
path: /data/CNPG_BACKUP_PATH
|
||||
value: "s3://k3s-prod-backup/barman"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# WHICH CREDENTIAL SIGNS THE REQUESTS TO THAT BUCKET (per-env credential split, 2026-09-07).
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The knobs above moved the ENDPOINT and the OBJECT LAYOUT per environment. The credential did
|
||||
# not move with them, and it could not: the four backup-credential Secrets are SHARED BASE
|
||||
# manifests, applied to both clusters by the same Flux tiers. Writing the R2 token into them
|
||||
# gave prod a working key by TAKING DEV'S AWAY — dev would have signed MinIO requests with an
|
||||
# R2 key id, and under archive_mode=on that is a filling pgdata PVC, not a missing backup.
|
||||
#
|
||||
# So each credential is now two objects, and this suffix selects prod's:
|
||||
# velero-cloud-credentials-prod the R2 token in the AWS INI `cloud` shape
|
||||
# <db>-backup-credentials-prod the same token as ACCESS_KEY_ID / ACCESS_SECRET_KEY
|
||||
# for keycloak-db, nextcloud-db and semaphore-db. Both halves ship to both clusters (they are
|
||||
# base resources); ONLY this value decides which one each consumer reads. It is a SUFFIX rather
|
||||
# than four secret-name knobs deliberately — four could be updated three-quarters of the way,
|
||||
# leaving one consumer on dev's MinIO key with nothing rendering wrong, and strict-mode
|
||||
# substitution cannot catch a defined-but-stale variable.
|
||||
#
|
||||
# THE R2 TOKEN IS THE ONE PROVEN FROM INSIDE PROD (chore/r2-credentials 0a6a862): head-bucket on
|
||||
# k3s-prod-backup returns BucketRegion WNAM, and PUT and DELETE both succeed. Rotating it now
|
||||
# touches ONLY the four `-prod` files; dev's MinIO keys are untouched objects. That isolation is
|
||||
# the structural reason for the split, independent of dev's eventual decommission.
|
||||
- op: replace
|
||||
path: /data/BACKUP_CREDENTIALS_SUFFIX
|
||||
value: "-prod"
|
||||
# TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist.
|
||||
+20
-4
@@ -84,6 +84,9 @@ toolchain: yamllint, check-jsonschema, kyverno, chainsaw, yq) lives in the image
|
||||
infra/devcontainer/dev.sh build # once
|
||||
infra/devcontainer/dev.sh run make -C ci gate # exactly what CI runs
|
||||
infra/devcontainer/dev.sh # interactive shell at /work
|
||||
|
||||
infra/devcontainer/dev.sh run kubectl get nodes # DEV (the default)
|
||||
CLUSTER=prod infra/devcontainer/dev.sh run kubectl get nodes # PROD
|
||||
```
|
||||
|
||||
Unlike the libvirt era, **tofu and ansible now run fine inside the container** — the
|
||||
@@ -95,10 +98,23 @@ which is what broke the old `dmacvicar/libvirt` provider in a container.
|
||||
file, and the sibling spec repo read-only at `/home-network-k3s` so the spec-immutability
|
||||
check runs instead of silently skipping.
|
||||
|
||||
> **Known gap.** `dev.sh` defaults `KUBECONFIG` to `/work/infra/ansible/.cache/dev.kubeconfig`,
|
||||
> a libvirt-era path that no longer exists, and does not mount `~/.kube`. In-container `kubectl`
|
||||
> therefore needs an explicit `DEV_KUBECONFIG=<path to a break-glass cert kubeconfig>` — the OIDC
|
||||
> context cannot work there anyway (no browser). Not yet fixed.
|
||||
> **Closed (was a known gap): `dev.sh` can now target either cluster.** It used to hard-wire
|
||||
> `KUBECONFIG` to `dev.kubeconfig`, leaving an undocumented — and misleadingly named —
|
||||
> `DEV_KUBECONFIG=<prod path>` as the only route to prod. Select with **`CLUSTER=dev|prod`**;
|
||||
> **dev remains the default**, so every existing invocation means what it always meant. Inside the
|
||||
> container `KUBECONFIG` is a chain of a tiny per-invocation selector file over a merged config
|
||||
> carrying both clusters as contexts **`k3s-dev`** and **`k3s-prod`** (never `default`), regenerated
|
||||
> each run by [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
|
||||
> `infra/ansible/.cache/clusters.kubeconfig`. The merged file deliberately carries **no**
|
||||
> `current-context`: it is shared by every container, and stamping the current cluster into it made
|
||||
> concurrent `CLUSTER=dev` and `CLUSTER=prod` runs silently swap clusters mid-command. Naming matters: k3s writes *both* source kubeconfigs
|
||||
> with context, cluster and user all called `default`, and kubectl's merge keeps the first
|
||||
> duplicate — so a plain `KUBECONFIG=dev:prod` would silently resolve `default` to **dev** while you
|
||||
> believed you were on prod. Requesting a cluster whose kubeconfig is absent is a hard error, never
|
||||
> a fallback. `DEV_KUBECONFIG` still works and still wins, but is **superseded** — it is opaque
|
||||
> about its target and yields an ambiguous `default` context. `~/.kube` is still not mounted and the
|
||||
> OIDC context still cannot work in a container (no browser); use the break-glass cert kubeconfigs
|
||||
> above. Details: [`devcontainer/README.md`](devcontainer/README.md).
|
||||
|
||||
## Off-repo state you must not lose
|
||||
|
||||
|
||||
@@ -33,6 +33,84 @@ cd infra/devcontainer
|
||||
```
|
||||
`ENGINE=docker ./dev.sh` to use docker instead of rootless podman.
|
||||
|
||||
## Choosing a cluster — `CLUSTER=dev|prod`
|
||||
|
||||
```bash
|
||||
./dev.sh run kubectl get nodes # DEV — the default, unchanged
|
||||
CLUSTER=prod ./dev.sh run kubectl get nodes # PROD
|
||||
CLUSTER=prod ./dev.sh # interactive shell pointed at prod
|
||||
```
|
||||
|
||||
An env-var **prefix**, so the target sits on the same line as the command. Every invocation prints
|
||||
a banner naming the cluster before the container starts, because `CLUSTER` can also be inherited
|
||||
from an exported shell variable.
|
||||
|
||||
Inside the container `KUBECONFIG` is a two-file **chain**,
|
||||
`<per-invocation selector>:<shared merged config>`:
|
||||
|
||||
| file | shared? | holds |
|
||||
|---|---|---|
|
||||
| selector, in a private `mktemp` dir | no — one per `dev.sh` invocation | only `current-context: k3s-$CLUSTER`; no credentials |
|
||||
| `infra/ansible/.cache/clusters.kubeconfig` | yes — bind-mounted, read by every container | both clusters under unambiguous names; **no `current-context`** |
|
||||
|
||||
kubectl merges the chain first-wins, so the selector supplies this invocation's default:
|
||||
|
||||
```
|
||||
CURRENT NAME CLUSTER AUTHINFO
|
||||
* k3s-dev k3s-dev k3s-dev-admin
|
||||
k3s-prod k3s-prod k3s-prod-admin
|
||||
```
|
||||
|
||||
So `kubectl config current-context` answers "which cluster am I on?" honestly, and
|
||||
`kubectl --context k3s-prod …` works from a dev-defaulted shell.
|
||||
|
||||
**Why the default context is not stored in the merged file.** It used to be, and that was a
|
||||
cross-cluster data-corruption bug. The merged file lives in the repo's `.cache`, which every
|
||||
container bind-mounts, so `current-context` was **shared mutable state**: two concurrent `dev.sh`
|
||||
runs with different `CLUSTER` values overwrote each other's default *mid-command*. A
|
||||
`CLUSTER=prod` script read prod for its first half and **dev** for its second; it was caught only
|
||||
because a dev-only PVC name showed up in a prod query. Moving selection into a per-invocation file
|
||||
removes the shared field rather than locking it — there is no window left to lose. Read on its own
|
||||
the merged file now resolves to nothing (`error: current-context is not set`), which is loud and
|
||||
safe; `--context k3s-dev` / `--context k3s-prod` against it are unaffected and remain the pattern
|
||||
to prefer in scripts. Concurrent `CLUSTER=dev` and `CLUSTER=prod` invocations are now safe.
|
||||
|
||||
Consequences worth knowing:
|
||||
|
||||
- Export what `kubeconfig.sh` prints **verbatim** — it is a colon-joined chain, not one path.
|
||||
- `kubectl config use-context …` in an interactive toolbox shell writes to the **first** file in
|
||||
the chain (the selector), so a manual switch stays local to that container.
|
||||
|
||||
**Why a merge script and not `KUBECONFIG=dev.kubeconfig:prod.kubeconfig`.** k3s names the context,
|
||||
the cluster *and* the user in **both** files `default`. kubectl's merge keeps the **first**
|
||||
occurrence of a duplicated name, so a colon-joined `KUBECONFIG` silently resolves `default` to
|
||||
**dev** and `--context default` can never reach prod — you would believe you were on prod while
|
||||
issuing commands against the live dev cluster. `kubeconfig.sh` therefore **renames on merge**
|
||||
(`k3s-dev` / `k3s-prod`, `*-admin` users) with `yq`, structurally — not with `sed`, which corrupts
|
||||
these files because the cluster and the user entry are both `- name: default` at identical indent.
|
||||
|
||||
- Generator: [`kubeconfig.sh`](kubeconfig.sh). Output: `infra/ansible/.cache/clusters.kubeconfig`,
|
||||
mode 0600, **regenerated on every `dev.sh run`** — a build artifact, never hand-edited, never
|
||||
committed (`.cache/` and `*.kubeconfig` are both gitignored; it embeds cluster-admin certs).
|
||||
Its content no longer depends on which cluster was asked for, so concurrent regenerations write
|
||||
identical bytes; the write is published by atomic rename so a concurrent reader can never load a
|
||||
half-written kubeconfig.
|
||||
- A cluster whose source kubeconfig is missing is simply omitted from the merge, so a machine with
|
||||
only `dev.kubeconfig` still works.
|
||||
- **Asking for a cluster whose kubeconfig is absent is a hard error.** `dev.sh` refuses on the host
|
||||
before starting a container and never falls back to the other cluster — that fallback is exactly
|
||||
how a prod command lands on dev.
|
||||
- `bootstrap/standup.sh` is unaffected: it exports its own `KUBECONFIG` from the mandatory
|
||||
`--kubeconfig` flag, which overrides what `dev.sh` sets. Its `--expect-server` guard remains the
|
||||
thing that makes it safe, not `CLUSTER`.
|
||||
|
||||
### `DEV_KUBECONFIG` — superseded, still honoured
|
||||
|
||||
`DEV_KUBECONFIG=<path> ./dev.sh run …` still pins `KUBECONFIG` verbatim and takes precedence over
|
||||
`CLUSTER` (nothing depending on it breaks), but it is **deprecated**: the name says "dev" while the
|
||||
established workaround set it to a *prod* path, and it produces a single-cluster config whose
|
||||
context is the ambiguous `default`. Use `CLUSTER=dev|prod`. Invocations that use it print a warning.
|
||||
|
||||
## What `dev.sh` mounts, and why each one matters
|
||||
|
||||
- `--network host` — Ansible SSHes to the nodes through the pfSense NAT (`192.168.2.102:225x`) and
|
||||
@@ -61,18 +139,16 @@ SSH, which `--network host` provides.
|
||||
|
||||
So: run everything in here. The host needs only podman (or docker).
|
||||
|
||||
## kubectl inside the container — known gap
|
||||
## kubectl inside the container — was a known gap, now fixed
|
||||
|
||||
`dev.sh` sets `KUBECONFIG` to `/work/infra/ansible/.cache/dev.kubeconfig`, a **libvirt-era path
|
||||
that no longer exists**, and does not mount `~/.kube`. Override it explicitly:
|
||||
`dev.sh` used to hard-wire `KUBECONFIG` to `dev.kubeconfig` with no supported way to reach prod, and
|
||||
it does not mount `~/.kube`. It now selects between `infra/ansible/.cache/dev.kubeconfig` and
|
||||
`prod.kubeconfig` via `CLUSTER=dev|prod` (see above) and merges both under distinct context names.
|
||||
|
||||
```bash
|
||||
DEV_KUBECONFIG=/work/path/to/break-glass.kubeconfig ./dev.sh run kubectl get nodes
|
||||
```
|
||||
|
||||
Use a **break-glass cert kubeconfig**, not the `k3s-oidc` context — OIDC login needs a browser,
|
||||
which a container shell does not have. See [`../PHASE1-STATUS.md`](../PHASE1-STATUS.md) for both
|
||||
access paths. (Not yet fixed in `dev.sh`.)
|
||||
The container still needs a **break-glass cert kubeconfig**, not the `k3s-oidc` context — OIDC login
|
||||
needs a browser, which a container shell does not have. See
|
||||
[`../PHASE1-STATUS.md`](../PHASE1-STATUS.md) for both access paths and for how to regenerate these
|
||||
two files from `/etc/rancher/k3s/k3s.yaml`.
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
@@ -11,7 +11,36 @@
|
||||
# ./dev.sh build # build the image only
|
||||
# ./dev.sh # build (if needed) + open a shell
|
||||
# ./dev.sh run tofu -version # run one command in the toolbox
|
||||
# ./dev.sh run kubectl get nodes # talks to the Proxmox k3s cluster directly (no tunnel)
|
||||
# ./dev.sh run kubectl get nodes # talks to the DEV k3s cluster directly (no tunnel)
|
||||
#
|
||||
# CHOOSING A CLUSTER — `CLUSTER=<dev|prod>`, defaulting to dev:
|
||||
#
|
||||
# ./dev.sh run kubectl get nodes # dev (the default; unchanged behaviour)
|
||||
# CLUSTER=prod ./dev.sh run kubectl get nodes # prod
|
||||
# CLUSTER=prod ./dev.sh # interactive shell pointed at prod
|
||||
#
|
||||
# The selector is an env-var PREFIX so the target is visible on the same line as the command, and
|
||||
# every invocation prints a banner naming the cluster it is about to talk to. Inside the container
|
||||
# KUBECONFIG is a two-file CHAIN, "<per-invocation selector>:<shared merged config>":
|
||||
#
|
||||
# * the shared merged config holds BOTH clusters under unambiguous names — contexts `k3s-dev`
|
||||
# and `k3s-prod`, never `default` — so `--context k3s-prod` actually works, and it carries NO
|
||||
# current-context of its own;
|
||||
# * the selector is a private per-invocation file holding only `current-context: k3s-$CLUSTER`.
|
||||
#
|
||||
# So `kubectl config current-context` still answers the "which cluster am I on?" question honestly,
|
||||
# but the answer is now PER INVOCATION. That is the point: the previous design stamped
|
||||
# current-context into the one shared merged file, so two concurrent dev.sh runs with different
|
||||
# CLUSTER values overwrote each other's default MID-COMMAND — a `CLUSTER=prod` script could read
|
||||
# prod for its first half and DEV for its second. See infra/devcontainer/kubeconfig.sh for the full
|
||||
# write-up, and for why the naive `KUBECONFIG=dev:prod` merge is dangerous in the first place (both
|
||||
# source files name everything `default`).
|
||||
#
|
||||
# Concurrency note: `CLUSTER=dev ./dev.sh run ...` and `CLUSTER=prod ./dev.sh run ...` may now run
|
||||
# at the same time and neither can affect the other's default cluster.
|
||||
#
|
||||
# Requesting a cluster whose kubeconfig is missing is a hard ERROR — it never falls back to the
|
||||
# other cluster.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
|
||||
@@ -24,6 +53,15 @@ source "${HERE}/versions.env"
|
||||
# the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443). No SSH tunnel is
|
||||
# opened. infra/ansible/.cache/dev.kubeconfig targets https://192.168.2.50:6443 with
|
||||
# tls-server-name=192.168.1.49 so TLS validates against the VIP's SAN through the NAT.
|
||||
# Prod is reached the same way via infra/ansible/.cache/prod.kubeconfig (https://192.168.2.9:6443).
|
||||
|
||||
# ------------------------------------------------------------------ cluster selection
|
||||
CLUSTER="${CLUSTER:-dev}"
|
||||
case "${CLUSTER}" in
|
||||
dev|prod) ;;
|
||||
*) printf '\033[31mERROR: CLUSTER=%s is not a known cluster (dev|prod).\033[0m\n' "${CLUSTER}" >&2; exit 2 ;;
|
||||
esac
|
||||
CLUSTER_KUBECONFIG="${REPO_ROOT}/infra/ansible/.cache/${CLUSTER}.kubeconfig"
|
||||
|
||||
build() {
|
||||
"$ENGINE" build \
|
||||
@@ -61,6 +99,26 @@ run() {
|
||||
if [ -n "${spec_host}" ] && [ -d "${spec_host}/.git" ]; then
|
||||
spec_mount=(-v "${spec_host}:/home-network-k3s:ro" -e SPEC_REPO=/home-network-k3s)
|
||||
fi
|
||||
|
||||
# Which cluster, said out loud, before anything runs. CLUSTER can be inherited from an exported
|
||||
# shell variable, so the banner is what stops "I thought I was on dev".
|
||||
if [ -n "${DEV_KUBECONFIG:-}" ]; then
|
||||
# SUPERSEDED but still honoured: DEV_KUBECONFIG used to be the only way to reach a cluster other
|
||||
# than the (then non-existent) default dev path. It wins over CLUSTER so nothing that depends on
|
||||
# it breaks, but it is opaque about its target — prefer CLUSTER=dev|prod.
|
||||
printf '\033[33m==> dev.sh: KUBECONFIG pinned by DEV_KUBECONFIG=%s (CLUSTER=%s ignored; DEV_KUBECONFIG is superseded — use CLUSTER=dev|prod)\033[0m\n' \
|
||||
"${DEV_KUBECONFIG}" "${CLUSTER}" >&2
|
||||
else
|
||||
# Fail here, on the host, before a container is even started: a missing kubeconfig must never
|
||||
# degrade into "whatever config we could find", because that is how a prod command lands on dev.
|
||||
if [ ! -f "${CLUSTER_KUBECONFIG}" ]; then
|
||||
printf '\033[31mERROR: dev.sh: CLUSTER=%s but its kubeconfig is MISSING: %s\n Refusing to run. dev.sh does NOT fall back to the other cluster.\033[0m\n' \
|
||||
"${CLUSTER}" "${CLUSTER_KUBECONFIG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '\033[1m==> dev.sh: cluster=%s (kube context k3s-%s)\033[0m\n' "${CLUSTER}" "${CLUSTER}" >&2
|
||||
fi
|
||||
|
||||
"$ENGINE" run --rm -it \
|
||||
--network host \
|
||||
--security-opt label=disable \
|
||||
@@ -69,8 +127,25 @@ run() {
|
||||
-v "${HOME}/.ssh:/root/.ssh:ro" \
|
||||
-v "${HOME}/.config/sops:/root/.config/sops" \
|
||||
-v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \
|
||||
-e KUBECONFIG="${DEV_KUBECONFIG:-/work/infra/ansible/.cache/dev.kubeconfig}" \
|
||||
"$IMAGE" bash -lc 'if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi' dev.sh "$@"
|
||||
-e CLUSTER="${CLUSTER}" \
|
||||
-e DEV_KUBECONFIG="${DEV_KUBECONFIG:-}" \
|
||||
"$IMAGE" bash -lc '
|
||||
set -e
|
||||
if [ -n "${DEV_KUBECONFIG:-}" ]; then
|
||||
export KUBECONFIG="${DEV_KUBECONFIG}"
|
||||
else
|
||||
# Regenerated every run: cheap, and it means an updated dev/prod.kubeconfig is picked up
|
||||
# without anyone remembering to refresh a hand-made merged file. Exits non-zero (set -e)
|
||||
# if the requested cluster is missing — no silent fallback.
|
||||
#
|
||||
# This prints a KUBECONFIG CHAIN ("<per-invocation selector>:<shared merged config>"), not a
|
||||
# single path. Export it verbatim; do not split it or take only the last element, or this
|
||||
# invocation loses its private default context and inherits whatever is shared.
|
||||
KUBECONFIG="$(/work/infra/devcontainer/kubeconfig.sh "${CLUSTER}")"
|
||||
export KUBECONFIG
|
||||
fi
|
||||
if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi
|
||||
' dev.sh "$@"
|
||||
}
|
||||
|
||||
case "${1:-shell}" in
|
||||
|
||||
Executable
+183
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# infra/devcontainer/kubeconfig.sh — build ONE merged kubeconfig that can address BOTH clusters
|
||||
# under unambiguous names, and select which of them is current WITHOUT mutating shared state.
|
||||
#
|
||||
# kubeconfig.sh <dev|prod> # regenerate the merged file, mint a per-invocation selector,
|
||||
# # print the KUBECONFIG *chain* to use (selector:merged)
|
||||
#
|
||||
# WHY THIS EXISTS — the name collision that makes the obvious fix dangerous
|
||||
# infra/ansible/.cache/dev.kubeconfig and .cache/prod.kubeconfig each name their context, their
|
||||
# cluster AND their user `default`. k3s writes them that way. So the "obvious" fix —
|
||||
# KUBECONFIG=dev.kubeconfig:prod.kubeconfig — is actively unsafe: kubectl's merge keeps the FIRST
|
||||
# occurrence of a duplicated name, so `default` silently resolves to DEV and `--context default`
|
||||
# can NEVER select prod. An operator (or an agent) would believe they were on prod while issuing
|
||||
# commands against the live dev cluster.
|
||||
#
|
||||
# Therefore we RENAME on merge rather than relying on file order:
|
||||
# dev.kubeconfig -> cluster/context `k3s-dev`, user `k3s-dev-admin`
|
||||
# prod.kubeconfig -> cluster/context `k3s-prod`, user `k3s-prod-admin`
|
||||
# The rename is done with `yq` (structural, YAML-aware). It is NOT done with sed: the cluster and
|
||||
# the user entry are both `- name: default` at identical indent, so a textual rename corrupts the
|
||||
# file. The two renamed copies are then merged by kubectl itself, which is safe once the names are
|
||||
# distinct.
|
||||
#
|
||||
# WHY THE MERGED FILE HAS NO `current-context` — the concurrency bug this shape fixes
|
||||
# This script used to stamp `current-context: k3s-<target>` into the ONE shared merged file. That
|
||||
# file lives in the repo's .cache, which every dev.sh container bind-mounts, so it is shared
|
||||
# mutable state across concurrent invocations. Two `dev.sh` runs with different CLUSTER values
|
||||
# therefore raced: `CLUSTER=prod dev.sh run <script>` would start on prod, a `CLUSTER=dev` run
|
||||
# would rewrite `current-context` to k3s-dev mid-script, and the second half of the prod script
|
||||
# would silently read the DEV cluster. This was hit live; it was caught only by luck (a PVC name
|
||||
# that could not exist on prod). Silent cross-cluster reads — and, worse, cross-cluster WRITES —
|
||||
# are the exact failure mode this file exists to prevent.
|
||||
#
|
||||
# The fix removes the shared mutable field entirely instead of trying to lock it:
|
||||
# * the merged file carries BOTH contexts and NO current-context. Read on its own it resolves
|
||||
# to nothing and kubectl errors with "current-context is not set" — loud, never a wrong
|
||||
# cluster. `--context k3s-dev` / `--context k3s-prod` against it keep working unchanged, and
|
||||
# that remains the pattern to prefer.
|
||||
# * the default context is supplied by a tiny PER-INVOCATION selector file — literally just
|
||||
# `current-context: k3s-<target>`, no credentials — created under a private mktemp dir, and
|
||||
# handed back FIRST in a KUBECONFIG chain: "<selector>:<merged>". kubectl merges the chain
|
||||
# first-wins, so the selector decides this invocation's default and nothing else's.
|
||||
# Nothing in the chain is shared and writable at once, so there is no lock to take and no window
|
||||
# to lose. A concurrent regeneration can only rewrite the merged file with byte-identical
|
||||
# content, and that write is done by atomic rename so a reader never sees a half-written file.
|
||||
# As a bonus, `kubectl config use-context` inside an interactive shell writes to the FIRST file
|
||||
# in the chain — the per-invocation selector — so even a manual context switch stays local to
|
||||
# that container instead of leaking into everyone else's session.
|
||||
#
|
||||
# OUTPUT
|
||||
# stdout: the KUBECONFIG chain "<selector>:<merged>" — export it verbatim, do not split it.
|
||||
# merged: ${KUBECONFIG_CACHE_DIR:-<repo>/infra/ansible/.cache}/clusters.kubeconfig, mode 0600.
|
||||
# That directory is gitignored (infra/ansible/.gitignore: `.cache/`) and `*.kubeconfig` is
|
||||
# ignored repo-wide. The file embeds cluster-admin client certs — it must NEVER be committed.
|
||||
# It is a BUILD ARTIFACT: delete it any time, this script rebuilds it.
|
||||
# selector: a ~90-byte file in a 0700 mktemp dir (override the parent with
|
||||
# KUBECONFIG_SELECTOR_DIR). It holds no credentials. It is deliberately NOT cleaned up on exit,
|
||||
# because the caller needs it for the whole life of its shell; inside dev.sh's `--rm` container
|
||||
# it dies with the container. Run standalone on a host it leaves one tiny dir behind.
|
||||
#
|
||||
# TOLERANCE / FAILURE
|
||||
# A cluster whose source kubeconfig is absent is simply left out of the merged file (so a machine
|
||||
# that only has dev still works). But asking for a cluster whose kubeconfig is absent is a hard
|
||||
# ERROR — never a silent fallback to the other cluster.
|
||||
#
|
||||
# Requires: kubectl, yq (both baked into the dev container image). Normally invoked by dev.sh from
|
||||
# inside the container; it runs fine anywhere those two binaries exist.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
|
||||
CACHE_DIR="${KUBECONFIG_CACHE_DIR:-${REPO_ROOT}/infra/ansible/.cache}"
|
||||
OUT="${CACHE_DIR}/clusters.kubeconfig"
|
||||
|
||||
# cluster name -> source kubeconfig. Add a cluster here and it flows through dev.sh unchanged.
|
||||
src_for() {
|
||||
case "$1" in
|
||||
dev) echo "${DEV_SRC_KUBECONFIG:-${CACHE_DIR}/dev.kubeconfig}" ;;
|
||||
prod) echo "${PROD_SRC_KUBECONFIG:-${CACHE_DIR}/prod.kubeconfig}" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
CLUSTERS=(dev prod)
|
||||
|
||||
die() { printf '\033[31mERROR: kubeconfig.sh: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
TARGET="${1:-}"
|
||||
[ -n "${TARGET}" ] || die "usage: kubeconfig.sh <$(IFS='|'; echo "${CLUSTERS[*]}")>"
|
||||
src_for "${TARGET}" >/dev/null 2>&1 || die "unknown cluster '${TARGET}' (known: ${CLUSTERS[*]})"
|
||||
|
||||
for bin in kubectl yq; do
|
||||
command -v "$bin" >/dev/null 2>&1 || die "'$bin' not found — run this inside infra/devcontainer."
|
||||
done
|
||||
|
||||
# --- FAIL LOUDLY, do not fall back -------------------------------------------------------------
|
||||
target_src="$(src_for "${TARGET}")"
|
||||
if [ ! -f "${target_src}" ]; then
|
||||
die "cluster '${TARGET}' was requested but its kubeconfig is MISSING: ${target_src}
|
||||
Refusing to continue. Falling back to the other cluster would run your '${TARGET}'
|
||||
commands against a DIFFERENT LIVE CLUSTER. Fetch the kubeconfig first (see
|
||||
infra/PHASE1-STATUS.md, 'break-glass cert kubeconfig')."
|
||||
fi
|
||||
|
||||
mkdir -p "${CACHE_DIR}"
|
||||
tmp="$(mktemp -d)"
|
||||
chmod 700 "${tmp}"
|
||||
staged=""
|
||||
trap 'rm -rf "${tmp}"; [ -n "${staged}" ] && rm -f "${staged}"; true' EXIT
|
||||
|
||||
parts=()
|
||||
present=()
|
||||
for c in "${CLUSTERS[@]}"; do
|
||||
s="$(src_for "$c")"
|
||||
[ -f "$s" ] || { printf ' kubeconfig.sh: %-4s kubeconfig absent (%s) — omitted from the merged config\n' "$c" "$s" >&2; continue; }
|
||||
ctx="k3s-${c}"; usr="k3s-${c}-admin"
|
||||
# Structural rename. Every kubeconfig k3s emits here has exactly one cluster/user/context, but
|
||||
# [] indexing renames all of them consistently rather than assuming index 0 exists.
|
||||
# No `current-context` is set on the parts, and none survives into the merged file: the merged
|
||||
# file must stay identical no matter which cluster was asked for, because it is SHARED across
|
||||
# concurrent invocations. Selection happens in the per-invocation selector below.
|
||||
ctx="$ctx" usr="$usr" yq '
|
||||
.clusters[].name = env(ctx)
|
||||
| .users[].name = env(usr)
|
||||
| .contexts[].name = env(ctx)
|
||||
| .contexts[].context.cluster = env(ctx)
|
||||
| .contexts[].context.user = env(usr)
|
||||
| del(."current-context")
|
||||
' "$s" > "${tmp}/${c}.yaml"
|
||||
chmod 600 "${tmp}/${c}.yaml"
|
||||
parts+=("${tmp}/${c}.yaml")
|
||||
present+=("$c")
|
||||
done
|
||||
|
||||
[ "${#parts[@]}" -gt 0 ] || die "no kubeconfigs found under ${CACHE_DIR}"
|
||||
|
||||
# Names are distinct now, so kubectl's merge cannot silently drop one. --raw keeps the embedded
|
||||
# credential data (a merged config without it authenticates as nobody). `view` re-emits an empty
|
||||
# `current-context: ""` even when no part had one; drop the key so the file is unambiguous.
|
||||
KUBECONFIG="$(IFS=:; echo "${parts[*]}")" kubectl config view --flatten --raw > "${tmp}/merged.yaml"
|
||||
yq -i 'del(."current-context")' "${tmp}/merged.yaml"
|
||||
chmod 600 "${tmp}/merged.yaml"
|
||||
|
||||
# Publish the merged file by ATOMIC RENAME, not by writing in place. Concurrent invocations produce
|
||||
# byte-identical content, but `install`/`>` truncate first, so an unlucky concurrent reader could
|
||||
# otherwise load a half-written kubeconfig. rename(2) within one filesystem swaps it whole.
|
||||
staged="$(mktemp "${CACHE_DIR}/.clusters.kubeconfig.XXXXXX")"
|
||||
cat "${tmp}/merged.yaml" > "${staged}"
|
||||
chmod 600 "${staged}"
|
||||
mv -f "${staged}" "${OUT}"
|
||||
|
||||
# Post-condition on the SHARED file: both renamed contexts present, nothing called `default`, and
|
||||
# no current-context to race over.
|
||||
if KUBECONFIG="${OUT}" kubectl config get-contexts -o name | grep -qx default; then
|
||||
die "internal: a context named 'default' survived the rename — refusing to hand back an ambiguous config"
|
||||
fi
|
||||
KUBECONFIG="${OUT}" kubectl config get-contexts -o name | grep -qx "k3s-${TARGET}" \
|
||||
|| die "internal: context 'k3s-${TARGET}' is missing from ${OUT}"
|
||||
if KUBECONFIG="${OUT}" kubectl config current-context >/dev/null 2>&1; then
|
||||
die "internal: ${OUT} carries a current-context — that field is shared mutable state and must stay unset"
|
||||
fi
|
||||
|
||||
# --- per-invocation selector: this invocation's default context, private to this invocation ------
|
||||
# `kubectl config set` (a config subcommand, never sed) writes the single scalar. It does not
|
||||
# validate the name, which is why the context's existence was asserted against the merged file
|
||||
# above.
|
||||
sel_dir="$(mktemp -d "${KUBECONFIG_SELECTOR_DIR:-${TMPDIR:-/tmp}}/kubeconfig-sel.XXXXXX")"
|
||||
chmod 700 "${sel_dir}"
|
||||
SEL="${sel_dir}/selector.kubeconfig"
|
||||
printf 'apiVersion: v1\nkind: Config\n' > "${SEL}"
|
||||
chmod 600 "${SEL}"
|
||||
KUBECONFIG="${SEL}" kubectl config set current-context "k3s-${TARGET}" >/dev/null
|
||||
|
||||
CHAIN="${SEL}:${OUT}"
|
||||
|
||||
# Post-condition on the CHAIN — the thing the caller will actually export. This is the end-to-end
|
||||
# assertion that this invocation defaults to the cluster that was asked for.
|
||||
cur="$(KUBECONFIG="${CHAIN}" kubectl config current-context)"
|
||||
[ "${cur}" = "k3s-${TARGET}" ] || die "internal: current-context is '${cur}', expected 'k3s-${TARGET}'"
|
||||
|
||||
printf ' kubeconfig.sh: merged [%s] -> %s (no current-context; shared)\n' "${present[*]}" "${OUT}" >&2
|
||||
printf ' kubeconfig.sh: this invocation defaults to %s via %s\n' "${cur}" "${SEL}" >&2
|
||||
echo "${CHAIN}"
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: keycloak-db-backup-credentials-prod
|
||||
namespace: keycloak
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:kQ5x5fXb6KdVaDsItGug2cpQMZw9tSvbsYDqTMxoP+A=,iv:dCTfwVxlpnjaMhGBIMa18d4yAiQ6SKhb4EMm1p1JyQ8=,tag:sNqMLs+rGQm/1MyV90InDQ==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:9lGvwvseAWK1bMXDXZJ+bjw4oHH/XVPi8+C8vfXGJ3jX8VW95rJrczPbEfV9rCudVIJFnxAdaQZZWaJCsC8fCQ==,iv:GOvPs2I77yiKh381bcv0kCwT5xbk8Fao6ctPCW2aCHg=,tag:hSjWjaZ+nEJwZyv3sIuq7A==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBN2NvVXJ5TW1mQlhYWnRC
|
||||
UlVLVjFtNk1Wdkk1NzdicXBQVGNtc3JZclNBCjl3UlVTZmFJWGJGam54VytVR0lZ
|
||||
djB3ZjBCOFFNOUFMZGxCcnZiZ3VDMUUKLS0tIFhDSWR3ZXZPOHFzVGNhZGhqdUQv
|
||||
WmF0NlNIeTJCeVV0ZGkxSFJIL3BqNlEKntEpxCdpy7Xd16ZKe6skQYL6hCqYOAIN
|
||||
NtwtQ9oF/8i4i0T7B8kcORB71cu8dPu6w6LtaSC+1qBV5TBSSH5U6Q==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:dcgB6z9BDzaft4do6f1SqjCqmQopc5mM0ckfU/HfE09eOSIRjpoZ+qMzJW8RIwUZNKrjEAwrrAEtrN67GRxL0jOEsw/6xNrWJRFv6KTeE3ECy5qMO4jLCQTeMvZuvfZE7iqwvHCCnjhkqkedEAD4Hwefq0c0EZ0ZxZK3z1tVdHk=,iv:Ue+ZJq9dJl9qbjJ1l8Q24VmsK+KW/uSRFu04xG6/tM8=,tag:MvJ6Ww3h3b/C2PqA/62abA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -28,9 +28,17 @@ resources:
|
||||
- keycloak-admin-credentials.sops.yaml
|
||||
# --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders ---
|
||||
- keycloak-realm-clients.sops.yaml
|
||||
# --- Scoped MinIO S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05) — SOPS-encrypted,
|
||||
# decrypted by the `platform` tier (which already has the decryption stanza). Lands in ns
|
||||
# keycloak; the keycloak-db Cluster (platform-config) reads it by name for WAL archiving. ---
|
||||
# --- PER-ENVIRONMENT S3 credential for the keycloak-db CNPG Barman backup (R-IDP-05), split
|
||||
# 2026-09-07. SOPS-encrypted, decrypted by the `platform` tier (which already has the
|
||||
# decryption stanza). BOTH land in ns keycloak on BOTH clusters; the keycloak-db Cluster
|
||||
# (platform-config) selects ONE by name via
|
||||
# `keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
|
||||
# prod "-prod"). Two objects, so an R2 token rotation cannot silently break dev's MinIO
|
||||
# WAL archiving — and archive_mode=on makes a broken archive a disk-full clock, not a
|
||||
# degraded backup (see postgres-cluster.yaml).
|
||||
# dev -> keycloak-db-backup-credentials (MinIO, s3://cnpg)
|
||||
# prod -> keycloak-db-backup-credentials-prod (Cloudflare R2, s3://k3s-prod-backup/barman)
|
||||
- keycloak-db-backup-credentials.sops.yaml
|
||||
- keycloak-db-backup-credentials-prod.sops.yaml
|
||||
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
||||
- grafana-dashboard.yaml
|
||||
@@ -34,25 +34,43 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# --- Off-cluster DR backup (R-IDP-05) — CNPG-native Barman → object storage ---
|
||||
# Continuous WAL archiving + periodic base backups give proper Postgres PITR (superior to a Velero
|
||||
# PVC snapshot for a live DB). destinationPath is the BUCKET ROOT; CNPG appends serverName (defaults
|
||||
# to the Cluster name), so objects land under s3://cnpg/keycloak-db/. Path-style HTTP MinIO endpoint.
|
||||
# PVC snapshot for a live DB). CNPG appends `serverName` (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/keycloak-db/ — dev
|
||||
# s3://cnpg/keycloak-db/ on the on-Proxmox MinIO, prod s3://k3s-prod-backup/barman/keycloak-db/ on
|
||||
# Cloudflare R2. Both endpointURL and destinationPath are cluster-vars knobs (R-ENV-02),
|
||||
# substituted by the `platform-config` Kustomization (this file sits under platform/keycloak/ but
|
||||
# platform-config is the only tier that lists it as a resource — see its kustomization.yaml).
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret keycloak-db-backup-credentials
|
||||
# (a Pod only reads same-ns Secrets), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000) — prod
|
||||
# MUST point at its own off-cluster target. Substituted by the `platform-config` Kustomization.
|
||||
# (a Pod only reads same-ns Secrets).
|
||||
#
|
||||
# *** archive_mode IS ON, SO A FAILING ARCHIVE COMMAND IS A DISK-FULL CLOCK, NOT A DEGRADED
|
||||
# BACKUP. *** Postgres will not recycle a WAL segment it has not successfully archived, so
|
||||
# whenever these two values are wrong the pgdata PVC fills monotonically until the primary
|
||||
# PANICs. This is not hypothetical: prod ran from bootstrap to 2026-09-07 with
|
||||
# `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT` (barman-cloud-wal-archive exit 4 on every segment)
|
||||
# and had burned ~579 MiB of a 5 GiB PVC on the way to taking SSO — and with it kubectl OIDC,
|
||||
# Grafana, Headlamp, Gitea and forward-auth — down with it. Never leave a placeholder here, and
|
||||
# treat `ContinuousArchiving=False` on this Cluster as an availability incident, not a backup one.
|
||||
backup:
|
||||
retentionPolicy: "30d" # keep 30 days of base backups + WAL (PITR window); prunes older
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
# PER-ENVIRONMENT credential (2026-09-07): dev reads keycloak-db-backup-credentials (MinIO),
|
||||
# prod reads keycloak-db-backup-credentials-prod (Cloudflare R2). Both Secrets are applied to
|
||||
# both clusters by platform/keycloak/kustomization.yaml; ${BACKUP_CREDENTIALS_SUFFIX} (dev ""
|
||||
# | prod "-prod") picks one. Before the split these were ONE shared Secret, so writing prod's
|
||||
# R2 token into it would have left dev signing MinIO requests with an R2 key — a 403 on every
|
||||
# archive command, which under archive_mode=on is the disk-full clock described above, not a
|
||||
# missing backup.
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: keycloak-db-backup-credentials
|
||||
name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: keycloak-db-backup-credentials
|
||||
name: keycloak-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
@@ -1,18 +1,29 @@
|
||||
---
|
||||
# Velero — off-cluster backup/DR of cluster resources + PVC data (ADR-0003, R-STATE-03). LIVE.
|
||||
#
|
||||
# TARGET: the MinIO S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars; dev http://192.168.1.9:9000 on
|
||||
# Proxmox — plain HTTP, path-style), bucket
|
||||
# `velero`. region us-east-1 is a placeholder MinIO ignores (S3 clients still require one).
|
||||
# TARGET: the S3 endpoint ${S3_BACKUP_ENDPOINT} (cluster-vars) at
|
||||
# s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX}, in region ${S3_BACKUP_REGION}. Four knobs,
|
||||
# because the two environments' object stores differ in shape as well as address:
|
||||
# dev http://192.168.1.9:9000 bucket `velero`, NO prefix, region us-east-1 (on-Proxmox MinIO,
|
||||
# plain HTTP, path-style; MinIO ignores the region)
|
||||
# prod https://<acct>.r2.cloudflarestorage.com bucket `k3s-prod-backup`, prefix `velero`,
|
||||
# region `auto` (Cloudflare R2; prod cannot route to that MinIO,
|
||||
# and R2 demands exactly `auto`)
|
||||
# Prod's bucket is SHARED with CNPG/barman (the operator created one bucket), so the `velero` prefix
|
||||
# is what keeps this controller's retention sweep away from the Postgres WAL stream. See the
|
||||
# "OBJECT-STORE TENANCY" block in clusters/base/cluster-vars.yaml.
|
||||
#
|
||||
# WHY THIS SHAPE:
|
||||
# - provider aws + velero-plugin-for-aws (init container) drives the S3-compatible MinIO target.
|
||||
# - File System Backup (kopia) via the node-agent DaemonSet — storage-agnostic PVC backup that does
|
||||
# NOT depend on CSI VolumeSnapshots or a data-mover (the simplest path that still captures the
|
||||
# Longhorn PVC *data*). snapshotsEnabled:false → no VolumeSnapshotLocation is created/needed.
|
||||
# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero) — the
|
||||
# SCOPED MinIO key limited to the velero+cnpg buckets. Kept in sync with the staging source of
|
||||
# truth platform/backup/minio-credentials.sops.yaml and the Proxmox host ansible-vault.
|
||||
# - Credentials: the `cloud` INI key of the SOPS Secret velero-cloud-credentials (ns velero). ONE
|
||||
# Secret serves both environments, and it now carries the Cloudflare R2 API token (written by
|
||||
# platform/backup/set-r2-credentials.sh, 2026-09-07) rather than the scoped MinIO key. That is
|
||||
# safe only because the R2 token is bucket-scoped to k3s-prod-backup, which dev never names:
|
||||
# dev's Velero would fail loudly against its MinIO endpoint with an R2 key, it could not
|
||||
# silently write somewhere unexpected. Rotating either target means re-running that script.
|
||||
#
|
||||
# VOLUME OPT-OUT (per the sizing decision) is implemented with TWO schedules (a single Backup has one
|
||||
# cluster-wide defaultVolumesToFsBackup, so per-namespace volume policy needs two): see `schedules`.
|
||||
@@ -72,14 +83,45 @@ spec:
|
||||
backupStorageLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
bucket: velero
|
||||
# Bucket and prefix are BOTH knobs (cluster-vars), because the two environments' object
|
||||
# stores are shaped differently. Dev's MinIO has a dedicated `velero` bucket and dev's
|
||||
# backups sit at its ROOT, so dev renders bucket=velero with an EMPTY prefix — and the
|
||||
# chart omits `objectStorage.prefix` entirely when it is empty (verified, chart 12.1.0),
|
||||
# leaving dev's live BackupStorageLocation byte-unchanged. Prod's operator created ONE R2
|
||||
# bucket shared with CNPG/barman, so prod renders bucket=k3s-prod-backup with prefix
|
||||
# `velero` — a disjoint top-level prefix from barman's. Getting this wrong is not a
|
||||
# cosmetic error: Velero ADOPTS every backup it finds under its location and TTL-DELETES
|
||||
# it on retention, so a prefix that overlaps another tenant deletes that tenant's data.
|
||||
bucket: ${VELERO_BACKUP_BUCKET}
|
||||
prefix: "${VELERO_BACKUP_PREFIX}"
|
||||
default: true
|
||||
config:
|
||||
region: us-east-1 # placeholder; MinIO ignores it (S3 clients require one)
|
||||
s3Url: ${S3_BACKUP_ENDPOINT} # on-Proxmox MinIO, plain HTTP (cluster-vars)
|
||||
s3ForcePathStyle: "true" # MinIO is path-style
|
||||
# Region is a knob, not a constant: S3 clients require SOME region, MinIO ignores
|
||||
# whatever it is, and Cloudflare R2 requires exactly `auto`. Hardcoding `us-east-1`
|
||||
# was correct-by-accident and broke the moment the target stopped being MinIO.
|
||||
region: ${S3_BACKUP_REGION}
|
||||
s3Url: ${S3_BACKUP_ENDPOINT} # cluster-vars; dev MinIO (http) or prod R2 (https)
|
||||
s3ForcePathStyle: "true" # MinIO is path-style; R2 accepts path-style too
|
||||
# *** REQUIRED FOR CLOUDFLARE R2 — DO NOT DROP THIS LINE. ***
|
||||
# velero-plugin-for-aws v1.14.2 bundles an AWS SDK with flexible checksums ON by
|
||||
# default, so it sends `x-amz-sdk-checksum-algorithm` / `x-amz-checksum-crc32` on every
|
||||
# PUT. R2 REJECTS those headers with HTTP 400 InvalidArgument, which would fail every
|
||||
# backup AND every kopia blob upload. The empty string is the documented opt-out, and
|
||||
# it must be PRESENT-but-empty rather than absent — the plugin branches on the key
|
||||
# existing, and the chart does emit an empty value (verified, chart 12.1.0). Harmless
|
||||
# against MinIO, which ignores it, so it is stated unconditionally rather than as a
|
||||
# fourth knob. If backups start failing with InvalidArgument after a plugin bump, look
|
||||
# here first.
|
||||
checksumAlgorithm: ""
|
||||
credentials:
|
||||
existingSecret: velero-cloud-credentials # SOPS Secret carrying the `cloud` INI key (ns velero)
|
||||
# PER-ENVIRONMENT credential (2026-09-07). dev renders velero-cloud-credentials (the scoped
|
||||
# MinIO key); prod renders velero-cloud-credentials-prod (the Cloudflare R2 token). BOTH
|
||||
# Secrets are applied to BOTH clusters by platform/velero/kustomization.yaml — the suffix
|
||||
# only chooses which one this Deployment mounts. cluster-vars: BACKUP_CREDENTIALS_SUFFIX,
|
||||
# dev "" | prod "-prod". A name that resolves to a MISSING Secret does not degrade the
|
||||
# backup, it wedges the pod in ContainerCreating (the volume never mounts), which is why
|
||||
# the base default is the empty suffix — the unsuffixed Secret always exists.
|
||||
existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX} # SOPS Secret carrying the `cloud` INI key (ns velero)
|
||||
# --- Daily backups. TWO schedules implement the opt-out volume policy: capture every PVC's data
|
||||
# by default (opt-out), but skip the two reconstructable heavy volumes' DATA while still
|
||||
# capturing their Kubernetes resources. ---
|
||||
|
||||
@@ -5,7 +5,19 @@ resources:
|
||||
- namespace.yaml
|
||||
- helmrepository.yaml
|
||||
- helmrelease.yaml
|
||||
# Scoped MinIO S3 credential (the `cloud` INI key the AWS plugin reads) — SOPS-encrypted, decrypted
|
||||
# per-env by the `velero` Flux Kustomization (clusters/dev). Kept in sync with the staging source of
|
||||
# truth platform/backup/minio-credentials.sops.yaml + the Proxmox host ansible-vault.
|
||||
# --- PER-ENVIRONMENT `cloud` INI credential (split 2026-09-07) -----------------------------------
|
||||
# BOTH Secrets are applied to BOTH clusters; the HelmRelease picks ONE by name via
|
||||
# `existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
|
||||
# prod "-prod"). They are separate objects so rotating one environment's key can never touch the
|
||||
# other's — the failure mode that made the split necessary (a single shared Secret carrying R2 keys
|
||||
# would have pointed dev's Velero at MinIO with credentials MinIO has never issued).
|
||||
#
|
||||
# dev -> velero-cloud-credentials : the scoped MinIO key for http://192.168.1.9:9000.
|
||||
# Kept in sync with the staging source of truth
|
||||
# platform/backup/minio-credentials.sops.yaml + the
|
||||
# Proxmox host ansible-vault.
|
||||
# prod -> velero-cloud-credentials-prod : the Cloudflare R2 token for k3s-prod-backup.
|
||||
#
|
||||
# Both are decrypted by the `velero` Flux Kustomization's SOPS stanza (clusters/dev + clusters/prod).
|
||||
- velero-cloud.sops.yaml
|
||||
- velero-cloud-prod.sops.yaml
|
||||
@@ -2,8 +2,13 @@
|
||||
# doc pointer, kept for parity with the other components' layout).
|
||||
#
|
||||
# LIVE wiring (see helmrelease.yaml):
|
||||
# - provider aws + velero-plugin-for-aws (init container) → MinIO S3 ${S3_BACKUP_ENDPOINT} (dev: http://192.168.1.9:9000)
|
||||
# (path-style, bucket `velero`, region us-east-1 placeholder).
|
||||
# - provider aws + velero-plugin-for-aws (init container) → S3 ${S3_BACKUP_ENDPOINT}, path-style,
|
||||
# at s3://${VELERO_BACKUP_BUCKET}/${VELERO_BACKUP_PREFIX} in region ${S3_BACKUP_REGION}. Four
|
||||
# cluster-vars knobs: dev = the on-Proxmox MinIO (http://192.168.1.9:9000, bucket `velero`, no
|
||||
# prefix, us-east-1); prod = Cloudflare R2 (bucket `k3s-prod-backup`, prefix `velero`, `auto`),
|
||||
# sharing one bucket with CNPG/barman under a disjoint prefix.
|
||||
# - config.checksumAlgorithm: "" — MANDATORY for R2, which rejects the AWS SDK's
|
||||
# x-amz-checksum-* headers with HTTP 400 InvalidArgument. Harmless against MinIO.
|
||||
# - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover).
|
||||
# - snapshotsEnabled:false → no VolumeSnapshotLocation.
|
||||
# - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero).
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: velero-cloud-credentials-prod
|
||||
namespace: velero
|
||||
type: Opaque
|
||||
stringData:
|
||||
cloud: ENC[AES256_GCM,data:Kq+hfYMq/NZJJqbK2zqwW97XKNW0LbT2Un20yc0B6evPXqp+aC0vV0odwos4XwV0puPIsEhU7IeD+JidVjU1eKv6Qax0rnoj90FVWihxWyLMNAMC+b8PxBnPhWdk6ZlrMKs13roGZwYh4bbrstnJamMqpD4oZpthEl/NEQNWVAb5ogJHsmPNiQOtHhtBWkTubfgPIw==,iv:gn8/4/XpUZ8c6+nlGe6SAODUyrhDqskQ+uVRx2YR3Qc=,tag:aTk3cRJTA0WjdIi1B9S0hg==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwWmV5UGpmdnFKbEhjeHlZ
|
||||
QUZCWm13ZlpVYU1URFErSmM5MEJ2WTRxZVFvCkZ5cEdJaXJ1NDNSaEF5d2lxdDQw
|
||||
NGpzQm5ESGt6cUlIZVdHYTBjRGVwaVUKLS0tIHlpWkVacXdIZndBOXdlVk1LZzlN
|
||||
NkZyRm1abk9CNlZRUERtbFM2RjVtS1UKWYtI7QZiTjPIHaLOOX9WvDK1cHwP2WfW
|
||||
26WdNW90tTmt3cYdaFGnkpAVCMS3BRtdInirulfoGRgvffjWNBDVuQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:4v4L29KoIhQI94TU/s0u3HKR5C6mMc6mRWWsZzwIfqkl4fayRvLG67tbJ88vVkb6AXgqHoKXKlhbpn6hTJPBrL72f8+fii3CeLloVJyW2AuzHaHCSi92U04ywoSc45hClZrOLvUFf5KbtawPPKfP53oaNq+AVEZ1nqKd8ssUyBA=,iv:WVrdja00ptK/MmCC/Chu+Gq4ZYpBvpFRWW3lSo7Mon0=,tag:GzFfpkBlHu6Nc8ZdnCBuKQ==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
Reference in new issue
Block a user