merge: main (velero R2 + arr config PVC) into feat/vpn-egress-prod
This commit is contained in:
jakman85 committed
commit
9b97e073e2
12 files changed
+159
-11
No files matched your search
@@ -74,7 +74,7 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
containers:
|
||||
- name: jackett
|
||||
@@ -97,6 +97,7 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -266,7 +267,7 @@ metadata:
|
||||
app.kubernetes.io/name: jackett
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
containers:
|
||||
- name: qbittorrent
|
||||
@@ -97,6 +97,7 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -266,7 +267,7 @@ metadata:
|
||||
app.kubernetes.io/name: qbittorrent
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -118,7 +118,7 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
containers:
|
||||
- name: radarr
|
||||
@@ -150,6 +150,7 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -346,7 +347,7 @@ metadata:
|
||||
app.kubernetes.io/name: radarr
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -118,7 +118,7 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
containers:
|
||||
- name: sonarr
|
||||
@@ -150,6 +150,7 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -346,7 +347,7 @@ metadata:
|
||||
app.kubernetes.io/name: sonarr
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -42,6 +42,15 @@ auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in an authorized group (R-IDP-06)
|
||||
# platform-admins is a SUPERSET, not a widening. --allowed-group is repeatable and OR-ed, so
|
||||
# listing both means "media consumers OR platform operators". Without it the media UIs are
|
||||
# reachable by NOBODY the moment media-users is empty: on prod the realm's only user is the
|
||||
# operator, who is in platform-admins only, so jackett/qbittorrent/radarr/sonarr answered an
|
||||
# immediate 403 at the oauth2-proxy callback (no login prompt, because the Keycloak SSO session
|
||||
# already existed) while pihole/crafty — which already gate on platform-admins — worked.
|
||||
# This grants no real privilege: platform-admins already maps to cluster-admin via
|
||||
# k8s-oidc-rbac.yaml (oidc:platform-admins), so a member can already exec into these pods.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=0.22.0" # stay on the Jackett 0.22.x stable line
|
||||
|
||||
@@ -57,6 +57,11 @@ auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in an authorized group (R-IDP-06)
|
||||
# See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
|
||||
# platform-admins here is a superset, not a widening (that group already carries cluster-admin
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=5.0.0 <6.0.0" # stay on the qBittorrent 5.x stable line
|
||||
|
||||
@@ -107,6 +107,11 @@ auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in the media group (R-IDP-06)
|
||||
# See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
|
||||
# platform-admins here is a superset, not a widening (that group already carries cluster-admin
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=5.0.0 <6.0.0" # stay on the Radarr v5 stable line
|
||||
|
||||
@@ -107,6 +107,11 @@ auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in the media group (R-IDP-06)
|
||||
# See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
|
||||
# platform-admins here is a superset, not a widening (that group already carries cluster-admin
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=4.0.0 <5.0.0" # stay on the Sonarr v4 stable line
|
||||
|
||||
@@ -65,11 +65,30 @@ spec:
|
||||
# acceptable: Velero's CRDs are stable, and this is far less fragile than pinning a shell-bearing
|
||||
# kubectl mirror (rancher/kubectl is distroless → no /bin/sh for the hook's copy-binary step). ---
|
||||
upgradeCRDs: false
|
||||
# --- AWS object-store plugin (BSL provider=aws → MinIO S3-compatible), loaded as an init container.
|
||||
# Image minor MUST track the Velero minor (chart 12.x = Velero 1.18 → plugin v1.14.x). ---
|
||||
# --- AWS object-store plugin (BSL provider=aws → the S3-compatible target), loaded as an init
|
||||
# container. Image minor MUST track the Velero minor (chart 12.x = Velero 1.18 → plugin v1.14.x).
|
||||
#
|
||||
# *** WHY AN -rc TAG, AND WHY IT CANNOT BE v1.14.2 (measured on prod, 2026-09-12). ***
|
||||
# v1.14.2 sets the S3 PutObject `Tagging` field UNCONDITIONALLY:
|
||||
# Tagging: aws.String(o.tagging) // o.tagging == "" when config.tagging is unset
|
||||
# so the SDK emits `x-amz-tagging: ` (empty) on EVERY PutObject. Cloudflare R2 does not implement
|
||||
# object tagging AT ALL — the S3 compatibility matrix lists PutObject's `x-amz-tagging`, plus
|
||||
# Put/Get/DeleteObjectTagging, as unimplemented — so R2 answers HTTP 501:
|
||||
# NotImplemented: Header 'x-amz-tagging' with value '' not implemented
|
||||
# Observed live on prod twice, on the write of `velero/backups/<name>/velero-backup.json`. The
|
||||
# Backup lands its kopia volume DATA (the node-agent's kopia client sends no such header) and then
|
||||
# goes phase=Failed with NO metadata object — an unrestorable backup. There is NO config that
|
||||
# avoids this: setting `config.tagging` to a real value only sends a tag set R2 also rejects.
|
||||
# The fix is upstream PR #309 ("Only set PutObject Tagging when tags are configured"), which makes
|
||||
# the field conditional (`if o.tagging != ""`). It landed on release-1.14 AFTER v1.14.2 was cut, so
|
||||
# the earliest build carrying it is v1.14.3-rc.2. There is no GA release with the fix yet.
|
||||
# EXIT CONDITION: move to v1.14.3 the moment it ships GA (Renovate will offer it). Do NOT
|
||||
# "clean this up" back to v1.14.2 — that silently breaks every prod backup's metadata.
|
||||
# Harmless on dev's MinIO, which ignores the header either way, so it is stated unconditionally
|
||||
# rather than becoming a fifth per-environment knob. ---
|
||||
initContainers:
|
||||
- name: velero-plugin-for-aws
|
||||
image: velero/velero-plugin-for-aws:v1.14.2
|
||||
image: velero/velero-plugin-for-aws:v1.14.3-rc.2
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- mountPath: /target
|
||||
@@ -128,6 +147,23 @@ spec:
|
||||
schedules:
|
||||
# (1) The whole cluster EXCEPT monitoring+unifi: all resources + all pod-volume data via kopia.
|
||||
# defaultVolumesToFsBackup:true is the OPT-OUT model — nothing with real data is forgotten.
|
||||
#
|
||||
# THE BULK TIER IS OPTED OUT BY VOLUME TYPE, NOT BY NAMESPACE (added 2026-09-12). Opt-out
|
||||
# + the external NFS bulk tier is a 8.6 TiB collision: prod carries four static NFS PVs
|
||||
# (plex-media, sonarr-media, radarr-media — all three bound to the SAME export
|
||||
# /mnt/Storage/plex, measured 8.32 TiB — plus nextcloud-legacy-archive at 270 GiB), and
|
||||
# their namespaces ALSO hold real Longhorn config/data PVCs that MUST be backed up. So
|
||||
# namespace granularity cannot express this: moving plex/sonarr/radarr/nextcloud into
|
||||
# schedule (2) would drop their real config PVCs, and leaving them here uploaded the media
|
||||
# library to R2 three times over (kopia dedups the STORED copy, not the three NFS reads).
|
||||
# `resourcePolicy` below therefore skips volumes whose source is NFS, whatever they are
|
||||
# called and whoever adds the next one — the bulk tier IS the NFS export (R-STATE-04), so
|
||||
# the rule tests the property rather than a name list that a future PV would slip past.
|
||||
# Media on that ZFS pool has its own protection and is reconstructable; it is deliberately
|
||||
# NOT cluster-DR scope. See platform/velero/volume-policies.yaml.
|
||||
# RESIDUAL GAP, stated rather than hidden: the rule matches NFS only. A future bulk volume
|
||||
# on some OTHER out-of-cluster source (iSCSI, CSI-attached object gateway) would be picked
|
||||
# up by opt-out again and needs its own condition added there.
|
||||
daily:
|
||||
disabled: false
|
||||
schedule: "${VELERO_BACKUP_SCHEDULE}"
|
||||
@@ -137,6 +173,9 @@ spec:
|
||||
storageLocation: default
|
||||
snapshotVolumes: false
|
||||
defaultVolumesToFsBackup: true
|
||||
resourcePolicy:
|
||||
kind: configmap
|
||||
name: velero-volume-policies
|
||||
excludedNamespaces:
|
||||
- monitoring # volume DATA excluded here; resources captured by schedule (2)
|
||||
- unifi
|
||||
@@ -159,6 +198,26 @@ spec:
|
||||
includedNamespaces:
|
||||
- monitoring
|
||||
- unifi
|
||||
# --- SCRAPE THE BACKUP CONTROLLER, or its failures are silent (added 2026-09-12) ------------
|
||||
# platform/monitoring/alerts.yaml already ships VeleroBackupFailure (critical),
|
||||
# VeleroBackupPartialFailure and VeleroNoRecentBackup — and NONE of them could ever fire,
|
||||
# because the chart defaults `metrics.serviceMonitor.enabled: false` and no ServiceMonitor
|
||||
# existed in ns velero (verified live on prod, 2026-09-12: `get servicemonitor -n velero` →
|
||||
# no resources). The series velero_backup_failure_total / velero_backup_last_successful_timestamp
|
||||
# were never scraped, so a nightly backup could fail — or stop running — indefinitely with no
|
||||
# alert. That is the precise failure mode R-STATE-03 exists to prevent, so the scrape is part of
|
||||
# the backup path, not an observability nice-to-have.
|
||||
# The chart gates these on BOTH `enabled` and capability detection of monitoring.coreos.com/v1
|
||||
# (verified: `helm template` emits neither object without --api-versions, and emits both with
|
||||
# it), so `enabled: true` is required and sufficient — helm-controller supplies real capabilities
|
||||
# from the API server. No additionalLabels needed: platform/monitoring sets
|
||||
# serviceMonitorSelectorNilUsesHelmValues / podMonitorSelectorNilUsesHelmValues false, so
|
||||
# Prometheus selects every ServiceMonitor/PodMonitor regardless of labels.
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: true # velero server: backup/restore success+failure counters
|
||||
nodeAgentPodMonitor:
|
||||
enabled: true # node-agent: kopia FSB (PodVolumeBackup) progress + failures
|
||||
# Modest home-dev footprint (mirror the other platform components).
|
||||
resources:
|
||||
requests:
|
||||
|
||||
@@ -5,6 +5,9 @@ resources:
|
||||
- namespace.yaml
|
||||
- helmrepository.yaml
|
||||
- helmrelease.yaml
|
||||
# Volume policies for schedule (1)'s opt-out FSB: skip the external NFS bulk tier by SOURCE TYPE.
|
||||
# Must exist before the Schedule references it by name (same Kustomization, so one apply).
|
||||
- volume-policies.yaml
|
||||
# --- PER-ENVIRONMENT `cloud` INI credential (split 2026-09-07) -----------------------------------
|
||||
# BOTH Secrets are applied to BOTH clusters; the HelmRelease picks ONE by name via
|
||||
# `existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "",
|
||||
|
||||
@@ -9,6 +9,14 @@
|
||||
# sharing one bucket with CNPG/barman under a disjoint prefix.
|
||||
# - config.checksumAlgorithm: "" — MANDATORY for R2, which rejects the AWS SDK's
|
||||
# x-amz-checksum-* headers with HTTP 400 InvalidArgument. Harmless against MinIO.
|
||||
# - velero-plugin-for-aws PINNED TO v1.14.3-rc.2 — also MANDATORY for R2. v1.14.2 sends an empty
|
||||
# `x-amz-tagging` header on every PutObject, which R2 answers with HTTP 501 NotImplemented, so
|
||||
# every Backup lands its kopia volume data and then FAILS writing velero-backup.json. Upstream
|
||||
# PR #309 makes the field conditional; rc.2 is the earliest build carrying it. Move to v1.14.3
|
||||
# when it ships GA. Measured on prod 2026-09-12.
|
||||
# - resourcePolicy on the FSB schedule -> ConfigMap velero-volume-policies (volume-policies.yaml):
|
||||
# skips volumes whose source is NFS, so the 8.6 TiB external bulk tier is not uploaded nightly
|
||||
# while the same namespaces' real Longhorn PVCs still are. Proven on prod 2026-09-12.
|
||||
# - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover).
|
||||
# - snapshotsEnabled:false → no VolumeSnapshotLocation.
|
||||
# - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero).
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
# Velero VOLUME POLICIES — the opt-out escape hatch for the external bulk tier (R-STATE-04).
|
||||
#
|
||||
# WHY THIS EXISTS. platform/velero/helmrelease.yaml schedule (1) runs
|
||||
# `defaultVolumesToFsBackup: true` — the OPT-OUT model, so no volume with real data is ever
|
||||
# forgotten. That model collides with the NFS bulk tier prod grew on 2026-09-11:
|
||||
#
|
||||
# plex-media 9Ti claim -> 192.168.2.102:/mnt/Storage/plex (8.32 TiB measured)
|
||||
# sonarr-media 9Ti claim -> the SAME export
|
||||
# radarr-media 9Ti claim -> the SAME export
|
||||
# nextcloud-legacy-archive 270Gi -> 192.168.2.102:/mnt/Storage/nextcloud
|
||||
#
|
||||
# Left to opt-out, one nightly run reads that library THREE TIMES over NFS and pushes ~8.6 TiB at
|
||||
# Cloudflare R2, every night, held for ${VELERO_RETENTION} (prod: 720h). kopia dedups the STORED
|
||||
# bytes, not the reads. That is not DR; it is a self-inflicted outage plus a recurring bill.
|
||||
#
|
||||
# WHY NOT namespace exclusion. plex / sonarr / radarr / nextcloud each ALSO hold a real Longhorn
|
||||
# PVC (plex-config 8Gi, sonarr-config 5Gi, radarr-config 5Gi, nextcloud-data 20Gi) whose loss is
|
||||
# unrecoverable. Excluding those namespaces — the shape schedule (2) uses for monitoring+unifi —
|
||||
# would silently drop exactly the data that matters. The opt-out has to be per VOLUME.
|
||||
#
|
||||
# WHY NOT the per-pod `backup.velero.io/backup-volumes-excludes` annotation. It works, but it fails
|
||||
# OPEN: the next NFS PV someone mounts is backed up in full until a human remembers to annotate its
|
||||
# pod, and nothing tells them. A volume policy tests the PROPERTY that makes a volume bulk-tier —
|
||||
# its source is the out-of-cluster NFS export — so it covers volumes that do not exist yet.
|
||||
#
|
||||
# `nfs: {}` is the documented "match any NFS volume source" condition (server/path may be given to
|
||||
# narrow it; deliberately not, so a second export is covered too). First matching rule wins.
|
||||
#
|
||||
# PROVEN, not assumed (prod, 2026-09-12, velero 1.18.1 / chart 12.1.0): a Backup of ns nextcloud
|
||||
# with defaultVolumesToFsBackup:true and this policy produced exactly ONE PodVolumeBackup —
|
||||
# `data` (nextcloud-data, Longhorn, 875,417,102 bytes, Completed). The NFS `legacy-archive` volume
|
||||
# produced NO PodVolumeBackup at all.
|
||||
#
|
||||
# Referenced by name from schedule (1)'s template (`resourcePolicy: {kind: configmap, name: ...}`).
|
||||
# Applied to BOTH environments: dev has no NFS PVs today, so the rule is a no-op there, and it is
|
||||
# already correct if dev ever grows one.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: velero-volume-policies
|
||||
namespace: velero
|
||||
data:
|
||||
policies.yaml: |
|
||||
version: v1
|
||||
volumePolicies:
|
||||
- conditions:
|
||||
nfs: {}
|
||||
action:
|
||||
type: skip
|
||||
Reference in new issue
Block a user