diff --git a/apps/wave-0/jackett.yaml b/apps/wave-0/jackett.yaml index 7af66d6..0f21bec 100644 --- a/apps/wave-0/jackett.yaml +++ b/apps/wave-0/jackett.yaml @@ -74,7 +74,7 @@ spec: homelab.keithjackson.dev/auth-mode: forward-auth annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: containers: - name: jackett @@ -97,6 +97,7 @@ spec: - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users + - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback - --cookie-domain=.${BASE_DOMAIN} @@ -266,7 +267,7 @@ metadata: app.kubernetes.io/name: jackett annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: parentRefs: - name: shared diff --git a/apps/wave-0/qbittorrent.yaml b/apps/wave-0/qbittorrent.yaml index fb17766..f956793 100644 --- a/apps/wave-0/qbittorrent.yaml +++ b/apps/wave-0/qbittorrent.yaml @@ -74,7 +74,7 @@ spec: homelab.keithjackson.dev/auth-mode: forward-auth annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: containers: - name: qbittorrent @@ -97,6 +97,7 @@ spec: - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users + - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback - --cookie-domain=.${BASE_DOMAIN} @@ -266,7 +267,7 @@ metadata: app.kubernetes.io/name: qbittorrent annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: parentRefs: - name: shared diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index cb406c2..dd83afd 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -118,7 +118,7 @@ spec: homelab.keithjackson.dev/auth-mode: forward-auth annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: containers: - name: radarr @@ -150,6 +150,7 @@ spec: - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users + - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback - --cookie-domain=.${BASE_DOMAIN} @@ -346,7 +347,7 @@ metadata: app.kubernetes.io/name: radarr annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: parentRefs: - name: shared diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index 6cfd516..e5fa7ab 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -118,7 +118,7 @@ spec: homelab.keithjackson.dev/auth-mode: forward-auth annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: containers: - name: sonarr @@ -150,6 +150,7 @@ spec: - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users + - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback - --cookie-domain=.${BASE_DOMAIN} @@ -346,7 +347,7 @@ metadata: app.kubernetes.io/name: sonarr annotations: homelab.keithjackson.dev/auth-mode: forward-auth - homelab.keithjackson.dev/auth-groups: media-users + homelab.keithjackson.dev/auth-groups: media-users,platform-admins spec: parentRefs: - name: shared diff --git a/catalog/jackett.yaml b/catalog/jackett.yaml index 6c4d742..cc0d250 100644 --- a/catalog/jackett.yaml +++ b/catalog/jackett.yaml @@ -42,6 +42,15 @@ auth: mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02) groups: - media-users # authenticated AND in an authorized group (R-IDP-06) + # platform-admins is a SUPERSET, not a widening. --allowed-group is repeatable and OR-ed, so + # listing both means "media consumers OR platform operators". Without it the media UIs are + # reachable by NOBODY the moment media-users is empty: on prod the realm's only user is the + # operator, who is in platform-admins only, so jackett/qbittorrent/radarr/sonarr answered an + # immediate 403 at the oauth2-proxy callback (no login prompt, because the Keycloak SSO session + # already existed) while pihole/crafty — which already gate on platform-admins — worked. + # This grants no real privilege: platform-admins already maps to cluster-admin via + # k8s-oidc-rbac.yaml (oidc:platform-admins), so a member can already exec into these pods. + - platform-admins # operators of these services must not be locked out of them update: channel: stable # R-UPD-01 range: ">=0.22.0" # stay on the Jackett 0.22.x stable line diff --git a/catalog/qbittorrent.yaml b/catalog/qbittorrent.yaml index 2efc778..f0fe0bc 100644 --- a/catalog/qbittorrent.yaml +++ b/catalog/qbittorrent.yaml @@ -57,6 +57,11 @@ auth: mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02) groups: - media-users # authenticated AND in an authorized group (R-IDP-06) + # See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so + # platform-admins here is a superset, not a widening (that group already carries cluster-admin + # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an + # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12. + - platform-admins # operators of these services must not be locked out of them update: channel: stable # R-UPD-01 range: ">=5.0.0 <6.0.0" # stay on the qBittorrent 5.x stable line diff --git a/catalog/radarr.yaml b/catalog/radarr.yaml index bf40ff5..ce90d7b 100644 --- a/catalog/radarr.yaml +++ b/catalog/radarr.yaml @@ -107,6 +107,11 @@ auth: mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02) groups: - media-users # authenticated AND in the media group (R-IDP-06) + # See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so + # platform-admins here is a superset, not a widening (that group already carries cluster-admin + # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an + # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12. + - platform-admins # operators of these services must not be locked out of them update: channel: stable # R-UPD-01 range: ">=5.0.0 <6.0.0" # stay on the Radarr v5 stable line diff --git a/catalog/sonarr.yaml b/catalog/sonarr.yaml index a164426..a91f2fb 100644 --- a/catalog/sonarr.yaml +++ b/catalog/sonarr.yaml @@ -107,6 +107,11 @@ auth: mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02) groups: - media-users # authenticated AND in the media group (R-IDP-06) + # See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so + # platform-admins here is a superset, not a widening (that group already carries cluster-admin + # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an + # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12. + - platform-admins # operators of these services must not be locked out of them update: channel: stable # R-UPD-01 range: ">=4.0.0 <5.0.0" # stay on the Sonarr v4 stable line diff --git a/platform/velero/helmrelease.yaml b/platform/velero/helmrelease.yaml index f41a0da..27a794b 100644 --- a/platform/velero/helmrelease.yaml +++ b/platform/velero/helmrelease.yaml @@ -65,11 +65,30 @@ spec: # acceptable: Velero's CRDs are stable, and this is far less fragile than pinning a shell-bearing # kubectl mirror (rancher/kubectl is distroless → no /bin/sh for the hook's copy-binary step). --- upgradeCRDs: false - # --- AWS object-store plugin (BSL provider=aws → MinIO S3-compatible), loaded as an init container. - # Image minor MUST track the Velero minor (chart 12.x = Velero 1.18 → plugin v1.14.x). --- + # --- AWS object-store plugin (BSL provider=aws → the S3-compatible target), loaded as an init + # container. Image minor MUST track the Velero minor (chart 12.x = Velero 1.18 → plugin v1.14.x). + # + # *** WHY AN -rc TAG, AND WHY IT CANNOT BE v1.14.2 (measured on prod, 2026-09-12). *** + # v1.14.2 sets the S3 PutObject `Tagging` field UNCONDITIONALLY: + # Tagging: aws.String(o.tagging) // o.tagging == "" when config.tagging is unset + # so the SDK emits `x-amz-tagging: ` (empty) on EVERY PutObject. Cloudflare R2 does not implement + # object tagging AT ALL — the S3 compatibility matrix lists PutObject's `x-amz-tagging`, plus + # Put/Get/DeleteObjectTagging, as unimplemented — so R2 answers HTTP 501: + # NotImplemented: Header 'x-amz-tagging' with value '' not implemented + # Observed live on prod twice, on the write of `velero/backups//velero-backup.json`. The + # Backup lands its kopia volume DATA (the node-agent's kopia client sends no such header) and then + # goes phase=Failed with NO metadata object — an unrestorable backup. There is NO config that + # avoids this: setting `config.tagging` to a real value only sends a tag set R2 also rejects. + # The fix is upstream PR #309 ("Only set PutObject Tagging when tags are configured"), which makes + # the field conditional (`if o.tagging != ""`). It landed on release-1.14 AFTER v1.14.2 was cut, so + # the earliest build carrying it is v1.14.3-rc.2. There is no GA release with the fix yet. + # EXIT CONDITION: move to v1.14.3 the moment it ships GA (Renovate will offer it). Do NOT + # "clean this up" back to v1.14.2 — that silently breaks every prod backup's metadata. + # Harmless on dev's MinIO, which ignores the header either way, so it is stated unconditionally + # rather than becoming a fifth per-environment knob. --- initContainers: - name: velero-plugin-for-aws - image: velero/velero-plugin-for-aws:v1.14.2 + image: velero/velero-plugin-for-aws:v1.14.3-rc.2 imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /target @@ -128,6 +147,23 @@ spec: schedules: # (1) The whole cluster EXCEPT monitoring+unifi: all resources + all pod-volume data via kopia. # defaultVolumesToFsBackup:true is the OPT-OUT model — nothing with real data is forgotten. + # + # THE BULK TIER IS OPTED OUT BY VOLUME TYPE, NOT BY NAMESPACE (added 2026-09-12). Opt-out + # + the external NFS bulk tier is a 8.6 TiB collision: prod carries four static NFS PVs + # (plex-media, sonarr-media, radarr-media — all three bound to the SAME export + # /mnt/Storage/plex, measured 8.32 TiB — plus nextcloud-legacy-archive at 270 GiB), and + # their namespaces ALSO hold real Longhorn config/data PVCs that MUST be backed up. So + # namespace granularity cannot express this: moving plex/sonarr/radarr/nextcloud into + # schedule (2) would drop their real config PVCs, and leaving them here uploaded the media + # library to R2 three times over (kopia dedups the STORED copy, not the three NFS reads). + # `resourcePolicy` below therefore skips volumes whose source is NFS, whatever they are + # called and whoever adds the next one — the bulk tier IS the NFS export (R-STATE-04), so + # the rule tests the property rather than a name list that a future PV would slip past. + # Media on that ZFS pool has its own protection and is reconstructable; it is deliberately + # NOT cluster-DR scope. See platform/velero/volume-policies.yaml. + # RESIDUAL GAP, stated rather than hidden: the rule matches NFS only. A future bulk volume + # on some OTHER out-of-cluster source (iSCSI, CSI-attached object gateway) would be picked + # up by opt-out again and needs its own condition added there. daily: disabled: false schedule: "${VELERO_BACKUP_SCHEDULE}" @@ -137,6 +173,9 @@ spec: storageLocation: default snapshotVolumes: false defaultVolumesToFsBackup: true + resourcePolicy: + kind: configmap + name: velero-volume-policies excludedNamespaces: - monitoring # volume DATA excluded here; resources captured by schedule (2) - unifi @@ -159,6 +198,26 @@ spec: includedNamespaces: - monitoring - unifi + # --- SCRAPE THE BACKUP CONTROLLER, or its failures are silent (added 2026-09-12) ------------ + # platform/monitoring/alerts.yaml already ships VeleroBackupFailure (critical), + # VeleroBackupPartialFailure and VeleroNoRecentBackup — and NONE of them could ever fire, + # because the chart defaults `metrics.serviceMonitor.enabled: false` and no ServiceMonitor + # existed in ns velero (verified live on prod, 2026-09-12: `get servicemonitor -n velero` → + # no resources). The series velero_backup_failure_total / velero_backup_last_successful_timestamp + # were never scraped, so a nightly backup could fail — or stop running — indefinitely with no + # alert. That is the precise failure mode R-STATE-03 exists to prevent, so the scrape is part of + # the backup path, not an observability nice-to-have. + # The chart gates these on BOTH `enabled` and capability detection of monitoring.coreos.com/v1 + # (verified: `helm template` emits neither object without --api-versions, and emits both with + # it), so `enabled: true` is required and sufficient — helm-controller supplies real capabilities + # from the API server. No additionalLabels needed: platform/monitoring sets + # serviceMonitorSelectorNilUsesHelmValues / podMonitorSelectorNilUsesHelmValues false, so + # Prometheus selects every ServiceMonitor/PodMonitor regardless of labels. + metrics: + serviceMonitor: + enabled: true # velero server: backup/restore success+failure counters + nodeAgentPodMonitor: + enabled: true # node-agent: kopia FSB (PodVolumeBackup) progress + failures # Modest home-dev footprint (mirror the other platform components). resources: requests: diff --git a/platform/velero/kustomization.yaml b/platform/velero/kustomization.yaml index feacb73..b812ac7 100644 --- a/platform/velero/kustomization.yaml +++ b/platform/velero/kustomization.yaml @@ -5,6 +5,9 @@ resources: - namespace.yaml - helmrepository.yaml - helmrelease.yaml + # Volume policies for schedule (1)'s opt-out FSB: skip the external NFS bulk tier by SOURCE TYPE. + # Must exist before the Schedule references it by name (same Kustomization, so one apply). + - volume-policies.yaml # --- PER-ENVIRONMENT `cloud` INI credential (split 2026-09-07) ----------------------------------- # BOTH Secrets are applied to BOTH clusters; the HelmRelease picks ONE by name via # `existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}` (cluster-vars: dev "", diff --git a/platform/velero/values.yaml b/platform/velero/values.yaml index e73db5e..d6de81b 100644 --- a/platform/velero/values.yaml +++ b/platform/velero/values.yaml @@ -9,6 +9,14 @@ # sharing one bucket with CNPG/barman under a disjoint prefix. # - config.checksumAlgorithm: "" — MANDATORY for R2, which rejects the AWS SDK's # x-amz-checksum-* headers with HTTP 400 InvalidArgument. Harmless against MinIO. +# - velero-plugin-for-aws PINNED TO v1.14.3-rc.2 — also MANDATORY for R2. v1.14.2 sends an empty +# `x-amz-tagging` header on every PutObject, which R2 answers with HTTP 501 NotImplemented, so +# every Backup lands its kopia volume data and then FAILS writing velero-backup.json. Upstream +# PR #309 makes the field conditional; rc.2 is the earliest build carrying it. Move to v1.14.3 +# when it ships GA. Measured on prod 2026-09-12. +# - resourcePolicy on the FSB schedule -> ConfigMap velero-volume-policies (volume-policies.yaml): +# skips volumes whose source is NFS, so the 8.6 TiB external bulk tier is not uploaded nightly +# while the same namespaces' real Longhorn PVCs still are. Proven on prod 2026-09-12. # - deployNodeAgent:true + uploaderType:kopia → File System Backup of PVC data (no CSI data-mover). # - snapshotsEnabled:false → no VolumeSnapshotLocation. # - credentials.existingSecret: velero-cloud-credentials (SOPS `cloud` INI key, ns velero). diff --git a/platform/velero/volume-policies.yaml b/platform/velero/volume-policies.yaml new file mode 100644 index 0000000..1848efc --- /dev/null +++ b/platform/velero/volume-policies.yaml @@ -0,0 +1,50 @@ +--- +# Velero VOLUME POLICIES — the opt-out escape hatch for the external bulk tier (R-STATE-04). +# +# WHY THIS EXISTS. platform/velero/helmrelease.yaml schedule (1) runs +# `defaultVolumesToFsBackup: true` — the OPT-OUT model, so no volume with real data is ever +# forgotten. That model collides with the NFS bulk tier prod grew on 2026-09-11: +# +# plex-media 9Ti claim -> 192.168.2.102:/mnt/Storage/plex (8.32 TiB measured) +# sonarr-media 9Ti claim -> the SAME export +# radarr-media 9Ti claim -> the SAME export +# nextcloud-legacy-archive 270Gi -> 192.168.2.102:/mnt/Storage/nextcloud +# +# Left to opt-out, one nightly run reads that library THREE TIMES over NFS and pushes ~8.6 TiB at +# Cloudflare R2, every night, held for ${VELERO_RETENTION} (prod: 720h). kopia dedups the STORED +# bytes, not the reads. That is not DR; it is a self-inflicted outage plus a recurring bill. +# +# WHY NOT namespace exclusion. plex / sonarr / radarr / nextcloud each ALSO hold a real Longhorn +# PVC (plex-config 8Gi, sonarr-config 5Gi, radarr-config 5Gi, nextcloud-data 20Gi) whose loss is +# unrecoverable. Excluding those namespaces — the shape schedule (2) uses for monitoring+unifi — +# would silently drop exactly the data that matters. The opt-out has to be per VOLUME. +# +# WHY NOT the per-pod `backup.velero.io/backup-volumes-excludes` annotation. It works, but it fails +# OPEN: the next NFS PV someone mounts is backed up in full until a human remembers to annotate its +# pod, and nothing tells them. A volume policy tests the PROPERTY that makes a volume bulk-tier — +# its source is the out-of-cluster NFS export — so it covers volumes that do not exist yet. +# +# `nfs: {}` is the documented "match any NFS volume source" condition (server/path may be given to +# narrow it; deliberately not, so a second export is covered too). First matching rule wins. +# +# PROVEN, not assumed (prod, 2026-09-12, velero 1.18.1 / chart 12.1.0): a Backup of ns nextcloud +# with defaultVolumesToFsBackup:true and this policy produced exactly ONE PodVolumeBackup — +# `data` (nextcloud-data, Longhorn, 875,417,102 bytes, Completed). The NFS `legacy-archive` volume +# produced NO PodVolumeBackup at all. +# +# Referenced by name from schedule (1)'s template (`resourcePolicy: {kind: configmap, name: ...}`). +# Applied to BOTH environments: dev has no NFS PVs today, so the rule is a no-op there, and it is +# already correct if dev ever grows one. +apiVersion: v1 +kind: ConfigMap +metadata: + name: velero-volume-policies + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + nfs: {} + action: + type: skip