diff --git a/apps/wave-0/plex.yaml b/apps/wave-0/plex.yaml index 6575b7b..d8d663b 100644 --- a/apps/wave-0/plex.yaml +++ b/apps/wave-0/plex.yaml @@ -35,6 +35,34 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume plex-media - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: plex-media + labels: + app.kubernetes.io/name: plex +spec: + capacity: + storage: 9Ti + accessModes: + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/plex + readOnly: true + claimRef: + namespace: plex + name: plex-media +--- # PVC plex-config (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +79,23 @@ spec: requests: storage: 8Gi --- +# PVC plex-media (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: plex-media + namespace: plex + labels: + app.kubernetes.io/name: plex +spec: + accessModes: + - ReadOnlyMany + storageClassName: '' + volumeName: plex-media + resources: + requests: + storage: 9Ti +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -90,10 +135,16 @@ spec: volumeMounts: - name: config mountPath: /config + - name: media + mountPath: /media + readOnly: true volumes: - name: config persistentVolumeClaim: claimName: plex-config + - name: media + persistentVolumeClaim: + claimName: plex-media --- # Service apiVersion: v1 diff --git a/apps/wave-1/nextcloud.yaml b/apps/wave-1/nextcloud.yaml index eb5a024..f54636a 100644 --- a/apps/wave-1/nextcloud.yaml +++ b/apps/wave-1/nextcloud.yaml @@ -35,6 +35,34 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume nextcloud-legacy-archive - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nextcloud-legacy-archive + labels: + app.kubernetes.io/name: nextcloud +spec: + capacity: + storage: 270Gi + accessModes: + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/nextcloud + readOnly: true + claimRef: + namespace: nextcloud + name: nextcloud-legacy-archive +--- # PVC nextcloud-data (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +79,23 @@ spec: requests: storage: 20Gi --- +# PVC nextcloud-legacy-archive (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nextcloud-legacy-archive + namespace: nextcloud + labels: + app.kubernetes.io/name: nextcloud +spec: + accessModes: + - ReadOnlyMany + storageClassName: '' + volumeName: nextcloud-legacy-archive + resources: + requests: + storage: 270Gi +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -119,6 +164,10 @@ spec: volumeMounts: - name: data mountPath: /var/www/html + - name: legacy-archive + mountPath: /mnt/legacy-nextcloud + subPath: nextcloud + readOnly: true lifecycle: postStart: exec: @@ -132,6 +181,9 @@ spec: - name: data persistentVolumeClaim: claimName: nextcloud-data + - name: legacy-archive + persistentVolumeClaim: + claimName: nextcloud-legacy-archive --- # Service apiVersion: v1 diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index 021bdd4..2aebd0d 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -35,6 +35,33 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume radarr-media - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: radarr-media + labels: + app.kubernetes.io/name: radarr +spec: + capacity: + storage: 9Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/plex + claimRef: + namespace: radarr + name: radarr-media +--- # PVC radarr-config (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +78,23 @@ spec: requests: storage: 5Gi --- +# PVC radarr-media (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: radarr-media + namespace: radarr + labels: + app.kubernetes.io/name: radarr +spec: + accessModes: + - ReadWriteMany + storageClassName: '' + volumeName: radarr-media + resources: + requests: + storage: 9Ti +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -81,9 +125,18 @@ spec: image: lscr.io/linuxserver/radarr:5.21.1 ports: - containerPort: 7878 + env: + - name: PUID + value: '1000' + - name: PGID + value: '3000' + - name: TZ + value: Etc/UTC volumeMounts: - name: config mountPath: /data/config + - name: media + mountPath: /media - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 imagePullPolicy: IfNotPresent @@ -148,10 +201,15 @@ spec: port: http initialDelaySeconds: 5 periodSeconds: 15 + securityContext: + fsGroup: 3000 volumes: - name: config persistentVolumeClaim: claimName: radarr-config + - name: media + persistentVolumeClaim: + claimName: radarr-media --- # Service apiVersion: v1 diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index 9af2717..4f90acd 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -35,6 +35,33 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume sonarr-media - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: sonarr-media + labels: + app.kubernetes.io/name: sonarr +spec: + capacity: + storage: 9Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/plex + claimRef: + namespace: sonarr + name: sonarr-media +--- # PVC sonarr-config (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +78,23 @@ spec: requests: storage: 5Gi --- +# PVC sonarr-media (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sonarr-media + namespace: sonarr + labels: + app.kubernetes.io/name: sonarr +spec: + accessModes: + - ReadWriteMany + storageClassName: '' + volumeName: sonarr-media + resources: + requests: + storage: 9Ti +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -81,9 +125,18 @@ spec: image: lscr.io/linuxserver/sonarr:4.0.15 ports: - containerPort: 8989 + env: + - name: PUID + value: '1000' + - name: PGID + value: '3000' + - name: TZ + value: Etc/UTC volumeMounts: - name: config mountPath: /data/config + - name: media + mountPath: /media - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 imagePullPolicy: IfNotPresent @@ -148,10 +201,15 @@ spec: port: http initialDelaySeconds: 5 periodSeconds: 15 + securityContext: + fsGroup: 3000 volumes: - name: config persistentVolumeClaim: claimName: sonarr-config + - name: media + persistentVolumeClaim: + claimName: sonarr-media --- # Service apiVersion: v1 diff --git a/catalog/nextcloud.yaml b/catalog/nextcloud.yaml index ab1e40d..fedca50 100644 --- a/catalog/nextcloud.yaml +++ b/catalog/nextcloud.yaml @@ -223,6 +223,44 @@ storage: mountPath: /var/www/html # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02). # The whole html dir is persisted so config.php + user data survive a # rebuild (the default /data/ would strand them on ephemeral rootfs). + # THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND + # NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision. + # + # WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed): + # /mnt/Storage/nextcloud/nextcloud/config 103 M + # /mnt/Storage/nextcloud/nextcloud/config_old 3.4 G + # /mnt/Storage/nextcloud/nextcloud/config_backup 267 G <- 99% of the dataset + # There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup + # blob — an ARCHIVE, not a live user-data tree. + # + # WHY READ-ONLY. Two independent reasons, either sufficient: + # 1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and + # 192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two + # Nextcloud instances writing one instance's config directory, each with its own database and + # therefore its own oc_filecache, is a corruption scenario with no clean recovery. + # 2. Nothing here needs writing. An archive is read. + # "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a + # MIGRATION (read this data in, then decommission the old instance), that is a different and + # larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the + # export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount. + # + # WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own + # state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does + # not own. A neutral path keeps the two completely separate. To expose it to users, add it as a + # Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision. + # + # subPath skips the redundant single-child directory: the dataset root holds exactly one entry, + # `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top. + # + # UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data + # (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.) + - name: legacy-archive + class: bulk + export: /mnt/Storage/nextcloud # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER} + subPath: nextcloud + mountPath: /mnt/legacy-nextcloud + readOnly: true + size: 270Gi # ADVISORY ONLY on a static NFS PV — the dataset's used size. dependencies: - to: nextcloud-db # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02). port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no diff --git a/catalog/plex.yaml b/catalog/plex.yaml index c2b9f0d..be48d97 100644 --- a/catalog/plex.yaml +++ b/catalog/plex.yaml @@ -4,21 +4,11 @@ # auth: native (Plex accounts, NOT Keycloak — docs/13 + docs/15 documented exception). # Validates against schemas/servicedefinition.schema.json. # -# BULK MEDIA (spec bulk-tier requirement, docs/04) — OPEN PREREQUISITE, intentionally NOT declared -# here yet (the requirement ID is cited in the WS-APPS report, not as a token here: this change does -# NOT exercise the bulk tier, so it must not be selected as a "touched" requirement whose in-cluster -# conformance test is still a Phase-1 gap). -# The Plex library belongs on the `bulk` tier (docs/04 / docs/13 cross-cutting note: metadata DB -# on replicated Longhorn, large library on bulk). The `bulk` StorageClass exists -# (platform/longhorn/storageclasses.yaml) and the schema+renderer both support `class: bulk`, but -# that class is a documented STUB: OQ-2 (external NFS/SMB backing) is UNRESOLVED and the NFS CSI -# driver is not installed, so a `bulk` PVC will not bind and would leave the Plex pod Pending. -# To onboard the running Plex shell now (compute + config + public exposure), the media volume is -# omitted. Once OQ-2 is resolved and `bulk` provisions, attach the library with a one-line add: -# storage: -# - name: media -# class: bulk # external mount; size omitted (sized outside the cluster) -# then re-render. See the WS-APPS report for the full bulk-media finding. +# BULK MEDIA — NOW ATTACHED (docs/04 bulk tier; the long-standing OQ-2 prerequisite, resolved for +# the STATIC case). The library is 8.33 T of ZFS on the Proxmox host at /mnt/Storage/plex, and it +# already exists: nothing is provisioned, an existing dataset is mounted. See +# docs/BULK-NFS-STORAGE.md for the layout, the export change and the uid analysis. (The bulk-tier +# requirement ID is deliberately still not typed here — see that document's "gate" note.) name: plex kind: container source: @@ -70,6 +60,31 @@ storage: # UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token # (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`, # so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off. + # THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies, + # /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`: + # downloads/ movies/ photos/ tv/). + # + # WHY THE ROOT AND NOT THREE subPath MOUNTS. Nothing is gained by narrowing: the mount is `ro`, + # so no subPath could remove a privilege that matters, while three mounts would triple the NFS + # mounts per pod and mean a manifest change every time a top-level library is added. + # + # WHY READ-ONLY, when the export is `rw` and the other consumers write. Plex is a SERVER: it + # indexes and streams. Every write on that tree belongs to Sonarr and Radarr. Marking it here + # costs nothing and removes Plex from the set of things that can damage 8.33 T. Enforced by the + # PV's nfs.readOnly (a real `ro` mount), not only by the volumeMount flag. + # + # UID: no change needed. The tree is 0755 owned 1000:3000 throughout, so it is world-readable and + # world-traversable, and PLEX_UID/PLEX_GID above already make the process uid 1000 in any case. + # (fsGroup would have been a no-op here regardless — the in-tree NFS plugin does no ownership + # management. That is the trap the arrs' PUID/PGID exists to avoid; Plex never hits it.) + - name: media + class: bulk + export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} + mountPath: /media + readOnly: true + size: 9Ti # ADVISORY ONLY (nothing enforces a static NFS PV's capacity): the + # dataset's 8.33 T used, rounded up, so `kubectl get pv` reports + # something true. The real limit is the ZFS pool. dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02) ingressRoutes: - host: plex.${BASE_DOMAIN} diff --git a/catalog/radarr.yaml b/catalog/radarr.yaml index ae385f8..8169c14 100644 --- a/catalog/radarr.yaml +++ b/catalog/radarr.yaml @@ -9,6 +9,23 @@ name: radarr kind: container source: image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement +# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are +# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911, +# which cannot write a 1000:3000-owned 0755 tree. +env: + - name: PUID + value: "1000" # owner uid of every directory under /mnt/Storage/plex + - name: PGID + value: "3000" # dominant group there; keeps files this writes readable to the + # EXISTING non-k3s consumers of the same export, which stay live + - name: TZ + value: Etc/UTC # same safe default plex uses; the arrs schedule by local time +podSecurityContext: + # For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin + # does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is + # what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning + # /config for itself when PUID changes. Matches PGID so the primary group owns the config too. + fsGroup: 3000 exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET) egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway, # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06) @@ -16,11 +33,55 @@ storage: - name: config size: 5Gi class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02) - # NOTE (bulk-media PREREQUISITE): docs/13 row 3 places the media library / downloads volume on the - # `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml) - # but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk` - # volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is - # therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Radarr deployable today. + # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving + # /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT. + # + # *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Radarr imports by HARDLINKING or + # atomically renaming a finished download into the library. Both are same-FILESYSTEM operations, + # and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is: + # /mnt/Storage/plex/movies/ the library + # /mnt/Storage/plex/downloads/complete/radarr/ where finished releases land + # — siblings under one dataset, so a hardlink between them is legal on the server. But mounting + # them as two subPath volumes (subPath: movies at /media/movies, subPath: downloads elsewhere) creates + # TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Radarr then + # silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the + # seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount + # is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard + # requirement, not a shortcut. (The cost is stated plainly: Radarr can also write /media/tv, + # which is sonarr's. Narrowing that would cost hardlinks, which is a worse trade.) + # + # *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the + # renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the + # config volume entirely; the pod would come up with its app database shadowed by an 8 T media + # tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for + # the same export, so when qbittorrent is eventually given this mount too, the arrs and the + # download client will already agree on absolute paths — which is the other precondition for + # hardlinked imports (otherwise every import needs a Remote Path Mapping). + # + # SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config + # is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run + # with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the + # Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment + # documents having already been bitten by. Fixing it means starting from an empty config volume, so + # it belongs in its own change with the operator present. + # + # UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash + # + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is + # the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can + # write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail + # every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner. + # + # AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no + # ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the + # lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup + # that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on + # the image re-chowning /config for itself. + - name: media + class: bulk + export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} + mountPath: /media # => /media/movies (Root Folder) and /media/downloads (import source) + size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used, + # rounded up, so `kubectl get pv` reports something true. dependencies: - to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow) port: 8080 diff --git a/catalog/sonarr.yaml b/catalog/sonarr.yaml index 25cc318..5060a76 100644 --- a/catalog/sonarr.yaml +++ b/catalog/sonarr.yaml @@ -9,6 +9,23 @@ name: sonarr kind: container source: image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement +# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are +# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911, +# which cannot write a 1000:3000-owned 0755 tree. +env: + - name: PUID + value: "1000" # owner uid of every directory under /mnt/Storage/plex + - name: PGID + value: "3000" # dominant group there; keeps files this writes readable to the + # EXISTING non-k3s consumers of the same export, which stay live + - name: TZ + value: Etc/UTC # same safe default plex uses; the arrs schedule by local time +podSecurityContext: + # For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin + # does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is + # what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning + # /config for itself when PUID changes. Matches PGID so the primary group owns the config too. + fsGroup: 3000 exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET) egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway, # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06) @@ -16,11 +33,55 @@ storage: - name: config size: 5Gi class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02) - # NOTE (bulk-media PREREQUISITE): docs/13 row 2 places the media library / downloads volume on the - # `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml) - # but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk` - # volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is - # therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Sonarr deployable today. + # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving + # /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT. + # + # *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Sonarr imports by HARDLINKING or + # atomically renaming a finished download into the library. Both are same-FILESYSTEM operations, + # and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is: + # /mnt/Storage/plex/tv/ the library + # /mnt/Storage/plex/downloads/complete/tv-sonarr/ where finished releases land + # — siblings under one dataset, so a hardlink between them is legal on the server. But mounting + # them as two subPath volumes (subPath: tv at /media/tv, subPath: downloads elsewhere) creates + # TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Sonarr then + # silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the + # seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount + # is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard + # requirement, not a shortcut. (The cost is stated plainly: Sonarr can also write /media/movies, + # which is radarr's. Narrowing that would cost hardlinks, which is a worse trade.) + # + # *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the + # renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the + # config volume entirely; the pod would come up with its app database shadowed by an 8 T media + # tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for + # the same export, so when qbittorrent is eventually given this mount too, the arrs and the + # download client will already agree on absolute paths — which is the other precondition for + # hardlinked imports (otherwise every import needs a Remote Path Mapping). + # + # SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config + # is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run + # with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the + # Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment + # documents having already been bitten by. Fixing it means starting from an empty config volume, so + # it belongs in its own change with the operator present. + # + # UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash + # + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is + # the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can + # write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail + # every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner. + # + # AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no + # ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the + # lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup + # that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on + # the image re-chowning /config for itself. + - name: media + class: bulk + export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} + mountPath: /media # => /media/tv (Root Folder) and /media/downloads (import source) + size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used, + # rounded up, so `kubectl get pv` reports something true. dependencies: - to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow) port: 8080 diff --git a/docs/BULK-NFS-STORAGE.md b/docs/BULK-NFS-STORAGE.md new file mode 100644 index 0000000..b9af86f --- /dev/null +++ b/docs/BULK-NFS-STORAGE.md @@ -0,0 +1,337 @@ +# Bulk NFS storage — the media library and the legacy Nextcloud archive + +The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`. +Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is +assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the +case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk +provisioning unimplemented, because nothing needs it. + +--- + +## 1. The storage host, and why prod cannot reach it + +The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface: + +``` +vmbr0 192.168.1.9/24 default via 192.168.1.1 (pfSense LAN) +``` + +| | nodes | reaches 192.168.1.9? | +|---|---|---| +| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) | +| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. | + +Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give +different values for the `BULK_NFS_SERVER` cluster-var: + +| outcome | `BULK_NFS_SERVER` for prod | +|---|---| +| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` | +| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` | + +Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder +behaves differently from the repo's other ones.** + +### It only needs ONE port + +The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that +must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed +four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The +server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the +conservative floor. + +### The export's client list is the SAME under both outcomes — measured, not assumed + +**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation +(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`: + +``` +SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22 +ss -tn -> 192.168.1.9:22 192.168.2.132:39768 +``` + +The original WAN-side source address survives the translation (return traffic finds its way back via +the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's +**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either +way. **It does not have to wait for the routing decision.** + +--- + +## 2. The layout, as found + +### `Storage/plex` → `/mnt/Storage/plex` — 8.33 T used, 6.91 T avail + +``` +/mnt/Storage/plex/ drwxr-xr-x 1000:3000 +├── downloads/ drwxr-xr-x 1000:1000 +│ ├── complete/ drwxr-xr-x 1000:3000 +│ │ ├── radarr/ drwxr-xr-x 1000:3000 +│ │ └── tv-sonarr/ drwxr-xr-x 1000:3000 +│ ├── incomplete/ drwxrwxr-x 1000:3000 +│ ├── downloads -> /shared/downloads (symlink OUTSIDE the export — see §8) +│ └── drwxr-xr-x 1000:1000 +├── movies/ 1165 entries drwxr-xr-x 1000:3000 +├── tv/ 107 entries drwxr-xr-x 1000:3000 +└── photos/ drwxr-xr-x 1000:3000 +``` + +**The single most important fact about this layout: `downloads/` is a SIBLING of `tv/` and `movies/` +inside the same dataset.** Sonarr's import source and its library are on one filesystem on the +server, and so are Radarr's. That is what makes hardlinked / atomically-renamed imports possible at +all — and §4 is about not throwing it away on the client side. + +Note the download client's completed folders are already named `radarr` and `tv-sonarr`, i.e. this +tree has been driven by an arr stack for a long time. The k3s arrs are joining an existing layout, +not defining one. + +### `Storage/nextcloud` → `/mnt/Storage/nextcloud` — 270 G used + +``` +/mnt/Storage/nextcloud/ drwxr-xr-x root:root +└── nextcloud/ drwxr-xr-x 1000:3000 + ├── config/ 103 M drwxr-xr-x 1000:3000 + ├── config_old/ 3.4 G drwxr-xr-x 1000:3000 + └── config_backup/ 267 G drwxr-xr-x 1000:3000 +``` + +**There is no `data/` directory.** 99% of the dataset is `config_backup`. This is the old, non-k3s +Nextcloud's configuration plus a backup blob — an **archive**, not a live user-data tree. That is +why it is mounted read-only; see §6. + +--- + +## 3. What each app mounts, and why + +| app | export | mountPath | subPath | mode | access mode | +|---|---|---|---|---|---| +| plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany | +| sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany | +| radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany | +| nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany | + +In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr +Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from +`/media/downloads/complete/{tv-sonarr,radarr}`. + +**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually +given the same export (it is not in this change), the arrs and the download client will already agree +on absolute paths, which is the second precondition for hardlinked imports — otherwise every import +needs a Remote Path Mapping and quietly falls back to copying when one is missing. + +**`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the +renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the +config volume entirely. See §8 for the separate latent bug that discovery exposed. + +--- + +## 4. Why the arrs mount the export ROOT, not two subPaths + +This is the design decision most likely to be "simplified" later by someone who has not read this. + +Sonarr and Radarr import a finished download into the library by **hardlinking** it (preserving the +seeding copy) or by an **atomic rename**. Both are same-filesystem operations, and **to the kernel, +"same filesystem" means "same mount."** + +The tempting narrower shape is two subPath volumes — `subPath: tv` and `subPath: downloads`. Those +are **two separate mounts** of the same export. `link()` across them returns **EXDEV** no matter that +the server knows perfectly well they are one dataset. Sonarr does not fail loudly at that: it +degrades to copy-then-delete. Every import then rewrites the entire file over NFS and destroys the +seeding copy. The symptom is "imports are slow and my ratio is gone", weeks later. + +The smallest subtree containing both `tv/` and `downloads/` **is the export root**. So the root mount +is the *minimal* mount that preserves hardlinks: least privilege bounded below by a hard requirement, +not a shortcut. + +The cost is stated rather than hidden: Sonarr can also write `/media/movies` and Radarr `/media/tv`. +Narrowing that would cost hardlinks, which is the worse trade. + +--- + +## 5. The export change the operator must apply — STRICTLY ADDITIVE + +**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the +exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and +192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the +live non-k3s prod stack. + +Current values (`zfs get sharenfs Storage`): + +``` +Storage/plex rw=@192.168.1.6:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash +Storage/nextcloud rw=@192.168.1.6:@192.168.1.5,no_subtree_check,no_root_squash +``` + +### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list + +```sh +zfs set sharenfs='rw=@192.168.1.6:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex +``` + +Every pre-existing host is preserved verbatim and no option changes; only clients are added. + +### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=` + +```sh +zfs set sharenfs='rw=@192.168.1.6:@192.168.1.5,ro=@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud +``` + +This is the one place read-only can be enforced **server-side**, and it is worth taking: only +Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a +misconfigured client cannot then damage the archive. + +**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but +all three may be scheduled on any node, so the server sees the same source addresses for all of +them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a +client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an +honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a +dedicated node pool for Plex, which is a much larger change. + +Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host +(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive +adding a node; it also grants the whole subnet. Per-node is the tighter default. + +### Verifying, without changing anything + +```sh +exportfs -v # the new client lines appear, existing ones unchanged +zfs get sharenfs Storage/plex Storage/nextcloud +``` + +--- + +## 6. uid / gid — the failure this design exists to avoid + +The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through +verbatim** and the server's on-disk ownership is the entire access story. The media tree is +`0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**. + +**`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS +volume plugin performs **no ownership management**, so `podSecurityContext.fsGroup` is a **no-op on +an NFS volume**. There is no recursive chown, no `g+rwX`, nothing. The container's own uid has to be +right. + +| app | container uid | writes? | outcome | +|---|---|---|---| +| plex | 1000 (`PLEX_UID`/`PLEX_GID`, already set) | no | reads fine — the tree is 0755, so world-readable anyway | +| sonarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine | +| radarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine | +| nextcloud | 33 (`www-data`) | no | reads and traverses 0755 fine | + +**Without the added `PUID`/`PGID`, sonarr and radarr would have failed exactly the way this tier +always fails: the mount succeeds, the pod is Ready, and every import fails EACCES.** LinuxServer +images default to uid/gid **911**, which owns nothing on that tree. + +`PGID: 3000` is chosen over 1000 so files these write are group-owned by the dominant group already +on the tree, keeping them readable to the live non-k3s consumers. + +`podSecurityContext.fsGroup: 3000` is also added to sonarr and radarr — **for the Longhorn config +PVC, not for the NFS mount.** fsGroup does work on a Longhorn filesystem, and it makes `/data/config` +writable by the now-uid-1000 process without depending on the image re-chowning it after `PUID` +changed. + +Nextcloud needs no uid work at all, which is one more reason its mount is read-only. + +--- + +## 7. The prod placeholder does NOT fail at apply time — read this before merging + +Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation +(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.** + +A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts +`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure +control-plane and never touches the server. The failure appears only when a pod mounts: + +``` +MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER +``` + +so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.** + +**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a +revision that carries these bulk volumes.** Dev renders a working value and is where these manifests +are meant to be proven first. + +### The static guard this convention leans on is missing from `main` + +`bootstrap/standup.sh` — which carries the `*_is_placeholder` guard family — **does not exist on +`main`** (verified: `git ls-tree main bootstrap/` lists only `README.md`, `flux/`, `gitea/`, `sops/`; +the file lives on unmerged prod branches). So +`conformance/tests/l0-static/placeholder-guard-coupling/check.sh` exits **3 SKIPPED**, and **nothing +in the gate currently refuses a placeholder-shaped overlay value.** + +When `standup.sh` lands, that check's assertions (2) and (3) cover this knob automatically — its +independent detector's SEED vocabulary already matches `TBD` — **provided** the guard gains an arm +for `BULK_NFS_SERVER` and a consumer that calls it. Without the arm, assertion (2) is what fails, by +design. + +--- + +## 8. Loose ends and things deliberately not done + +1. **`/data/config` is very likely the wrong home for the arrs' config PVC — a pre-existing latent + bug, not fixed here.** LinuxServer's arr images run with `-data=/config`, so the app writes + `/config`, which is the container's **ephemeral layer**, while the Longhorn PVC mounted at + `/data/config` sits unused. This is the exact failure `catalog/plex.yaml`'s `mountPath` comment + records having already been bitten by ("the PVC sat unused and Plex wrote to the container's + EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it + means those apps start from an empty config volume, so it belongs in its own change with the + operator present. **This change does not make it worse** — that is why the media mount is `/media`. +2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy + non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s + qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were + chosen so it needs no Remote Path Mapping. +3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside + the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log + it while scanning `downloads/`. +4. **`nfsvers=4.2` was not chosen**, though the server advertises it. Nothing in these workloads uses + what 4.2 adds (server-side copy, sparse-file `SEEK`/`ALLOCATE`), and 4.1 is the version the + existing `bulk` StorageClass stub already named. +5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning + change that should be measured, not guessed, and it changes client behaviour under server + failure. +6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this + workstation (browser-OIDC only), and this change is manifest-authoring under an explicit + do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and + mounting* is reasoned from the manifests. +7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.** + `ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the + in-cluster conformance test for the bulk tier is still a gap, so typing it would select a + requirement the gate cannot satisfy — the hazard `catalog/qbittorrent.yaml` already warns about. + Scaffolding that test needs a cluster and is the correct follow-up. + +--- + +## 9. Validating on dev, once the export is extended + +Dev can prove all of this before prod's network path exists. Read-only checks first: + +```sh +infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv +infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive' +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide +``` + +Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three +of them naming the *same* export, one per namespace, which is the cross-namespace model) — each +`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`. + +Then the two things that actually matter, from inside a pod: + +```sh +# (a) the writer can write — this is the uid check, and it is the one that fails in practice +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \ + sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK' + +# (b) hardlinks work BETWEEN the library and the download path — the root-mount check +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \ + sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link; + echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK' + +# (c) the reader really is read-only +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \ + sh -c 'touch /media/.probe 2>&1 | head -1' # expect "Read-only file system" +``` + +`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it +prints `EXDEV`, something has split the single mount back into two.