diff --git a/apps/wave-0/plex.yaml b/apps/wave-0/plex.yaml index 6575b7b..d8d663b 100644 --- a/apps/wave-0/plex.yaml +++ b/apps/wave-0/plex.yaml @@ -35,6 +35,34 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume plex-media - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: plex-media + labels: + app.kubernetes.io/name: plex +spec: + capacity: + storage: 9Ti + accessModes: + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/plex + readOnly: true + claimRef: + namespace: plex + name: plex-media +--- # PVC plex-config (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +79,23 @@ spec: requests: storage: 8Gi --- +# PVC plex-media (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: plex-media + namespace: plex + labels: + app.kubernetes.io/name: plex +spec: + accessModes: + - ReadOnlyMany + storageClassName: '' + volumeName: plex-media + resources: + requests: + storage: 9Ti +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -90,10 +135,16 @@ spec: volumeMounts: - name: config mountPath: /config + - name: media + mountPath: /media + readOnly: true volumes: - name: config persistentVolumeClaim: claimName: plex-config + - name: media + persistentVolumeClaim: + claimName: plex-media --- # Service apiVersion: v1 diff --git a/apps/wave-1/nextcloud.yaml b/apps/wave-1/nextcloud.yaml index eb5a024..f54636a 100644 --- a/apps/wave-1/nextcloud.yaml +++ b/apps/wave-1/nextcloud.yaml @@ -35,6 +35,34 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume nextcloud-legacy-archive - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nextcloud-legacy-archive + labels: + app.kubernetes.io/name: nextcloud +spec: + capacity: + storage: 270Gi + accessModes: + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/nextcloud + readOnly: true + claimRef: + namespace: nextcloud + name: nextcloud-legacy-archive +--- # PVC nextcloud-data (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +79,23 @@ spec: requests: storage: 20Gi --- +# PVC nextcloud-legacy-archive (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nextcloud-legacy-archive + namespace: nextcloud + labels: + app.kubernetes.io/name: nextcloud +spec: + accessModes: + - ReadOnlyMany + storageClassName: '' + volumeName: nextcloud-legacy-archive + resources: + requests: + storage: 270Gi +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -119,6 +164,10 @@ spec: volumeMounts: - name: data mountPath: /var/www/html + - name: legacy-archive + mountPath: /mnt/legacy-nextcloud + subPath: nextcloud + readOnly: true lifecycle: postStart: exec: @@ -132,6 +181,9 @@ spec: - name: data persistentVolumeClaim: claimName: nextcloud-data + - name: legacy-archive + persistentVolumeClaim: + claimName: nextcloud-legacy-archive --- # Service apiVersion: v1 diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index 021bdd4..2aebd0d 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -35,6 +35,33 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume radarr-media - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: radarr-media + labels: + app.kubernetes.io/name: radarr +spec: + capacity: + storage: 9Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/plex + claimRef: + namespace: radarr + name: radarr-media +--- # PVC radarr-config (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +78,23 @@ spec: requests: storage: 5Gi --- +# PVC radarr-media (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: radarr-media + namespace: radarr + labels: + app.kubernetes.io/name: radarr +spec: + accessModes: + - ReadWriteMany + storageClassName: '' + volumeName: radarr-media + resources: + requests: + storage: 9Ti +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -81,9 +125,18 @@ spec: image: lscr.io/linuxserver/radarr:5.21.1 ports: - containerPort: 7878 + env: + - name: PUID + value: '1000' + - name: PGID + value: '3000' + - name: TZ + value: Etc/UTC volumeMounts: - name: config mountPath: /data/config + - name: media + mountPath: /media - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 imagePullPolicy: IfNotPresent @@ -148,10 +201,15 @@ spec: port: http initialDelaySeconds: 5 periodSeconds: 15 + securityContext: + fsGroup: 3000 volumes: - name: config persistentVolumeClaim: claimName: radarr-config + - name: media + persistentVolumeClaim: + claimName: radarr-media --- # Service apiVersion: v1 diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index 9af2717..4f90acd 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -35,6 +35,33 @@ spec: - protocol: TCP port: 53 --- +# PersistentVolume sonarr-media - static external NFS (bulk tier) +apiVersion: v1 +kind: PersistentVolume +metadata: + name: sonarr-media + labels: + app.kubernetes.io/name: sonarr +spec: + capacity: + storage: 9Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: '' + mountOptions: + - nfsvers=4.1 + - hard + - noatime + - timeo=600 + - retrans=2 + nfs: + server: ${BULK_NFS_SERVER} + path: /mnt/Storage/plex + claimRef: + namespace: sonarr + name: sonarr-media +--- # PVC sonarr-config (R-STATE) apiVersion: v1 kind: PersistentVolumeClaim @@ -51,6 +78,23 @@ spec: requests: storage: 5Gi --- +# PVC sonarr-media (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sonarr-media + namespace: sonarr + labels: + app.kubernetes.io/name: sonarr +spec: + accessModes: + - ReadWriteMany + storageClassName: '' + volumeName: sonarr-media + resources: + requests: + storage: 9Ti +--- # Deployment (R-PLAT-02) apiVersion: apps/v1 kind: Deployment @@ -81,9 +125,18 @@ spec: image: lscr.io/linuxserver/sonarr:4.0.15 ports: - containerPort: 8989 + env: + - name: PUID + value: '1000' + - name: PGID + value: '3000' + - name: TZ + value: Etc/UTC volumeMounts: - name: config mountPath: /data/config + - name: media + mountPath: /media - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 imagePullPolicy: IfNotPresent @@ -148,10 +201,15 @@ spec: port: http initialDelaySeconds: 5 periodSeconds: 15 + securityContext: + fsGroup: 3000 volumes: - name: config persistentVolumeClaim: claimName: sonarr-config + - name: media + persistentVolumeClaim: + claimName: sonarr-media --- # Service apiVersion: v1 diff --git a/catalog/nextcloud.yaml b/catalog/nextcloud.yaml index ab1e40d..fedca50 100644 --- a/catalog/nextcloud.yaml +++ b/catalog/nextcloud.yaml @@ -223,6 +223,44 @@ storage: mountPath: /var/www/html # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02). # The whole html dir is persisted so config.php + user data survive a # rebuild (the default /data/ would strand them on ephemeral rootfs). + # THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND + # NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision. + # + # WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed): + # /mnt/Storage/nextcloud/nextcloud/config 103 M + # /mnt/Storage/nextcloud/nextcloud/config_old 3.4 G + # /mnt/Storage/nextcloud/nextcloud/config_backup 267 G <- 99% of the dataset + # There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup + # blob — an ARCHIVE, not a live user-data tree. + # + # WHY READ-ONLY. Two independent reasons, either sufficient: + # 1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and + # 192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two + # Nextcloud instances writing one instance's config directory, each with its own database and + # therefore its own oc_filecache, is a corruption scenario with no clean recovery. + # 2. Nothing here needs writing. An archive is read. + # "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a + # MIGRATION (read this data in, then decommission the old instance), that is a different and + # larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the + # export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount. + # + # WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own + # state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does + # not own. A neutral path keeps the two completely separate. To expose it to users, add it as a + # Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision. + # + # subPath skips the redundant single-child directory: the dataset root holds exactly one entry, + # `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top. + # + # UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data + # (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.) + - name: legacy-archive + class: bulk + export: /mnt/Storage/nextcloud # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER} + subPath: nextcloud + mountPath: /mnt/legacy-nextcloud + readOnly: true + size: 270Gi # ADVISORY ONLY on a static NFS PV — the dataset's used size. dependencies: - to: nextcloud-db # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02). port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no diff --git a/catalog/plex.yaml b/catalog/plex.yaml index c2b9f0d..be48d97 100644 --- a/catalog/plex.yaml +++ b/catalog/plex.yaml @@ -4,21 +4,11 @@ # auth: native (Plex accounts, NOT Keycloak — docs/13 + docs/15 documented exception). # Validates against schemas/servicedefinition.schema.json. # -# BULK MEDIA (spec bulk-tier requirement, docs/04) — OPEN PREREQUISITE, intentionally NOT declared -# here yet (the requirement ID is cited in the WS-APPS report, not as a token here: this change does -# NOT exercise the bulk tier, so it must not be selected as a "touched" requirement whose in-cluster -# conformance test is still a Phase-1 gap). -# The Plex library belongs on the `bulk` tier (docs/04 / docs/13 cross-cutting note: metadata DB -# on replicated Longhorn, large library on bulk). The `bulk` StorageClass exists -# (platform/longhorn/storageclasses.yaml) and the schema+renderer both support `class: bulk`, but -# that class is a documented STUB: OQ-2 (external NFS/SMB backing) is UNRESOLVED and the NFS CSI -# driver is not installed, so a `bulk` PVC will not bind and would leave the Plex pod Pending. -# To onboard the running Plex shell now (compute + config + public exposure), the media volume is -# omitted. Once OQ-2 is resolved and `bulk` provisions, attach the library with a one-line add: -# storage: -# - name: media -# class: bulk # external mount; size omitted (sized outside the cluster) -# then re-render. See the WS-APPS report for the full bulk-media finding. +# BULK MEDIA — NOW ATTACHED (docs/04 bulk tier; the long-standing OQ-2 prerequisite, resolved for +# the STATIC case). The library is 8.33 T of ZFS on the Proxmox host at /mnt/Storage/plex, and it +# already exists: nothing is provisioned, an existing dataset is mounted. See +# docs/BULK-NFS-STORAGE.md for the layout, the export change and the uid analysis. (The bulk-tier +# requirement ID is deliberately still not typed here — see that document's "gate" note.) name: plex kind: container source: @@ -70,6 +60,31 @@ storage: # UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token # (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`, # so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off. + # THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies, + # /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`: + # downloads/ movies/ photos/ tv/). + # + # WHY THE ROOT AND NOT THREE subPath MOUNTS. Nothing is gained by narrowing: the mount is `ro`, + # so no subPath could remove a privilege that matters, while three mounts would triple the NFS + # mounts per pod and mean a manifest change every time a top-level library is added. + # + # WHY READ-ONLY, when the export is `rw` and the other consumers write. Plex is a SERVER: it + # indexes and streams. Every write on that tree belongs to Sonarr and Radarr. Marking it here + # costs nothing and removes Plex from the set of things that can damage 8.33 T. Enforced by the + # PV's nfs.readOnly (a real `ro` mount), not only by the volumeMount flag. + # + # UID: no change needed. The tree is 0755 owned 1000:3000 throughout, so it is world-readable and + # world-traversable, and PLEX_UID/PLEX_GID above already make the process uid 1000 in any case. + # (fsGroup would have been a no-op here regardless — the in-tree NFS plugin does no ownership + # management. That is the trap the arrs' PUID/PGID exists to avoid; Plex never hits it.) + - name: media + class: bulk + export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} + mountPath: /media + readOnly: true + size: 9Ti # ADVISORY ONLY (nothing enforces a static NFS PV's capacity): the + # dataset's 8.33 T used, rounded up, so `kubectl get pv` reports + # something true. The real limit is the ZFS pool. dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02) ingressRoutes: - host: plex.${BASE_DOMAIN} diff --git a/catalog/radarr.yaml b/catalog/radarr.yaml index ae385f8..8169c14 100644 --- a/catalog/radarr.yaml +++ b/catalog/radarr.yaml @@ -9,6 +9,23 @@ name: radarr kind: container source: image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement +# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are +# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911, +# which cannot write a 1000:3000-owned 0755 tree. +env: + - name: PUID + value: "1000" # owner uid of every directory under /mnt/Storage/plex + - name: PGID + value: "3000" # dominant group there; keeps files this writes readable to the + # EXISTING non-k3s consumers of the same export, which stay live + - name: TZ + value: Etc/UTC # same safe default plex uses; the arrs schedule by local time +podSecurityContext: + # For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin + # does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is + # what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning + # /config for itself when PUID changes. Matches PGID so the primary group owns the config too. + fsGroup: 3000 exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET) egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway, # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06) @@ -16,11 +33,55 @@ storage: - name: config size: 5Gi class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02) - # NOTE (bulk-media PREREQUISITE): docs/13 row 3 places the media library / downloads volume on the - # `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml) - # but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk` - # volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is - # therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Radarr deployable today. + # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving + # /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT. + # + # *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Radarr imports by HARDLINKING or + # atomically renaming a finished download into the library. Both are same-FILESYSTEM operations, + # and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is: + # /mnt/Storage/plex/movies/ the library + # /mnt/Storage/plex/downloads/complete/radarr/ where finished releases land + # — siblings under one dataset, so a hardlink between them is legal on the server. But mounting + # them as two subPath volumes (subPath: movies at /media/movies, subPath: downloads elsewhere) creates + # TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Radarr then + # silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the + # seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount + # is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard + # requirement, not a shortcut. (The cost is stated plainly: Radarr can also write /media/tv, + # which is sonarr's. Narrowing that would cost hardlinks, which is a worse trade.) + # + # *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the + # renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the + # config volume entirely; the pod would come up with its app database shadowed by an 8 T media + # tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for + # the same export, so when qbittorrent is eventually given this mount too, the arrs and the + # download client will already agree on absolute paths — which is the other precondition for + # hardlinked imports (otherwise every import needs a Remote Path Mapping). + # + # SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config + # is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run + # with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the + # Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment + # documents having already been bitten by. Fixing it means starting from an empty config volume, so + # it belongs in its own change with the operator present. + # + # UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash + # + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is + # the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can + # write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail + # every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner. + # + # AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no + # ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the + # lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup + # that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on + # the image re-chowning /config for itself. + - name: media + class: bulk + export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} + mountPath: /media # => /media/movies (Root Folder) and /media/downloads (import source) + size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used, + # rounded up, so `kubectl get pv` reports something true. dependencies: - to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow) port: 8080 diff --git a/catalog/render.py b/catalog/render.py index 3533e46..9fd868b 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -53,6 +53,30 @@ LBL = "homelab.keithjackson.dev" # Hardcoding the literal here is what let prod be authored to serve dev's public hostnames, # and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10). ZONE = "${BASE_DOMAIN}" +# The bulk (external NFS) tier's SERVER, emitted as a Flux post-build TOKEN for exactly the same +# reason as ZONE: dev and prod reach the same Proxmox ZFS host by DIFFERENT addresses (dev's nodes +# are on the host's own LAN and dial it directly; prod's nodes are on the far side of pfSense and +# must be given whichever address the still-pending routing decision produces). Hardcoding either +# one authors a manifest that is silently wrong in the other environment. Declared in +# clusters/base/cluster-vars.yaml and patched by BOTH overlays; written BARE (it is a real knob, +# not a shell expansion), so StrictPostBuildSubstitutions fails the tier if an overlay forgets it. +BULK_NFS_SERVER = "${BULK_NFS_SERVER}" +# Mount options for every bulk NFS volume. These are a PLATFORM decision, not a per-service one, +# so they live here rather than on the ServiceDefinition surface: +# nfsvers=4.1 NFSv4 carries mount+lock IN the protocol, so it needs ONE port (tcp/2049) and no +# rpcbind/111, mountd or statd. That is what makes the prod path a single pfSense +# rule instead of four, and it is why v4 is chosen over v3. The server advertises +# +3 +4 +4.1 +4.2; 4.1 is the conservative floor every node kernel already speaks. +# hard A `soft` mount turns a server hiccup into a short read / EIO handed to the +# application, which for a media library and an archive is silent corruption. +# `hard` blocks instead. The cost is explicit: an NFS outage HANGS I/O rather than +# failing it, so an outage shows up as stuck pods, not as damaged files. Chosen +# deliberately -- hung is recoverable, truncated is not. +# noatime The datasets are relatime, so every read would otherwise generate a metadata +# WRITE back to the pool. Plex walking an 8 T library must not do that. +# timeo=600/retrans=2 the TCP defaults, stated explicitly so a future reader knows the retry +# budget was considered rather than inherited. +BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retrans=2"] GATEWAY_NAME = "shared" GATEWAY_NS = "gateway" KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label @@ -111,8 +135,10 @@ def apply_defaults(sd: dict) -> dict: obs.setdefault("slis", []) s["observability"] = obs - # storage entry class default is longhorn (replicated tier). - s["storage"] = [{**{"class": "longhorn"}, **v} for v in s["storage"]] + # storage entry class default is longhorn (replicated tier); readOnly default is False + # (grant-nothing-extra is expressed by SETTING it, so the default stays back-compatible and + # every pre-existing golden renders byte-identical). + s["storage"] = [{**{"class": "longhorn", "readOnly": False}, **v} for v in s["storage"]] # dependency protocol default is tcp. s["dependencies"] = [{**{"protocol": "tcp"}, **d} for d in s["dependencies"]] return s @@ -200,18 +226,139 @@ def _service_ports(s: dict) -> list: return [{"name": f"port-{p}", "port": p, "targetPort": p} for p in ports] +def _bulk_access_mode(vol: dict) -> str: + """A bulk volume's access mode follows its read/write posture. + + Every bulk volume is `...Many`: the SAME export is consumed by several namespaces at once + (plex, sonarr and radarr all sit on /mnt/Storage/plex) and by several nodes at once, which is + precisely what ReadWriteOnce cannot express. WHICH `Many` it is says whether this consumer + writes -- so `kubectl get pv` reports the intent, not merely the topology. + """ + return "ReadOnlyMany" if vol.get("readOnly") else "ReadWriteMany" + + +def r_bulk_pvs(s: dict) -> list: + """One STATIC PersistentVolume per `class: bulk` storage entry (docs/04 bulk tier). + + WHY A PV AT ALL, RATHER THAN AN INLINE `nfs:` POD VOLUME + ------------------------------------------------------- + An inline volume is fewer objects and it would work. It is rejected for four reasons, in + descending order of what they would cost to discover later: + + 1. RECLAIM POLICY. `persistentVolumeReclaimPolicy: Retain` is a declared, reviewable + property of a PV. There is no equivalent for an inline volume -- and the data behind + these mounts is 8.33 TiB of media and a 267 GiB archive that ALSO has live non-k3s + consumers. A statement in git that nothing in this cluster may reclaim it is worth an + object. + 2. READ-ONLY LIVES IN ONE PLACE. `spec.nfs.readOnly: true` becomes a real `ro` mount option + for every pod that ever binds this volume. The same intent expressed on an inline volume + is one Deployment edit away from being dropped silently, with no diff anywhere near the + storage. (It is still a CLIENT-side guarantee -- the export cannot distinguish plex from + sonarr, since both may run on any node -- but it is one that lives in one reviewable + place.) + 3. VISIBILITY. `kubectl get pv,pvc` is where an operator looks for "what external storage is + attached, and who holds it". An inline volume is invisible there, and invisible to + anything that reasons over PVCs. + 4. IT IS THE CONTRACT THE REPO ALREADY HAS. Every other storage entry is a PVC; the + renderer, the Kyverno ServiceDefinition policy and the backup wiring all key on that. An + inline volume would be a second, parallel storage model inside the same renderer. + + CROSS-NAMESPACE. A PV binds 1:1 to a PVC, so "plex, sonarr and radarr share one export" is NOT + one shared PV -- it is three PVs, one per namespace, each naming the same server+path. That is + the correct model rather than a workaround: the sharing happens on the NFS server, which is + built for concurrent clients, and each namespace keeps its own independently reviewable access + mode (plex ReadOnlyMany, the arrs ReadWriteMany). One shared PV would have forced a single + posture on all three. + + BINDING IS PINNED FROM BOTH ENDS -- `claimRef` here and `volumeName` on the PVC. Either alone + leaves a race in which some other PVC of matching size and mode grabs the volume first; + together the binding is deterministic. `storageClassName: ""` (explicitly empty, NOT the `bulk` + StorageClass) is what tells the binder this volume is statically provisioned: it suppresses the + default-StorageClass annotation and guarantees no provisioner is ever asked to create anything. + It also sidesteps a trap -- a StorageClass's `provisioner` is IMMUTABLE, so re-pointing the + existing `bulk` class (whose stub names an NFS CSI driver that is not installed) at this data + would make Flux fail forever on that one object. + """ + out = [] + for vol in s["storage"]: + if vol.get("class") != "bulk": + continue + name = f"{s['name']}-{vol['name']}" + # `export` and `size` are both structurally required of a bulk entry, and the JSON + # Schema only enforces the first. A PersistentVolume with no `capacity` is rejected by + # the API server, so a missing size must fail HERE with a sentence a human can act on + # rather than as a bare KeyError three frames down. + for field, why in ( + ("export", "the absolute NFS export path this volume attaches"), + ("size", "the PV capacity (advisory, but the API requires the field)"), + ): + if not vol.get(field): + raise ValueError( + f"{s['name']}: storage entry {vol['name']!r} has class 'bulk' but no " + f"'{field}' — {why}. A bulk volume attaches storage that already exists " + f"outside the cluster, so it cannot be rendered without it." + ) + nfs = {"server": BULK_NFS_SERVER, "path": vol["export"]} + if vol.get("readOnly"): + nfs["readOnly"] = True + spec = { + "capacity": {"storage": vol["size"]}, + "accessModes": [_bulk_access_mode(vol)], + # Retain: deleting the PVC must NEVER be able to reach the data. There is nothing to + # "reclaim" here in any case -- the volume was not provisioned by the cluster. + "persistentVolumeReclaimPolicy": "Retain", + "storageClassName": "", + "mountOptions": list(BULK_NFS_MOUNT_OPTIONS), + "nfs": nfs, + "claimRef": {"namespace": s["name"], "name": name}, + } + out.append( + { + "apiVersion": "v1", + "kind": "PersistentVolume", + "metadata": { + "name": name, + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": spec, + } + ) + return out + + def r_pvcs(s: dict) -> list: """One PVC per storage entry (R-STATE-01/02/04). - longhorn -> 3-replica replicated tier; bulk -> external media tier (not 3x replicated). + longhorn -> 3-replica replicated tier, dynamically provisioned, ReadWriteOnce. + bulk -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName. """ out = [] for vol in s["storage"]: cls = vol.get("class", "longhorn") - storage_class = "longhorn" if cls == "longhorn" else "bulk" + if cls == "bulk": + # Mirror image of the PV: the same explicitly-empty class (so nothing dynamically + # provisions), the same access mode, and volumeName completing the two-sided binding. + out.append( + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{s['name']}-{vol['name']}", + "namespace": s["name"], + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": { + "accessModes": [_bulk_access_mode(vol)], + "storageClassName": "", + "volumeName": f"{s['name']}-{vol['name']}", + "resources": {"requests": {"storage": vol["size"]}}, + }, + } + ) + continue spec = { "accessModes": ["ReadWriteOnce"], - "storageClassName": storage_class, + "storageClassName": "longhorn", } if vol.get("size"): spec["resources"] = {"requests": {"storage": vol["size"]}} @@ -257,7 +404,17 @@ def r_deployment(s: dict) -> dict: # it verbatim for apps that need a fixed on-disk layout (e.g. Gitea keeps all state # directly under /data). Omitting it preserves the /data/ default (back-compat). mount_path = vol.get("mountPath", f"/data/{vol['name']}") - volume_mounts.append({"name": vol["name"], "mountPath": mount_path}) + mount = {"name": vol["name"], "mountPath": mount_path} + # subPath mounts a subtree of the volume instead of its root. Emitted only when declared, + # so every pre-existing golden stays byte-identical. + if vol.get("subPath"): + mount["subPath"] = vol["subPath"] + # readOnly on the mount as well as on the PV's nfs source. Belt and braces on purpose: the + # PV field is what makes the kernel mount `ro`; this one is what a reviewer reading the + # Deployment sees. An edit that drops one still leaves the other. + if vol.get("readOnly"): + mount["readOnly"] = True + volume_mounts.append(mount) volumes.append( { "name": vol["name"], @@ -1095,6 +1252,14 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None) if s["kind"] == "container": # Container PVCs. (For a vm, the CDI DataVolume owns/creates the disk PVC, so we # never render a plain PVC for the VM root disk — it would collide with CDI.) + # The static bulk PV is rendered BEFORE the PVC that names it. Document order inside one + # kustomize build is not an apply-ordering guarantee, but a PVC whose PV does not exist + # yet merely stays Pending and binds on the next reconcile, so this is a readability + # convention rather than a correctness requirement. + for pv in r_bulk_pvs(s): + docs.append( + (f"PersistentVolume {pv['metadata']['name']} - static external NFS (bulk tier)", pv) + ) for pvc in r_pvcs(s): docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc)) docs.append(("Deployment (R-PLAT-02)", r_deployment(s))) diff --git a/catalog/sonarr.yaml b/catalog/sonarr.yaml index 25cc318..5060a76 100644 --- a/catalog/sonarr.yaml +++ b/catalog/sonarr.yaml @@ -9,6 +9,23 @@ name: sonarr kind: container source: image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement +# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are +# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911, +# which cannot write a 1000:3000-owned 0755 tree. +env: + - name: PUID + value: "1000" # owner uid of every directory under /mnt/Storage/plex + - name: PGID + value: "3000" # dominant group there; keeps files this writes readable to the + # EXISTING non-k3s consumers of the same export, which stay live + - name: TZ + value: Etc/UTC # same safe default plex uses; the arrs schedule by local time +podSecurityContext: + # For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin + # does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is + # what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning + # /config for itself when PUID changes. Matches PGID so the primary group owns the config too. + fsGroup: 3000 exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET) egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway, # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06) @@ -16,11 +33,55 @@ storage: - name: config size: 5Gi class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02) - # NOTE (bulk-media PREREQUISITE): docs/13 row 2 places the media library / downloads volume on the - # `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml) - # but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk` - # volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is - # therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Sonarr deployable today. + # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving + # /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT. + # + # *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Sonarr imports by HARDLINKING or + # atomically renaming a finished download into the library. Both are same-FILESYSTEM operations, + # and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is: + # /mnt/Storage/plex/tv/ the library + # /mnt/Storage/plex/downloads/complete/tv-sonarr/ where finished releases land + # — siblings under one dataset, so a hardlink between them is legal on the server. But mounting + # them as two subPath volumes (subPath: tv at /media/tv, subPath: downloads elsewhere) creates + # TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Sonarr then + # silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the + # seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount + # is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard + # requirement, not a shortcut. (The cost is stated plainly: Sonarr can also write /media/movies, + # which is radarr's. Narrowing that would cost hardlinks, which is a worse trade.) + # + # *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the + # renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the + # config volume entirely; the pod would come up with its app database shadowed by an 8 T media + # tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for + # the same export, so when qbittorrent is eventually given this mount too, the arrs and the + # download client will already agree on absolute paths — which is the other precondition for + # hardlinked imports (otherwise every import needs a Remote Path Mapping). + # + # SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config + # is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run + # with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the + # Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment + # documents having already been bitten by. Fixing it means starting from an empty config volume, so + # it belongs in its own change with the operator present. + # + # UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash + # + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is + # the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can + # write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail + # every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner. + # + # AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no + # ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the + # lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup + # that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on + # the image re-chowning /config for itself. + - name: media + class: bulk + export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} + mountPath: /media # => /media/tv (Root Folder) and /media/downloads (import source) + size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used, + # rounded up, so `kubectl get pv` reports something true. dependencies: - to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow) port: 8080 diff --git a/ci/l0.sh b/ci/l0.sh index 7101823..0c2bb28 100755 --- a/ci/l0.sh +++ b/ci/l0.sh @@ -175,6 +175,44 @@ else esac fi +# ---- Stage 3e: bulk tier distinct from the replicated tier (UNCONDITIONAL) ---- +# The `bulk` StorageClass sat as a REPLACE-ME stub for six roadmap phases, and the only +# thing keeping a multi-terabyte media library off the 3-replica `longhorn` class was four +# hand-written comments in ServiceDefinitions. A bulk volume that lands on the replicated +# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data +# three mini-PCs cannot hold. The check asserts the tier separation, that a bulk volume is +# statically pre-bound from BOTH ends (so nothing can dynamically provision an EMPTY +# stand-in that mounts and Binds perfectly while the real data is never attached), Retain, +# hard/NFSv4 mount options, the read-only coupling between PV and volumeMount, that no +# workload mounts one export twice (which breaks Sonarr/Radarr hardlinks with EXDEV), and +# that the server is a per-environment knob every overlay supplies. +# +# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c/3d: Stage 3b scopes the conformance +# run to the TOUCHED requirements (R-TEST-03), and ci/touched-requirements.sh's path map +# keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement. So the change most +# likely to break this — moving a bulk volume onto the replicated tier, or narrowing one +# export into two mounts — could attribute no requirement at all and be gated by nothing. +# The invariant spans catalog/, apps/ and clusters/, so it runs on EVERY gate invocation. +BULK_TIER_CHECK="${BULK_TIER_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh}" +log "Stage: bulk tier distinct from replicated tier (${BULK_TIER_CHECK})" +if [ ! -f "$BULK_TIER_CHECK" ]; then + skip "bulk tier check not present at '${BULK_TIER_CHECK}' — UNWIRED, not green" + summary "- bulk-tier: **SKIPPED** (check missing)" + if [ "$rc" -eq 0 ]; then rc=3; fi +else + bulk_rc=0 + bash "$BULK_TIER_CHECK" || bulk_rc=$? + case "$bulk_rc" in + 0) ok "Bulk tier distinctness + static binding passed" + summary "- bulk-tier: **passed**" ;; + 3) skip "bulk tier check SKIPPED (catalog/ or apps/ absent) — not a pass" + summary "- bulk-tier: **SKIPPED** (nothing to audit)" + if [ "$rc" -eq 0 ]; then rc=3; fi ;; + *) err "Bulk tier distinctness + static binding FAILED" + summary "- bulk-tier: **FAILED**"; rc=1 ;; + esac +fi + case "$rc" in 0) ok "L0 stage complete" ;; 3) skip "L0 stage incomplete (unwired seam / missing tool) — not a pass" ;; diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index 2762d67..5df3c1f 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -160,6 +160,28 @@ data: UNIFI_L4_VIP: "192.168.1.54" # UniFi controller VM: inform/STUN/adoption (apps/vms/unifi) CRAFTY_L4_VIP: "192.168.1.55" # Crafty: Minecraft Java (TCP) + Bedrock (UDP) game ports + # --- Bulk (external NFS) storage host (docs/04 bulk tier) --------------------------------------- + # THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media + # library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive + # (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's + # `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is + # declared in the ServiceDefinition, because the path is identical in every environment and only + # the ADDRESS is not. + # + # WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense. + # The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN. + # dev nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one + # NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in + # /etc/exports), so this path is proven, not theoretical. Base default = the dev value, + # per the convention of the LAN block above. + # prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today; + # 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening + # the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give + # DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route). + # So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the + # blast radius of shipping it unset, which is NOT the usual "fails to apply". + BULK_NFS_SERVER: "192.168.1.9" + # Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style. # Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db, # nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port. diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index cb2cb1a..0e38564 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -225,6 +225,15 @@ patches: - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.1.55" + # Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes + # (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes). + # Restated verbatim rather than inherited, per this overlay's "supply every knob" convention. + # Dev is therefore the environment in which the bulk manifests can actually be VALIDATED + # end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=` + # change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses. + - op: replace + path: /data/BULK_NFS_SERVER + value: "192.168.1.9" # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG # barmanObjectStore and Velero's backupStorageLocation write to. # diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index 78c5da0..92ee79c 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -392,6 +392,61 @@ patches: - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.2.22" + # ------------------------------------------------------------------------------------------ + # BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. *** + # ------------------------------------------------------------------------------------------ + # Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr, + # radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud). + # + # THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface, + # vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN. + # 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102; + # 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate + # outcomes give DIFFERENT values here: + # port-forward 192.168.2.102:2049 -> 192.168.1.9:2049 => "192.168.2.102" + # a WAN->LAN route to 192.168.1.0/24 => "192.168.1.9" + # Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess + # ever became reachable — mounts something unintended. So it stays a placeholder. + # + # WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and + # NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4 + # over v3 was in part this decision's cost: v3 would have needed four ports and three of them + # on ephemeral numbers. + # + # WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes. + # Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward: + # the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN + # address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via + # 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node + # addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md. + # + # *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. *** + # A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this + # placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The + # failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server + # TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in + # ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at + # apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles + # a revision containing the bulk volumes; the dev overlay renders a working value, which is + # where these manifests are meant to be proven first. + - op: replace + path: /data/BULK_NFS_SERVER + # 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox + # host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot + # reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and + # keeps the base value 192.168.1.9. + # + # PROVEN from k3s-prod-1 on 2026-09-12, not inferred: + # - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp + # - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export + # carries no_root_squash, so a root write would have passed and proved nothing, while + # LSIO images default to PUID 911 and would have gone Ready then failed every import + # with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer. + # - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get + # real atomic moves instead of silently degrading to copy-then-delete on EXDEV. + # pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer), + # so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP. + value: "192.168.2.102" # ========================================================================================== # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07). # ========================================================================================== diff --git a/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/chainsaw-test.yaml b/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/chainsaw-test.yaml new file mode 100644 index 0000000..c589489 --- /dev/null +++ b/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/chainsaw-test.yaml @@ -0,0 +1,99 @@ +# ============================================================================= +# L0 STATIC — the bulk tier's capacity strategy is DISTINCT from the replicated tier's +# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster. +# +# THE BUG CLASS THIS CLOSES. The `bulk` StorageClass existed from Phase 0 as a stub: +# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME`, TODO(OQ-2). +# Four ServiceDefinitions carried a comment promising to attach their media once OQ-2 was +# resolved, and the ONLY thing keeping a multi-terabyte library off the 3-replica +# `longhorn` class was those four comments. A bulk volume that lands on the replicated +# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data +# three mini-PCs cannot hold. R-STATE-04 is precisely that the two tiers are distinct, so +# the distinctness is asserted rather than reviewed. +# +# It also pins the four decisions that make the STATIC form of this tier safe, each with a +# plausible "simplification" that breaks something silently: `Retain` (bulk data pre-exists +# the cluster and has consumers outside it, so anything else makes `delete pvc` a data-loss +# operation); NO dynamic provisioning (a dynamic bulk class hands out a FRESH EMPTY +# directory that mounts and Binds perfectly while the real 8 TiB is never attached — the +# most deceptive failure available here); `hard` not `soft` (a soft mount hands a server +# hiccup to the app as a short read, which for a media library is silent corruption); and +# NFSv4 (mount and lock in-protocol ⇒ ONE reachable port, tcp/2049, which is what the +# operator must open in pfSense for prod to reach this server at all). +# +# AND the EXDEV trap, which is not about storage. Sonarr/Radarr import by hardlinking or +# atomically renaming, both same-filesystem operations, and to the kernel "same filesystem" +# means SAME MOUNT. Splitting one export into two subPath mounts looks TIGHTER and breaks +# link() with EXDEV; the arrs then degrade to copy-then-delete, rewriting every import over +# NFS and destroying the seeding copy. It surfaces weeks later as "imports are slow and my +# ratio is gone". A reviewer cannot see that; this check can. +# +# GIVEN the repository as authored, +# WHEN every `class: bulk` storage entry in catalog/ is compared against the objects the +# COMMITTED apps/ goldens carry for it (what Flux reconciles — deliberately not the +# renderer, which would only prove it agrees with itself), +# THEN (1) no bulk volume is on a Longhorn-provisioned class, and its class is explicitly +# EMPTY so nothing can dynamically provision an empty stand-in; +# (2) its capacity is declared and is at least an order of magnitude above the same +# service's replicated volume — the requirement stated as arithmetic; +# (3) PV and PVC are pre-bound from BOTH ends (claimRef + volumeName) and agree; +# (4) the PV reclaim policy is Retain; +# (5) mountOptions carry `hard` and exactly one `nfsvers=4.x`, and never `soft`; +# (6) no workload mounts one export more than once (the EXDEV trap); +# (7) a readOnly entry renders nfs.readOnly AND a readOnly volumeMount AND +# ReadOnlyMany; a writable one renders neither and is ReadWriteMany; every bulk +# access mode is a `...Many`, since one export serves several namespaces and +# several nodes; +# (8) the PV's nfs.server is a BARE cluster-vars token (never a literal — dev and +# prod reach this host by different addresses), that knob has a base default, and +# EVERY overlay supplies it; +# AND (9) the check is LOAD-BEARING (ci/l0.sh runs it unconditionally, ci/gate.sh blocks +# on the l0 stage) and NON-VACUOUS (an empty bulk corpus FAILS rather than +# passing trivially, and a floor on the assertion count keeps a pass from +# degenerating into a no-op). +# +# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library + +# PyYAML); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test wraps the +# SAME script. +# +# NOT proven — the runtime half, which needs a cluster: that the server is reachable, that +# the export admits these clients, that the PVC binds, that a write succeeds, or that a +# hardlink between the library and the download path actually works. The uid analysis is +# reasoning (docs/BULK-NFS-STORAGE.md §6) and the in-cluster probes that would settle it +# are written down in §9 of that document, unrun. Nor can this prove prod's address is +# CORRECT — only that it is a supplied knob; while it is a placeholder, prod's pods will +# not mount, and catching that statically needs the placeholder guard, which is absent from +# main (§7). Read-only on the media tree is a CLIENT-side guarantee only: the export serves +# one reader and two writers from the same node addresses, so the server cannot tell them +# apart (§5). +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: bulk-tier-distinct-from-replicated + labels: + conformance.home.k3s/R-STATE-04: "true" + annotations: + conformance.home.k3s/requirements: "R-STATE-04" + conformance.home.k3s/layer: "L0" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Bulk media storage is provably NOT on the 3-replica Longhorn tier, is sized for a + different order of magnitude, cannot be silently replaced by a dynamically + provisioned empty volume, is never reclaimable by the cluster, mounts hard over + NFSv4, and is never split into two mounts of one export — so hardlinked imports keep + working. Its server is a per-environment knob, not a literal, because the two + clusters reach the same storage host by different addresses. +spec: + steps: + - name: assert-bulk-tier-is-distinct-and-statically-bound + description: >- + Read every `class: bulk` storage entry from catalog/, pair it with the + PersistentVolume, PersistentVolumeClaim and volumeMount the committed apps/ goldens + carry for it, and assert the tier separation, the static pre-binding, the reclaim + and mount-option decisions, the read-only coupling, the one-export-one-mount + invariant, and that the server address is supplied by every environment overlay. + try: + - script: + content: bash ./check.sh diff --git a/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh b/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh new file mode 100755 index 0000000..da8dcc5 --- /dev/null +++ b/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh @@ -0,0 +1,480 @@ +#!/usr/bin/env bash +# ============================================================================= +# L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated +# Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the +# replicated tier." — spec docs/04-storage-and-state.md) +# +# WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY +# ---------------------------------------------------------------------------------------- +# `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with +# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2). +# Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is +# resolved", and each of them said, correctly, that adding `class: bulk` before then would +# leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing +# behind it, and the only thing preventing a multi-terabyte media library from being +# authored onto the 3-replica `longhorn` class was four hand-written comments. +# +# That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk +# storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the +# replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on +# `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the +# tier fills up. Distinctness is therefore a property worth asserting statically, on every +# gate run, rather than trusting to review. +# +# It also pins the four decisions that make the STATIC form of this tier safe, each of +# which has a plausible-looking "simplification" that silently breaks something: +# +# * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has +# consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an +# inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl +# delete pvc` a data-loss operation. +# * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The +# pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of +# existing data was never attached. This is the single most deceptive failure available +# in this tier, so `storageClassName` must be explicitly empty and binding must be +# pinned from BOTH ends (claimRef + volumeName). +# * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or +# EIO handed to the application. For a media library that is silent corruption. `hard` +# hangs instead — recoverable, where truncated is not. +# * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port +# (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is +# a FIREWALL fact as much as a storage one: prod reaches this server through pfSense, +# and the number of ports the operator must open is a consequence of this field. +# +# AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6). +# Sonarr and Radarr import by hardlinking or atomically renaming a finished download into +# the library. Both are same-filesystem operations, and to the kernel "same filesystem" +# means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`) +# looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no +# matter that the server knows they are one dataset. The arrs do not fail loudly on that: +# they degrade to copy-then-delete, rewriting every imported file over NFS and destroying +# the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is +# gone". A reviewer cannot see it, so this check does: one export, at most one mount, per +# workload. +# +# GIVEN the repository as authored, +# WHEN every `class: bulk` storage entry in catalog/ is compared against the manifests +# render.py produces for it (read from the committed apps/ goldens, i.e. what Flux +# actually reconciles — not from the renderer, which would only agree with itself), +# THEN (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC +# storageClassName is explicitly empty — never `longhorn`, never any class the +# repo declares with the Longhorn provisioner — and its PV's source is an +# EXTERNAL one (nfs), not a Longhorn volume; +# (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the +# bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own +# replicated volume. This is the requirement stated as arithmetic: the two tiers +# exist because the data has two magnitudes, so a "bulk" volume sized like a +# config PVC means the strategy was named and not actually applied; +# (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its +# PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and +# the two agree — so no other claim can win a race, and nothing can provision an +# empty replacement; +# (4) RECLAIM: every bulk PV is `Retain`; +# (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an +# `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`; +# (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above); +# (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH +# `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount, +# and its access mode is ReadOnlyMany; a writable one renders NEITHER and is +# ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed +# by several namespaces and several nodes, which ReadWriteOnce cannot express; +# (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk +# PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that +# knob has a base default; and EVERY environment overlay supplies it — so the two +# clusters, which reach this storage by different addresses, cannot be served by +# one hardcoded value; +# AND (9) LOAD-BEARING + NON-VACUOUS: ci/l0.sh runs this check unconditionally and +# ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT +# empty — a repo with zero bulk volumes fails here rather than passing trivially, +# because a check that is satisfiable by deleting the feature proves nothing. +# +# WHAT THIS DOES *NOT* PROVE (honest gaps) +# - Nothing about RUNTIME. Not that the server is reachable, not that the export permits +# these clients, not that the PVC binds, not that a write succeeds. The uid analysis in +# docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the +# in-cluster proof — they need a cluster and are not run here. +# - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While +# prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md +# §7); catching that needs the placeholder guard, which is absent from main. +# - Read-only for the media tree is a CLIENT-side guarantee, and this check can only +# assert the client side. The export serves a reader and two writers from the same node +# addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5). +# - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It +# catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a +# static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit. +# - Assertion (6) reasons over ONE workload's mounts. It cannot see two different +# services hardlinking between each other's mounts, which is not a thing that happens. +# ============================================================================= +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root +PYTHON="${PYTHON:-python3}" + +cd "${ROOT}" + +if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then + echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2 + exit 3 +fi + +"${PYTHON}" - "${ROOT}" <<'PY' +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", + file=sys.stderr) + raise SystemExit(1) + +ROOT = Path(sys.argv[1]) +FAILURES: list[str] = [] +CHECKS = 0 + + +def check(cond, msg): + global CHECKS + CHECKS += 1 + if not cond: + FAILURES.append(msg) + + +def bail(msg): + """A failure that makes the rest of the check meaningless: report and stop.""" + print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr) + raise SystemExit(1) + + +def qty_to_bytes(q): + """Kubernetes quantity -> bytes. Only the binary suffixes the schema permits.""" + m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip()) + if not m: + return None + return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)] + + +# ===================================================================================== +# The replicated tier, named by the repo rather than by this file. Any StorageClass the +# repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1); +# deriving it means adding a new Longhorn class cannot create a blind spot here. +# ===================================================================================== +REPLICATED_CLASSES: set[str] = set() +LONGHORN_DIR = ROOT / "platform" / "longhorn" +if LONGHORN_DIR.is_dir(): + for f in sorted(LONGHORN_DIR.rglob("*.yaml")): + for doc in yaml.safe_load_all(f.read_text()): + if isinstance(doc, dict) and doc.get("kind") == "StorageClass": + if "longhorn" in str(doc.get("provisioner", "")): + REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", "")) +# `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in +# the set even if platform/longhorn/ is not present in this checkout. +REPLICATED_CLASSES.add("longhorn") +REPLICATED_CLASSES.discard("") + +# ===================================================================================== +# The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired +# with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling +# the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is +# a separate gate stage. Asking the renderer would only prove it agrees with itself. +# ===================================================================================== +sds: dict[str, dict] = {} +for f in sorted((ROOT / "catalog").glob("*.yaml")): + doc = yaml.safe_load(f.read_text()) + if isinstance(doc, dict) and doc.get("name"): + sds[doc["name"]] = doc + +if not sds: + bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier") + +# Every rendered app-tier document, indexed by (kind, name). +rendered: dict[tuple, dict] = {} +for f in sorted((ROOT / "apps").rglob("*.yaml")): + if f.name == "kustomization.yaml" or ".sops." in f.name: + continue + try: + docs = list(yaml.safe_load_all(f.read_text())) + except yaml.YAMLError: + continue + for doc in docs: + if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"): + rendered[(doc["kind"], doc["metadata"]["name"])] = doc + +# (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2). +corpus: list[tuple[str, dict]] = [] +replicated_bytes: dict[str, list[int]] = {} +for svc, sd in sds.items(): + for vol in sd.get("storage") or []: + cls = vol.get("class", "longhorn") + if cls == "bulk": + corpus.append((svc, vol)) + else: + b = qty_to_bytes(vol.get("size")) + if b: + replicated_bytes.setdefault(svc, []).append(b) + +# ------------------------------------------------------------------- (9) NON-VACUOUS +# A corpus of zero would make every loop below trivially satisfied. The bulk tier is +# DELIVERED; a change that removes the last bulk volume must fail here and be argued for, +# not sail through on an empty iteration. +check(len(corpus) > 0, + "NO `class: bulk` storage entry exists in catalog/ — this check would then pass " + "vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME " + "stub for six roadmap phases. If the tier is genuinely being retired, retire this " + "check in the same change and say so; do not let it pass on an empty corpus") +if not corpus: + bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)") + +inventory: list[str] = [] + +for svc, vol in corpus: + vname = vol.get("name", "") + label = f"{svc}/{vname}" + obj = f"{svc}-{vname}" + ro = bool(vol.get("readOnly")) + + # `export` is what the schema requires of a bulk entry; without it there is nothing to + # attach and the rest of the assertions have no subject. + export = vol.get("export") + check(bool(export) and str(export).startswith("/"), + f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an " + f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)") + + pv = rendered.get(("PersistentVolume", obj)) + pvc = rendered.get(("PersistentVolumeClaim", obj)) + check(pv is not None, + f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk " + f"entry that renders no PV has NO external storage behind it — the pod would get " + f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory") + check(pvc is not None, + f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens") + if pv is None or pvc is None: + continue + + pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {} + + # ------------------------------------------------------ (1) DISTINCT FROM REPLICATED + for what, spec in (("PV", pvs), ("PVC", pvcs)): + sc = spec.get("storageClassName") + check(sc not in REPLICATED_CLASSES, + f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares " + f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier " + f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not " + f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)") + check(sc == "", + f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY " + f"string. Empty is what disables dynamic provisioning AND the default-class " + f"annotation. Any named class risks a provisioner handing out a fresh EMPTY " + f"directory that mounts and binds cleanly while the real data is never attached") + check("nfs" in pvs, + f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole " + f"premise is that durability lives OUTSIDE the cluster (docs/04)") + + # ------------------------------------------------------------ (4) RECLAIM POLICY + check(pvs.get("persistentVolumeReclaimPolicy") == "Retain", + f"{label}: bulk PV reclaim policy is " + f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists " + f"the cluster and has consumers outside it; anything but Retain makes `kubectl " + f"delete pvc` a data-loss operation") + + # ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS + check(pvcs.get("volumeName") == obj, + f"{label}: bulk PVC does not name its PV via volumeName (got " + f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be " + f"satisfied by some OTHER volume of matching size and mode") + cref = pvs.get("claimRef") or {} + check(cref.get("name") == obj and cref.get("namespace") == svc, + f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} " + f"— pre-binding must be pinned from BOTH ends or another PVC can win the race") + + # ------------------------------------------------------------- (5) MOUNT SAFETY + mopts = [str(o) for o in (pvs.get("mountOptions") or [])] + check(bool(mopts), + f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS " + f"version are whatever the client happens to default to, and both are decisions " + f"with consequences (silent short reads; how many firewall ports must open)") + check("hard" in mopts, + f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount " + f"hands a server hiccup to the application as a short read or EIO — silent " + f"corruption for a media library. `hard` hangs instead, which is recoverable") + check("soft" not in mopts, + f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the " + f"one option that turns an outage into damaged data") + vers = [o for o in mopts if o.startswith("nfsvers=")] + check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]), + f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). " + f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port " + f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the " + f"operator has to open in pfSense for prod to reach this server at all") + + # ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT + want_mode = "ReadOnlyMany" if ro else "ReadWriteMany" + for what, spec in (("PV", pvs), ("PVC", pvcs)): + modes = spec.get("accessModes") or [] + check(modes == [want_mode], + f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk " + f"volume is a `...Many` (one export, several namespaces, several nodes — " + f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write " + f"posture stays legible in `kubectl get pv`") + nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly")) + check(nfs_ro == ro, + f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. " + f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared " + f"posture is not the one the kernel enforces") + + # The volumeMount half, read out of the rendered Deployment. + dep = rendered.get(("Deployment", svc)) + check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against") + if dep is not None: + containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or [] + # The app container is containers[0] by the renderer's contract (a forward-auth + # sidecar is appended AFTER it). + mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else [] + check(len(mounts) == 1, + f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, " + f"expected exactly 1 — a bulk volume that is declared and never mounted is a " + f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)") + if len(mounts) == 1: + m = mounts[0] + check(bool(m.get("readOnly")) == ro, + f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the " + f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, " + f"so that dropping one still leaves the other") + check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"), + f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the " + f"declared subPath {vol.get('subPath')!r}") + + # ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV + server = (pvs.get("nfs") or {}).get("server") + m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or "")) + check(m is not None, + f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars " + f"token, not a literal address. dev and prod reach this storage host by DIFFERENT " + f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct " + f"in at most one environment and silently wrong in the other (R-ENV-02)") + + cap = (pvs.get("capacity") or {}).get("storage") + req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage") + check(cap is not None and qty_to_bytes(cap) is not None, + f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, " + f"and it is what makes `kubectl get pv` report something true about this tier") + check(cap == req, + f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the " + f"capacity never binds") + + inventory.append( + f" {label:<28} export={export} {'ro' if ro else 'rw'} {want_mode:<14} " + f"cap={cap} server={server} opts={','.join(mopts)}" + ) + + # ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes + cap_b = qty_to_bytes(cap) + reps = replicated_bytes.get(svc) or [] + if cap_b and reps: + biggest = max(reps) + # ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed: + # the media library is ~460x its service's config volume, and the Nextcloud archive + # ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate + # bulk volume, and this check would be teaching people to weaken it. 10x cannot be + # met by accident and still catches the regression that matters: a "bulk" volume + # sized like a config PVC, i.e. the strategy named but never applied. + check(cap_b >= 10 * biggest, + f"{label}: bulk capacity {cap} is less than 10x this service's largest " + f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the " + f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps " + f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A " + f"bulk volume sized like a config volume means the tier was named, not applied") + +# ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT +# The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod. +by_service: dict[str, list[dict]] = {} +for svc, vol in corpus: + by_service.setdefault(svc, []).append(vol) +for svc, vols in sorted(by_service.items()): + seen: dict[str, str] = {} + for vol in vols: + exp = vol.get("export") + prev = seen.get(exp) + check(prev is None, + f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and " + f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the " + f"kernel, so link() between them fails EXDEV even though the server knows " + f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they " + f"degrade to copy-then-delete, rewriting every imported file over NFS and " + f"destroying the seeding copy. Mount the common ancestor ONCE instead " + f"(docs/BULK-NFS-STORAGE.md §4)") + seen[exp] = vol.get("name") + +# ------------------------------------- (8b) the knob has a base default AND every overlay +tokens = set() +for _svc, vol in corpus: + pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}")) + if pv: + mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or "")) + if mm: + tokens.add(mm.group(1)) + +cv = ROOT / "clusters" / "base" / "cluster-vars.yaml" +base_keys: set[str] = set() +if cv.is_file(): + doc = yaml.safe_load(cv.read_text()) or {} + base_keys = set((doc.get("data") or {}).keys()) + +overlays = sorted( + d for d in (ROOT / "clusters").iterdir() + if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file() +) if (ROOT / "clusters").is_dir() else [] + +for tok in sorted(tokens): + check(tok in base_keys, + f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no " + f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without " + f"it the token ships into the cluster as a literal server name") + for ov in overlays: + text = (ov / "kustomization.yaml").read_text() + check(f"/data/{tok}" in text, + f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV " + f"value by this repo's convention, so an overlay that omits it silently " + f"inherits an address on the WRONG SIDE of pfSense — a mount that either " + f"never connects or connects to something unintended (R-ENV-02)") + +# --------------------------------------------------------------- (9) LOAD-BEARING +L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else "" +GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else "" +check("bulk-tier-distinct-from-replicated/check.sh" in L0, + "ci/l0.sh does not run bulk-tier-distinct-from-replicated/check.sh. Stage 3b scopes " + "the conformance run to TOUCHED requirements, and ci/touched-requirements.sh maps " + "neither apps/* nor platform/longhorn/* to the bulk requirement — so a change that " + "moves a bulk volume onto the replicated tier could attribute no requirement at all " + "and be gated by nothing. It must be invoked directly and unconditionally") +check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE), + "ci/gate.sh does not set the merge block when the l0 stage fails") + +# ------------------------------------------------------------------------- report +print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):") +for line in inventory: + print(line) +print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): " + f"{', '.join(sorted(REPLICATED_CLASSES))}") +print(f"bulk volumes: {len(corpus)} distinct exports: " + f"{len({v.get('export') for _s, v in corpus})} " + f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}") + +# A floor on the assertion count, for the same reason the corpus may not be empty: a pass +# must not be able to degenerate into "nothing was actually asked". +MIN_CHECKS = 40 +check(CHECKS >= MIN_CHECKS, + f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not " + f"evidence; the corpus or the extraction has degraded") + +print() +if FAILURES: + print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)", + file=sys.stderr) + for f in FAILURES: + print(f" - {f}", file=sys.stderr) + raise SystemExit(1) + +print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)") +PY diff --git a/docs/BULK-NFS-STORAGE.md b/docs/BULK-NFS-STORAGE.md new file mode 100644 index 0000000..b9af86f --- /dev/null +++ b/docs/BULK-NFS-STORAGE.md @@ -0,0 +1,337 @@ +# Bulk NFS storage — the media library and the legacy Nextcloud archive + +The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`. +Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is +assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the +case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk +provisioning unimplemented, because nothing needs it. + +--- + +## 1. The storage host, and why prod cannot reach it + +The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface: + +``` +vmbr0 192.168.1.9/24 default via 192.168.1.1 (pfSense LAN) +``` + +| | nodes | reaches 192.168.1.9? | +|---|---|---| +| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) | +| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. | + +Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give +different values for the `BULK_NFS_SERVER` cluster-var: + +| outcome | `BULK_NFS_SERVER` for prod | +|---|---| +| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` | +| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` | + +Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder +behaves differently from the repo's other ones.** + +### It only needs ONE port + +The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that +must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed +four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The +server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the +conservative floor. + +### The export's client list is the SAME under both outcomes — measured, not assumed + +**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation +(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`: + +``` +SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22 +ss -tn -> 192.168.1.9:22 192.168.2.132:39768 +``` + +The original WAN-side source address survives the translation (return traffic finds its way back via +the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's +**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either +way. **It does not have to wait for the routing decision.** + +--- + +## 2. The layout, as found + +### `Storage/plex` → `/mnt/Storage/plex` — 8.33 T used, 6.91 T avail + +``` +/mnt/Storage/plex/ drwxr-xr-x 1000:3000 +├── downloads/ drwxr-xr-x 1000:1000 +│ ├── complete/ drwxr-xr-x 1000:3000 +│ │ ├── radarr/ drwxr-xr-x 1000:3000 +│ │ └── tv-sonarr/ drwxr-xr-x 1000:3000 +│ ├── incomplete/ drwxrwxr-x 1000:3000 +│ ├── downloads -> /shared/downloads (symlink OUTSIDE the export — see §8) +│ └── drwxr-xr-x 1000:1000 +├── movies/ 1165 entries drwxr-xr-x 1000:3000 +├── tv/ 107 entries drwxr-xr-x 1000:3000 +└── photos/ drwxr-xr-x 1000:3000 +``` + +**The single most important fact about this layout: `downloads/` is a SIBLING of `tv/` and `movies/` +inside the same dataset.** Sonarr's import source and its library are on one filesystem on the +server, and so are Radarr's. That is what makes hardlinked / atomically-renamed imports possible at +all — and §4 is about not throwing it away on the client side. + +Note the download client's completed folders are already named `radarr` and `tv-sonarr`, i.e. this +tree has been driven by an arr stack for a long time. The k3s arrs are joining an existing layout, +not defining one. + +### `Storage/nextcloud` → `/mnt/Storage/nextcloud` — 270 G used + +``` +/mnt/Storage/nextcloud/ drwxr-xr-x root:root +└── nextcloud/ drwxr-xr-x 1000:3000 + ├── config/ 103 M drwxr-xr-x 1000:3000 + ├── config_old/ 3.4 G drwxr-xr-x 1000:3000 + └── config_backup/ 267 G drwxr-xr-x 1000:3000 +``` + +**There is no `data/` directory.** 99% of the dataset is `config_backup`. This is the old, non-k3s +Nextcloud's configuration plus a backup blob — an **archive**, not a live user-data tree. That is +why it is mounted read-only; see §6. + +--- + +## 3. What each app mounts, and why + +| app | export | mountPath | subPath | mode | access mode | +|---|---|---|---|---|---| +| plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany | +| sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany | +| radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany | +| nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany | + +In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr +Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from +`/media/downloads/complete/{tv-sonarr,radarr}`. + +**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually +given the same export (it is not in this change), the arrs and the download client will already agree +on absolute paths, which is the second precondition for hardlinked imports — otherwise every import +needs a Remote Path Mapping and quietly falls back to copying when one is missing. + +**`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the +renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the +config volume entirely. See §8 for the separate latent bug that discovery exposed. + +--- + +## 4. Why the arrs mount the export ROOT, not two subPaths + +This is the design decision most likely to be "simplified" later by someone who has not read this. + +Sonarr and Radarr import a finished download into the library by **hardlinking** it (preserving the +seeding copy) or by an **atomic rename**. Both are same-filesystem operations, and **to the kernel, +"same filesystem" means "same mount."** + +The tempting narrower shape is two subPath volumes — `subPath: tv` and `subPath: downloads`. Those +are **two separate mounts** of the same export. `link()` across them returns **EXDEV** no matter that +the server knows perfectly well they are one dataset. Sonarr does not fail loudly at that: it +degrades to copy-then-delete. Every import then rewrites the entire file over NFS and destroys the +seeding copy. The symptom is "imports are slow and my ratio is gone", weeks later. + +The smallest subtree containing both `tv/` and `downloads/` **is the export root**. So the root mount +is the *minimal* mount that preserves hardlinks: least privilege bounded below by a hard requirement, +not a shortcut. + +The cost is stated rather than hidden: Sonarr can also write `/media/movies` and Radarr `/media/tv`. +Narrowing that would cost hardlinks, which is the worse trade. + +--- + +## 5. The export change the operator must apply — STRICTLY ADDITIVE + +**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the +exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and +192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the +live non-k3s prod stack. + +Current values (`zfs get sharenfs Storage`): + +``` +Storage/plex rw=@192.168.1.6:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash +Storage/nextcloud rw=@192.168.1.6:@192.168.1.5,no_subtree_check,no_root_squash +``` + +### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list + +```sh +zfs set sharenfs='rw=@192.168.1.6:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex +``` + +Every pre-existing host is preserved verbatim and no option changes; only clients are added. + +### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=` + +```sh +zfs set sharenfs='rw=@192.168.1.6:@192.168.1.5,ro=@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud +``` + +This is the one place read-only can be enforced **server-side**, and it is worth taking: only +Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a +misconfigured client cannot then damage the archive. + +**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but +all three may be scheduled on any node, so the server sees the same source addresses for all of +them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a +client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an +honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a +dedicated node pool for Plex, which is a much larger change. + +Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host +(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive +adding a node; it also grants the whole subnet. Per-node is the tighter default. + +### Verifying, without changing anything + +```sh +exportfs -v # the new client lines appear, existing ones unchanged +zfs get sharenfs Storage/plex Storage/nextcloud +``` + +--- + +## 6. uid / gid — the failure this design exists to avoid + +The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through +verbatim** and the server's on-disk ownership is the entire access story. The media tree is +`0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**. + +**`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS +volume plugin performs **no ownership management**, so `podSecurityContext.fsGroup` is a **no-op on +an NFS volume**. There is no recursive chown, no `g+rwX`, nothing. The container's own uid has to be +right. + +| app | container uid | writes? | outcome | +|---|---|---|---| +| plex | 1000 (`PLEX_UID`/`PLEX_GID`, already set) | no | reads fine — the tree is 0755, so world-readable anyway | +| sonarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine | +| radarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine | +| nextcloud | 33 (`www-data`) | no | reads and traverses 0755 fine | + +**Without the added `PUID`/`PGID`, sonarr and radarr would have failed exactly the way this tier +always fails: the mount succeeds, the pod is Ready, and every import fails EACCES.** LinuxServer +images default to uid/gid **911**, which owns nothing on that tree. + +`PGID: 3000` is chosen over 1000 so files these write are group-owned by the dominant group already +on the tree, keeping them readable to the live non-k3s consumers. + +`podSecurityContext.fsGroup: 3000` is also added to sonarr and radarr — **for the Longhorn config +PVC, not for the NFS mount.** fsGroup does work on a Longhorn filesystem, and it makes `/data/config` +writable by the now-uid-1000 process without depending on the image re-chowning it after `PUID` +changed. + +Nextcloud needs no uid work at all, which is one more reason its mount is read-only. + +--- + +## 7. The prod placeholder does NOT fail at apply time — read this before merging + +Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation +(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.** + +A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts +`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure +control-plane and never touches the server. The failure appears only when a pod mounts: + +``` +MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER +``` + +so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.** + +**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a +revision that carries these bulk volumes.** Dev renders a working value and is where these manifests +are meant to be proven first. + +### The static guard this convention leans on is missing from `main` + +`bootstrap/standup.sh` — which carries the `*_is_placeholder` guard family — **does not exist on +`main`** (verified: `git ls-tree main bootstrap/` lists only `README.md`, `flux/`, `gitea/`, `sops/`; +the file lives on unmerged prod branches). So +`conformance/tests/l0-static/placeholder-guard-coupling/check.sh` exits **3 SKIPPED**, and **nothing +in the gate currently refuses a placeholder-shaped overlay value.** + +When `standup.sh` lands, that check's assertions (2) and (3) cover this knob automatically — its +independent detector's SEED vocabulary already matches `TBD` — **provided** the guard gains an arm +for `BULK_NFS_SERVER` and a consumer that calls it. Without the arm, assertion (2) is what fails, by +design. + +--- + +## 8. Loose ends and things deliberately not done + +1. **`/data/config` is very likely the wrong home for the arrs' config PVC — a pre-existing latent + bug, not fixed here.** LinuxServer's arr images run with `-data=/config`, so the app writes + `/config`, which is the container's **ephemeral layer**, while the Longhorn PVC mounted at + `/data/config` sits unused. This is the exact failure `catalog/plex.yaml`'s `mountPath` comment + records having already been bitten by ("the PVC sat unused and Plex wrote to the container's + EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it + means those apps start from an empty config volume, so it belongs in its own change with the + operator present. **This change does not make it worse** — that is why the media mount is `/media`. +2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy + non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s + qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were + chosen so it needs no Remote Path Mapping. +3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside + the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log + it while scanning `downloads/`. +4. **`nfsvers=4.2` was not chosen**, though the server advertises it. Nothing in these workloads uses + what 4.2 adds (server-side copy, sparse-file `SEEK`/`ALLOCATE`), and 4.1 is the version the + existing `bulk` StorageClass stub already named. +5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning + change that should be measured, not guessed, and it changes client behaviour under server + failure. +6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this + workstation (browser-OIDC only), and this change is manifest-authoring under an explicit + do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and + mounting* is reasoned from the manifests. +7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.** + `ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the + in-cluster conformance test for the bulk tier is still a gap, so typing it would select a + requirement the gate cannot satisfy — the hazard `catalog/qbittorrent.yaml` already warns about. + Scaffolding that test needs a cluster and is the correct follow-up. + +--- + +## 9. Validating on dev, once the export is extended + +Dev can prove all of this before prod's network path exists. Read-only checks first: + +```sh +infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv +infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive' +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide +``` + +Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three +of them naming the *same* export, one per namespace, which is the cross-namespace model) — each +`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`. + +Then the two things that actually matter, from inside a pod: + +```sh +# (a) the writer can write — this is the uid check, and it is the one that fails in practice +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \ + sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK' + +# (b) hardlinks work BETWEEN the library and the download path — the root-mount check +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \ + sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link; + echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK' + +# (c) the reader really is read-only +infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \ + sh -c 'touch /media/.probe 2>&1 | head -1' # expect "Read-only file system" +``` + +`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it +prints `EXDEV`, something has split the single mount back into two. diff --git a/schemas/renderer-contract.md b/schemas/renderer-contract.md index 8e647a6..01d53a7 100644 --- a/schemas/renderer-contract.md +++ b/schemas/renderer-contract.md @@ -72,11 +72,48 @@ platform must render, and cites the requirement each mapping satisfies. Changing - **Requirements:** **R-NET-01** (default-deny baseline), **R-NET-02** (explicit allow / deny undeclared). Auditable via Hubble (R-SEC-03, R-OBS-04). -### `storage[]` (`name` / `size` / `class`) — PVCs +### `storage[]` (`name` / `size` / `class` / `mountPath` / `export` / `subPath` / `readOnly`) — PVs + PVCs - `class: longhorn` → a **PVC** on the 3-replica Longhorn StorageClass sized by `size` (replicated - config/DB/app state). For a `vm`, the disk is a Longhorn volume via CDI. -- `class: bulk` → binding to the external **bulk** media tier (NFS/SMB indirection, sized outside the - cluster; `size` optional). Not 3× replicated. + config/DB/app state), `ReadWriteOnce`, dynamically provisioned. For a `vm`, the disk is a Longhorn + volume via CDI. +- `class: bulk` → a **static, pre-bound external NFS PersistentVolume _and_ its PVC**. The entry's + `export` is the absolute path on the bulk server; the SERVER is never named in a ServiceDefinition + — it is the `${BULK_NFS_SERVER}` cluster-vars knob, so one authored definition promotes across + environments that reach the same storage by different addresses (**R-ENV-02**). Not 3× replicated: + durability is the storage host's ZFS pool, not Longhorn. + - `storageClassName: ""` on **both** PV and PVC (explicitly empty ⇒ statically provisioned; no + provisioner is ever asked to create anything, and the default-StorageClass annotation is + suppressed). Deliberately NOT the `bulk` StorageClass: a StorageClass `provisioner` is + **immutable**, so re-pointing that stub would make Flux fail permanently on it. + - Binding is pinned from both ends — `claimRef` on the PV, `volumeName` on the PVC — so no other + claim of matching size/mode can win a race for the volume. + - `persistentVolumeReclaimPolicy: Retain`, always. Bulk data pre-exists the cluster, is far larger + than the cluster, and may have consumers outside it; nothing here may reclaim it. + - Access mode follows the read/write posture and is always `…Many`, because one export is + consumed by several namespaces and several nodes at once: `ReadOnlyMany` when `readOnly`, + `ReadWriteMany` otherwise. `ReadWriteOnce` cannot express either. + - `mountOptions` are a **platform** decision held in the renderer, not a per-service field: + `nfsvers=4.1,hard,noatime,timeo=600,retrans=2`. NFSv4 carries mount and lock in-protocol, so + the only port that must be reachable is **tcp/2049** — no rpcbind/111, mountd or statd, which is + what keeps the firewall surface to a single rule. `hard` is chosen over `soft` because a soft + mount converts a server hiccup into a short read handed to the application. + - `size` on a bulk entry is the PV's **advertised capacity and nothing more** — no component + enforces it (the ZFS pool is the real limit), so it should state the dataset's actual size so + `kubectl get pv` reports something true. + - **fsGroup does not apply.** The in-tree NFS volume plugin performs no ownership management, so + `podSecurityContext.fsGroup` is a **no-op** on a bulk volume. A bulk mount that must be written + therefore requires the container's own uid/gid to be one the export's ownership permits (e.g. + Plex's `PLEX_UID`/`PLEX_GID`, LinuxServer images' `PUID`/`PGID`) — or a server-side squash. A + mount that succeeds but cannot write is the characteristic failure of this tier. +- `mountPath` (any class) overrides the `/data/` default verbatim. +- `subPath` (any class) mounts a subtree instead of the volume root. **Two `subPath` mounts of one + export are two separate mounts**, so `link()` between them fails `EXDEV`; a workload that needs + hardlinks or atomic moves between two directories (Sonarr/Radarr importing from a download path + into a library) must mount their common ancestor **once** instead. +- `readOnly` (any class) marks the volumeMount read-only, and on a bulk entry also sets the PV's + `nfs.readOnly` so the kernel mount really is `ro`. Note this is a **client-side** guarantee: one + export serving both a reader and a writer that may land on any node cannot distinguish them + server-side. - State separation rule: anything that must survive a rebuild lives on a named volume here, never on ephemeral fs/hostPath — so the workload can be destroyed and re-applied from git with data intact. - **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication), diff --git a/schemas/servicedefinition.schema.json b/schemas/servicedefinition.schema.json index 449e980..20ce03b 100644 --- a/schemas/servicedefinition.schema.json +++ b/schemas/servicedefinition.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json", "title": "ServiceDefinition", - "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.3.0 (adds optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", + "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", "type": "object", "additionalProperties": false, "required": ["name", "source"], @@ -153,7 +153,7 @@ }, "storage": { "type": "array", - "description": "Persistent volumes. If it must survive a rebuild it lives here, never on ephemeral/hostPath (docs/04; R-STATE-01/02/04).", + "description": "Persistent volumes. If it must survive a rebuild it lives here, never on ephemeral/hostPath (docs/04; R-STATE-01/02/04). A `longhorn` entry renders a dynamically provisioned PVC; a `bulk` entry renders a STATIC, pre-bound NFS PersistentVolume + its PVC over data that already exists outside the cluster.", "default": [], "items": { "type": "object", @@ -167,7 +167,7 @@ }, "size": { "type": "string", - "description": "PVC request in Kubernetes quantity form (e.g. 5Gi). Required for the longhorn tier; may be omitted for the bulk tier (external mount, sized outside the cluster - docs/04).", + "description": "PVC request in Kubernetes quantity form (e.g. 5Gi). Required for the longhorn tier. For the bulk tier it is the STATIC PV's advertised capacity: NOTHING enforces it (the ZFS pool is the real limit), so state the dataset's current size so `kubectl get pv` reports something true. Omitted => the bulk PV/PVC carry no capacity, which the API rejects, so bulk in practice sets it.", "pattern": "^[1-9][0-9]*(Mi|Gi|Ti)$" }, "mountPath": { @@ -177,11 +177,33 @@ }, "class": { "type": "string", - "description": "longhorn = 3-replica block storage for app/config/DB state (R-STATE-02); bulk = large external media tier, NOT 3x replicated (R-STATE-04).", + "description": "longhorn = 3-replica block storage for app/config/DB state (R-STATE-02); bulk = large external media tier, NOT 3x replicated, backed by a STATIC NFS PersistentVolume the renderer emits (requires `export`).", "enum": ["longhorn", "bulk"], "default": "longhorn" + }, + "export": { + "type": "string", + "description": "bulk ONLY: the absolute NFS export path on the bulk server (e.g. /mnt/Storage/plex). The SERVER is NOT named here - it is the ${BULK_NFS_SERVER} cluster-vars knob, so the same ServiceDefinition promotes across environments (R-ENV-02). Required whenever class is bulk; forbidden otherwise.", + "pattern": "^/.*[^/]$" + }, + "subPath": { + "type": "string", + "description": "Optional relative path WITHIN the volume to mount instead of its root. NOTE for hardlink/atomic-move workloads (Sonarr/Radarr): two subPath mounts of the same export are two separate mounts, so link() across them fails EXDEV. Mount the common ancestor once instead.", + "pattern": "^[^/].*[^/]$" + }, + "readOnly": { + "type": "boolean", + "description": "Mount this volume read-only. For a bulk volume this also sets the PersistentVolume's nfs.readOnly (the mount really is `ro`) and makes its access mode ReadOnlyMany instead of ReadWriteMany.", + "default": false } - } + }, + "allOf": [ + { + "if": { "properties": { "class": { "const": "bulk" } }, "required": ["class"] }, + "then": { "required": ["export"] }, + "else": { "not": { "required": ["export"] } } + } + ] } }, "dependencies": {