Home-Network/home-network-k3s-impl
2
0 Fork 0
Code Issues 2 Pull Requests 5 Actions Packages Projects Releases Wiki Activity

merge: bulk NFS volumes for plex, sonarr, radarr and nextcloud
gate / gate (push) Has been cancelled
Details
gate / conformance (push) Has been cancelled
Details

Browse Source
This commit is contained in:
jakman85 committed 2026-09-12 16:26:31 -05:00
2 parents cac413e87a daac05e7e8
commit f5e62d9169
18 files changed
+1698 -40
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/wave-0/plex.yaml
+51
View File
@@ -35,6 +35,34 @@ spec:
- protocol: TCP
port: 53
---
# PersistentVolume plex-media - static external NFS (bulk tier)
apiVersion: v1
kind: PersistentVolume
metadata:
name: plex-media
labels:
app.kubernetes.io/name: plex
spec:
capacity:
storage: 9Ti
accessModes:
- ReadOnlyMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ''
mountOptions:
- nfsvers=4.1
- hard
- noatime
- timeo=600
- retrans=2
nfs:
server: ${BULK_NFS_SERVER}
path: /mnt/Storage/plex
readOnly: true
claimRef:
namespace: plex
name: plex-media
---
# PVC plex-config (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
@@ -51,6 +79,23 @@ spec:
requests:
storage: 8Gi
---
# PVC plex-media (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: plex-media
namespace: plex
labels:
app.kubernetes.io/name: plex
spec:
accessModes:
- ReadOnlyMany
storageClassName: ''
volumeName: plex-media
resources:
requests:
storage: 9Ti
---
# Deployment (R-PLAT-02)
apiVersion: apps/v1
kind: Deployment
@@ -90,10 +135,16 @@ spec:
volumeMounts:
- name: config
mountPath: /config
- name: media
mountPath: /media
readOnly: true
volumes:
- name: config
persistentVolumeClaim:
claimName: plex-config
- name: media
persistentVolumeClaim:
claimName: plex-media
---
# Service
apiVersion: v1
apps/wave-1/nextcloud.yaml
+52
View File
@@ -35,6 +35,34 @@ spec:
- protocol: TCP
port: 53
---
# PersistentVolume nextcloud-legacy-archive - static external NFS (bulk tier)
apiVersion: v1
kind: PersistentVolume
metadata:
name: nextcloud-legacy-archive
labels:
app.kubernetes.io/name: nextcloud
spec:
capacity:
storage: 270Gi
accessModes:
- ReadOnlyMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ''
mountOptions:
- nfsvers=4.1
- hard
- noatime
- timeo=600
- retrans=2
nfs:
server: ${BULK_NFS_SERVER}
path: /mnt/Storage/nextcloud
readOnly: true
claimRef:
namespace: nextcloud
name: nextcloud-legacy-archive
---
# PVC nextcloud-data (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
@@ -51,6 +79,23 @@ spec:
requests:
storage: 20Gi
---
# PVC nextcloud-legacy-archive (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nextcloud-legacy-archive
namespace: nextcloud
labels:
app.kubernetes.io/name: nextcloud
spec:
accessModes:
- ReadOnlyMany
storageClassName: ''
volumeName: nextcloud-legacy-archive
resources:
requests:
storage: 270Gi
---
# Deployment (R-PLAT-02)
apiVersion: apps/v1
kind: Deployment
@@ -119,6 +164,10 @@ spec:
volumeMounts:
- name: data
mountPath: /var/www/html
- name: legacy-archive
mountPath: /mnt/legacy-nextcloud
subPath: nextcloud
readOnly: true
lifecycle:
postStart:
exec:
@@ -132,6 +181,9 @@ spec:
- name: data
persistentVolumeClaim:
claimName: nextcloud-data
- name: legacy-archive
persistentVolumeClaim:
claimName: nextcloud-legacy-archive
---
# Service
apiVersion: v1
apps/wave-1/radarr.yaml
+58
View File
@@ -35,6 +35,33 @@ spec:
- protocol: TCP
port: 53
---
# PersistentVolume radarr-media - static external NFS (bulk tier)
apiVersion: v1
kind: PersistentVolume
metadata:
name: radarr-media
labels:
app.kubernetes.io/name: radarr
spec:
capacity:
storage: 9Ti
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ''
mountOptions:
- nfsvers=4.1
- hard
- noatime
- timeo=600
- retrans=2
nfs:
server: ${BULK_NFS_SERVER}
path: /mnt/Storage/plex
claimRef:
namespace: radarr
name: radarr-media
---
# PVC radarr-config (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
@@ -51,6 +78,23 @@ spec:
requests:
storage: 5Gi
---
# PVC radarr-media (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: radarr-media
namespace: radarr
labels:
app.kubernetes.io/name: radarr
spec:
accessModes:
- ReadWriteMany
storageClassName: ''
volumeName: radarr-media
resources:
requests:
storage: 9Ti
---
# Deployment (R-PLAT-02)
apiVersion: apps/v1
kind: Deployment
@@ -81,9 +125,18 @@ spec:
image: lscr.io/linuxserver/radarr:5.21.1
ports:
- containerPort: 7878
env:
- name: PUID
value: '1000'
- name: PGID
value: '3000'
- name: TZ
value: Etc/UTC
volumeMounts:
- name: config
mountPath: /data/config
- name: media
mountPath: /media
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
imagePullPolicy: IfNotPresent
@@ -148,10 +201,15 @@ spec:
port: http
initialDelaySeconds: 5
periodSeconds: 15
securityContext:
fsGroup: 3000
volumes:
- name: config
persistentVolumeClaim:
claimName: radarr-config
- name: media
persistentVolumeClaim:
claimName: radarr-media
---
# Service
apiVersion: v1
apps/wave-1/sonarr.yaml
+58
View File
@@ -35,6 +35,33 @@ spec:
- protocol: TCP
port: 53
---
# PersistentVolume sonarr-media - static external NFS (bulk tier)
apiVersion: v1
kind: PersistentVolume
metadata:
name: sonarr-media
labels:
app.kubernetes.io/name: sonarr
spec:
capacity:
storage: 9Ti
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ''
mountOptions:
- nfsvers=4.1
- hard
- noatime
- timeo=600
- retrans=2
nfs:
server: ${BULK_NFS_SERVER}
path: /mnt/Storage/plex
claimRef:
namespace: sonarr
name: sonarr-media
---
# PVC sonarr-config (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
@@ -51,6 +78,23 @@ spec:
requests:
storage: 5Gi
---
# PVC sonarr-media (R-STATE)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: sonarr-media
namespace: sonarr
labels:
app.kubernetes.io/name: sonarr
spec:
accessModes:
- ReadWriteMany
storageClassName: ''
volumeName: sonarr-media
resources:
requests:
storage: 9Ti
---
# Deployment (R-PLAT-02)
apiVersion: apps/v1
kind: Deployment
@@ -81,9 +125,18 @@ spec:
image: lscr.io/linuxserver/sonarr:4.0.15
ports:
- containerPort: 8989
env:
- name: PUID
value: '1000'
- name: PGID
value: '3000'
- name: TZ
value: Etc/UTC
volumeMounts:
- name: config
mountPath: /data/config
- name: media
mountPath: /media
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
imagePullPolicy: IfNotPresent
@@ -148,10 +201,15 @@ spec:
port: http
initialDelaySeconds: 5
periodSeconds: 15
securityContext:
fsGroup: 3000
volumes:
- name: config
persistentVolumeClaim:
claimName: sonarr-config
- name: media
persistentVolumeClaim:
claimName: sonarr-media
---
# Service
apiVersion: v1
catalog/nextcloud.yaml
+38
View File
@@ -223,6 +223,44 @@ storage:
mountPath: /var/www/html # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02).
# The whole html dir is persisted so config.php + user data survive a
# rebuild (the default /data/<name> would strand them on ephemeral rootfs).
# THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND
# NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision.
#
# WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed):
# /mnt/Storage/nextcloud/nextcloud/config 103 M
# /mnt/Storage/nextcloud/nextcloud/config_old 3.4 G
# /mnt/Storage/nextcloud/nextcloud/config_backup 267 G <- 99% of the dataset
# There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup
# blob — an ARCHIVE, not a live user-data tree.
#
# WHY READ-ONLY. Two independent reasons, either sufficient:
# 1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and
# 192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two
# Nextcloud instances writing one instance's config directory, each with its own database and
# therefore its own oc_filecache, is a corruption scenario with no clean recovery.
# 2. Nothing here needs writing. An archive is read.
# "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a
# MIGRATION (read this data in, then decommission the old instance), that is a different and
# larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the
# export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount.
#
# WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own
# state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does
# not own. A neutral path keeps the two completely separate. To expose it to users, add it as a
# Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision.
#
# subPath skips the redundant single-child directory: the dataset root holds exactly one entry,
# `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top.
#
# UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data
# (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.)
- name: legacy-archive
class: bulk
export: /mnt/Storage/nextcloud # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER}
subPath: nextcloud
mountPath: /mnt/legacy-nextcloud
readOnly: true
size: 270Gi # ADVISORY ONLY on a static NFS PV — the dataset's used size.
dependencies:
- to: nextcloud-db # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02).
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
catalog/plex.yaml
+30 -15
View File
@@ -4,21 +4,11 @@
# auth: native (Plex accounts, NOT Keycloak — docs/13 + docs/15 documented exception).
# Validates against schemas/servicedefinition.schema.json.
#
# BULK MEDIA (spec bulk-tier requirement, docs/04) — OPEN PREREQUISITE, intentionally NOT declared
# here yet (the requirement ID is cited in the WS-APPS report, not as a token here: this change does
# NOT exercise the bulk tier, so it must not be selected as a "touched" requirement whose in-cluster
# conformance test is still a Phase-1 gap).
# The Plex library belongs on the `bulk` tier (docs/04 / docs/13 cross-cutting note: metadata DB
# on replicated Longhorn, large library on bulk). The `bulk` StorageClass exists
# (platform/longhorn/storageclasses.yaml) and the schema+renderer both support `class: bulk`, but
# that class is a documented STUB: OQ-2 (external NFS/SMB backing) is UNRESOLVED and the NFS CSI
# driver is not installed, so a `bulk` PVC will not bind and would leave the Plex pod Pending.
# To onboard the running Plex shell now (compute + config + public exposure), the media volume is
# omitted. Once OQ-2 is resolved and `bulk` provisions, attach the library with a one-line add:
# storage:
# - name: media
# class: bulk # external mount; size omitted (sized outside the cluster)
# then re-render. See the WS-APPS report for the full bulk-media finding.
# BULK MEDIA — NOW ATTACHED (docs/04 bulk tier; the long-standing OQ-2 prerequisite, resolved for
# the STATIC case). The library is 8.33 T of ZFS on the Proxmox host at /mnt/Storage/plex, and it
# already exists: nothing is provisioned, an existing dataset is mounted. See
# docs/BULK-NFS-STORAGE.md for the layout, the export change and the uid analysis. (The bulk-tier
# requirement ID is deliberately still not typed here — see that document's "gate" note.)
name: plex
kind: container
source:
@@ -70,6 +60,31 @@ storage:
# UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token
# (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`,
# so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
# THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies,
# /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`:
# downloads/ movies/ photos/ tv/).
#
# WHY THE ROOT AND NOT THREE subPath MOUNTS. Nothing is gained by narrowing: the mount is `ro`,
# so no subPath could remove a privilege that matters, while three mounts would triple the NFS
# mounts per pod and mean a manifest change every time a top-level library is added.
#
# WHY READ-ONLY, when the export is `rw` and the other consumers write. Plex is a SERVER: it
# indexes and streams. Every write on that tree belongs to Sonarr and Radarr. Marking it here
# costs nothing and removes Plex from the set of things that can damage 8.33 T. Enforced by the
# PV's nfs.readOnly (a real `ro` mount), not only by the volumeMount flag.
#
# UID: no change needed. The tree is 0755 owned 1000:3000 throughout, so it is world-readable and
# world-traversable, and PLEX_UID/PLEX_GID above already make the process uid 1000 in any case.
# (fsGroup would have been a no-op here regardless — the in-tree NFS plugin does no ownership
# management. That is the trap the arrs' PUID/PGID exists to avoid; Plex never hits it.)
- name: media
class: bulk
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
mountPath: /media
readOnly: true
size: 9Ti # ADVISORY ONLY (nothing enforces a static NFS PV's capacity): the
# dataset's 8.33 T used, rounded up, so `kubectl get pv` reports
# something true. The real limit is the ZFS pool.
dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
ingressRoutes:
- host: plex.${BASE_DOMAIN}
catalog/radarr.yaml
+66 -5
View File
@@ -9,6 +9,23 @@ name: radarr
kind: container
source:
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
# which cannot write a 1000:3000-owned 0755 tree.
env:
- name: PUID
value: "1000" # owner uid of every directory under /mnt/Storage/plex
- name: PGID
value: "3000" # dominant group there; keeps files this writes readable to the
# EXISTING non-k3s consumers of the same export, which stay live
- name: TZ
value: Etc/UTC # same safe default plex uses; the arrs schedule by local time
podSecurityContext:
# For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
# does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
# what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
# /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
fsGroup: 3000
exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET)
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
@@ -16,11 +33,55 @@ storage:
- name: config
size: 5Gi
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
# NOTE (bulk-media PREREQUISITE): docs/13 row 3 places the media library / downloads volume on the
# `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
# but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
# volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
# therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Radarr deployable today.
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
# /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
#
# *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Radarr imports by HARDLINKING or
# atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
# and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
# /mnt/Storage/plex/movies/ the library
# /mnt/Storage/plex/downloads/complete/radarr/ where finished releases land
# — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
# them as two subPath volumes (subPath: movies at /media/movies, subPath: downloads elsewhere) creates
# TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Radarr then
# silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
# seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
# is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
# requirement, not a shortcut. (The cost is stated plainly: Radarr can also write /media/tv,
# which is sonarr's. Narrowing that would cost hardlinks, which is a worse trade.)
#
# *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
# renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
# config volume entirely; the pod would come up with its app database shadowed by an 8 T media
# tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
# the same export, so when qbittorrent is eventually given this mount too, the arrs and the
# download client will already agree on absolute paths — which is the other precondition for
# hardlinked imports (otherwise every import needs a Remote Path Mapping).
#
# SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
# is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
# with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
# Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
# documents having already been bitten by. Fixing it means starting from an empty config volume, so
# it belongs in its own change with the operator present.
#
# UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
# + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
# the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
# write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
# every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
#
# AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
# ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
# lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
# that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
# the image re-chowning /config for itself.
- name: media
class: bulk
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
mountPath: /media # => /media/movies (Root Folder) and /media/downloads (import source)
size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
# rounded up, so `kubectl get pv` reports something true.
dependencies:
- to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
port: 8080
catalog/render.py
+171 -6
View File
@@ -53,6 +53,30 @@ LBL = "homelab.keithjackson.dev"
# Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
# and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10).
ZONE = "${BASE_DOMAIN}"
# The bulk (external NFS) tier's SERVER, emitted as a Flux post-build TOKEN for exactly the same
# reason as ZONE: dev and prod reach the same Proxmox ZFS host by DIFFERENT addresses (dev's nodes
# are on the host's own LAN and dial it directly; prod's nodes are on the far side of pfSense and
# must be given whichever address the still-pending routing decision produces). Hardcoding either
# one authors a manifest that is silently wrong in the other environment. Declared in
# clusters/base/cluster-vars.yaml and patched by BOTH overlays; written BARE (it is a real knob,
# not a shell expansion), so StrictPostBuildSubstitutions fails the tier if an overlay forgets it.
BULK_NFS_SERVER = "${BULK_NFS_SERVER}"
# Mount options for every bulk NFS volume. These are a PLATFORM decision, not a per-service one,
# so they live here rather than on the ServiceDefinition surface:
# nfsvers=4.1 NFSv4 carries mount+lock IN the protocol, so it needs ONE port (tcp/2049) and no
# rpcbind/111, mountd or statd. That is what makes the prod path a single pfSense
# rule instead of four, and it is why v4 is chosen over v3. The server advertises
# +3 +4 +4.1 +4.2; 4.1 is the conservative floor every node kernel already speaks.
# hard A `soft` mount turns a server hiccup into a short read / EIO handed to the
# application, which for a media library and an archive is silent corruption.
# `hard` blocks instead. The cost is explicit: an NFS outage HANGS I/O rather than
# failing it, so an outage shows up as stuck pods, not as damaged files. Chosen
# deliberately -- hung is recoverable, truncated is not.
# noatime The datasets are relatime, so every read would otherwise generate a metadata
# WRITE back to the pool. Plex walking an 8 T library must not do that.
# timeo=600/retrans=2 the TCP defaults, stated explicitly so a future reader knows the retry
# budget was considered rather than inherited.
BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retrans=2"]
GATEWAY_NAME = "shared"
GATEWAY_NS = "gateway"
KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label
@@ -111,8 +135,10 @@ def apply_defaults(sd: dict) -> dict:
obs.setdefault("slis", [])
s["observability"] = obs
# storage entry class default is longhorn (replicated tier).
s["storage"] = [{**{"class": "longhorn"}, **v} for v in s["storage"]]
# storage entry class default is longhorn (replicated tier); readOnly default is False
# (grant-nothing-extra is expressed by SETTING it, so the default stays back-compatible and
# every pre-existing golden renders byte-identical).
s["storage"] = [{**{"class": "longhorn", "readOnly": False}, **v} for v in s["storage"]]
# dependency protocol default is tcp.
s["dependencies"] = [{**{"protocol": "tcp"}, **d} for d in s["dependencies"]]
return s
@@ -200,18 +226,139 @@ def _service_ports(s: dict) -> list:
return [{"name": f"port-{p}", "port": p, "targetPort": p} for p in ports]
def _bulk_access_mode(vol: dict) -> str:
"""A bulk volume's access mode follows its read/write posture.
Every bulk volume is `...Many`: the SAME export is consumed by several namespaces at once
(plex, sonarr and radarr all sit on /mnt/Storage/plex) and by several nodes at once, which is
precisely what ReadWriteOnce cannot express. WHICH `Many` it is says whether this consumer
writes -- so `kubectl get pv` reports the intent, not merely the topology.
"""
return "ReadOnlyMany" if vol.get("readOnly") else "ReadWriteMany"
def r_bulk_pvs(s: dict) -> list:
"""One STATIC PersistentVolume per `class: bulk` storage entry (docs/04 bulk tier).
WHY A PV AT ALL, RATHER THAN AN INLINE `nfs:` POD VOLUME
-------------------------------------------------------
An inline volume is fewer objects and it would work. It is rejected for four reasons, in
descending order of what they would cost to discover later:
1. RECLAIM POLICY. `persistentVolumeReclaimPolicy: Retain` is a declared, reviewable
property of a PV. There is no equivalent for an inline volume -- and the data behind
these mounts is 8.33 TiB of media and a 267 GiB archive that ALSO has live non-k3s
consumers. A statement in git that nothing in this cluster may reclaim it is worth an
object.
2. READ-ONLY LIVES IN ONE PLACE. `spec.nfs.readOnly: true` becomes a real `ro` mount option
for every pod that ever binds this volume. The same intent expressed on an inline volume
is one Deployment edit away from being dropped silently, with no diff anywhere near the
storage. (It is still a CLIENT-side guarantee -- the export cannot distinguish plex from
sonarr, since both may run on any node -- but it is one that lives in one reviewable
place.)
3. VISIBILITY. `kubectl get pv,pvc` is where an operator looks for "what external storage is
attached, and who holds it". An inline volume is invisible there, and invisible to
anything that reasons over PVCs.
4. IT IS THE CONTRACT THE REPO ALREADY HAS. Every other storage entry is a PVC; the
renderer, the Kyverno ServiceDefinition policy and the backup wiring all key on that. An
inline volume would be a second, parallel storage model inside the same renderer.
CROSS-NAMESPACE. A PV binds 1:1 to a PVC, so "plex, sonarr and radarr share one export" is NOT
one shared PV -- it is three PVs, one per namespace, each naming the same server+path. That is
the correct model rather than a workaround: the sharing happens on the NFS server, which is
built for concurrent clients, and each namespace keeps its own independently reviewable access
mode (plex ReadOnlyMany, the arrs ReadWriteMany). One shared PV would have forced a single
posture on all three.
BINDING IS PINNED FROM BOTH ENDS -- `claimRef` here and `volumeName` on the PVC. Either alone
leaves a race in which some other PVC of matching size and mode grabs the volume first;
together the binding is deterministic. `storageClassName: ""` (explicitly empty, NOT the `bulk`
StorageClass) is what tells the binder this volume is statically provisioned: it suppresses the
default-StorageClass annotation and guarantees no provisioner is ever asked to create anything.
It also sidesteps a trap -- a StorageClass's `provisioner` is IMMUTABLE, so re-pointing the
existing `bulk` class (whose stub names an NFS CSI driver that is not installed) at this data
would make Flux fail forever on that one object.
"""
out = []
for vol in s["storage"]:
if vol.get("class") != "bulk":
continue
name = f"{s['name']}-{vol['name']}"
# `export` and `size` are both structurally required of a bulk entry, and the JSON
# Schema only enforces the first. A PersistentVolume with no `capacity` is rejected by
# the API server, so a missing size must fail HERE with a sentence a human can act on
# rather than as a bare KeyError three frames down.
for field, why in (
("export", "the absolute NFS export path this volume attaches"),
("size", "the PV capacity (advisory, but the API requires the field)"),
):
if not vol.get(field):
raise ValueError(
f"{s['name']}: storage entry {vol['name']!r} has class 'bulk' but no "
f"'{field}' — {why}. A bulk volume attaches storage that already exists "
f"outside the cluster, so it cannot be rendered without it."
)
nfs = {"server": BULK_NFS_SERVER, "path": vol["export"]}
if vol.get("readOnly"):
nfs["readOnly"] = True
spec = {
"capacity": {"storage": vol["size"]},
"accessModes": [_bulk_access_mode(vol)],
# Retain: deleting the PVC must NEVER be able to reach the data. There is nothing to
# "reclaim" here in any case -- the volume was not provisioned by the cluster.
"persistentVolumeReclaimPolicy": "Retain",
"storageClassName": "",
"mountOptions": list(BULK_NFS_MOUNT_OPTIONS),
"nfs": nfs,
"claimRef": {"namespace": s["name"], "name": name},
}
out.append(
{
"apiVersion": "v1",
"kind": "PersistentVolume",
"metadata": {
"name": name,
"labels": {"app.kubernetes.io/name": s["name"]},
},
"spec": spec,
}
)
return out
def r_pvcs(s: dict) -> list:
"""One PVC per storage entry (R-STATE-01/02/04).
longhorn -> 3-replica replicated tier; bulk -> external media tier (not 3x replicated).
longhorn -> 3-replica replicated tier, dynamically provisioned, ReadWriteOnce.
bulk -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName.
"""
out = []
for vol in s["storage"]:
cls = vol.get("class", "longhorn")
storage_class = "longhorn" if cls == "longhorn" else "bulk"
if cls == "bulk":
# Mirror image of the PV: the same explicitly-empty class (so nothing dynamically
# provisions), the same access mode, and volumeName completing the two-sided binding.
out.append(
{
"apiVersion": "v1",
"kind": "PersistentVolumeClaim",
"metadata": {
"name": f"{s['name']}-{vol['name']}",
"namespace": s["name"],
"labels": {"app.kubernetes.io/name": s["name"]},
},
"spec": {
"accessModes": [_bulk_access_mode(vol)],
"storageClassName": "",
"volumeName": f"{s['name']}-{vol['name']}",
"resources": {"requests": {"storage": vol["size"]}},
},
}
)
continue
spec = {
"accessModes": ["ReadWriteOnce"],
"storageClassName": storage_class,
"storageClassName": "longhorn",
}
if vol.get("size"):
spec["resources"] = {"requests": {"storage": vol["size"]}}
@@ -257,7 +404,17 @@ def r_deployment(s: dict) -> dict:
# it verbatim for apps that need a fixed on-disk layout (e.g. Gitea keeps all state
# directly under /data). Omitting it preserves the /data/<name> default (back-compat).
mount_path = vol.get("mountPath", f"/data/{vol['name']}")
volume_mounts.append({"name": vol["name"], "mountPath": mount_path})
mount = {"name": vol["name"], "mountPath": mount_path}
# subPath mounts a subtree of the volume instead of its root. Emitted only when declared,
# so every pre-existing golden stays byte-identical.
if vol.get("subPath"):
mount["subPath"] = vol["subPath"]
# readOnly on the mount as well as on the PV's nfs source. Belt and braces on purpose: the
# PV field is what makes the kernel mount `ro`; this one is what a reviewer reading the
# Deployment sees. An edit that drops one still leaves the other.
if vol.get("readOnly"):
mount["readOnly"] = True
volume_mounts.append(mount)
volumes.append(
{
"name": vol["name"],
@@ -1095,6 +1252,14 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
if s["kind"] == "container":
# Container PVCs. (For a vm, the CDI DataVolume owns/creates the disk PVC, so we
# never render a plain PVC for the VM root disk — it would collide with CDI.)
# The static bulk PV is rendered BEFORE the PVC that names it. Document order inside one
# kustomize build is not an apply-ordering guarantee, but a PVC whose PV does not exist
# yet merely stays Pending and binds on the next reconcile, so this is a readability
# convention rather than a correctness requirement.
for pv in r_bulk_pvs(s):
docs.append(
(f"PersistentVolume {pv['metadata']['name']} - static external NFS (bulk tier)", pv)
)
for pvc in r_pvcs(s):
docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc))
docs.append(("Deployment (R-PLAT-02)", r_deployment(s)))
catalog/sonarr.yaml
+66 -5
View File
@@ -9,6 +9,23 @@ name: sonarr
kind: container
source:
image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
# which cannot write a 1000:3000-owned 0755 tree.
env:
- name: PUID
value: "1000" # owner uid of every directory under /mnt/Storage/plex
- name: PGID
value: "3000" # dominant group there; keeps files this writes readable to the
# EXISTING non-k3s consumers of the same export, which stay live
- name: TZ
value: Etc/UTC # same safe default plex uses; the arrs schedule by local time
podSecurityContext:
# For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
# does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
# what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
# /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
fsGroup: 3000
exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET)
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
@@ -16,11 +33,55 @@ storage:
- name: config
size: 5Gi
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
# NOTE (bulk-media PREREQUISITE): docs/13 row 2 places the media library / downloads volume on the
# `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
# but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
# volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
# therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Sonarr deployable today.
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
# /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
#
# *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Sonarr imports by HARDLINKING or
# atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
# and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
# /mnt/Storage/plex/tv/ the library
# /mnt/Storage/plex/downloads/complete/tv-sonarr/ where finished releases land
# — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
# them as two subPath volumes (subPath: tv at /media/tv, subPath: downloads elsewhere) creates
# TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Sonarr then
# silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
# seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
# is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
# requirement, not a shortcut. (The cost is stated plainly: Sonarr can also write /media/movies,
# which is radarr's. Narrowing that would cost hardlinks, which is a worse trade.)
#
# *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
# renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
# config volume entirely; the pod would come up with its app database shadowed by an 8 T media
# tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
# the same export, so when qbittorrent is eventually given this mount too, the arrs and the
# download client will already agree on absolute paths — which is the other precondition for
# hardlinked imports (otherwise every import needs a Remote Path Mapping).
#
# SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
# is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
# with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
# Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
# documents having already been bitten by. Fixing it means starting from an empty config volume, so
# it belongs in its own change with the operator present.
#
# UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
# + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
# the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
# write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
# every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
#
# AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
# ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
# lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
# that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
# the image re-chowning /config for itself.
- name: media
class: bulk
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
mountPath: /media # => /media/tv (Root Folder) and /media/downloads (import source)
size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
# rounded up, so `kubectl get pv` reports something true.
dependencies:
- to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
port: 8080
ci/l0.sh
+38
View File
@@ -175,6 +175,44 @@ else
esac
fi
# ---- Stage 3e: bulk tier distinct from the replicated tier (UNCONDITIONAL) ----
# The `bulk` StorageClass sat as a REPLACE-ME stub for six roadmap phases, and the only
# thing keeping a multi-terabyte media library off the 3-replica `longhorn` class was four
# hand-written comments in ServiceDefinitions. A bulk volume that lands on the replicated
# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
# three mini-PCs cannot hold. The check asserts the tier separation, that a bulk volume is
# statically pre-bound from BOTH ends (so nothing can dynamically provision an EMPTY
# stand-in that mounts and Binds perfectly while the real data is never attached), Retain,
# hard/NFSv4 mount options, the read-only coupling between PV and volumeMount, that no
# workload mounts one export twice (which breaks Sonarr/Radarr hardlinks with EXDEV), and
# that the server is a per-environment knob every overlay supplies.
#
# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c/3d: Stage 3b scopes the conformance
# run to the TOUCHED requirements (R-TEST-03), and ci/touched-requirements.sh's path map
# keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement. So the change most
# likely to break this — moving a bulk volume onto the replicated tier, or narrowing one
# export into two mounts — could attribute no requirement at all and be gated by nothing.
# The invariant spans catalog/, apps/ and clusters/, so it runs on EVERY gate invocation.
BULK_TIER_CHECK="${BULK_TIER_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh}"
log "Stage: bulk tier distinct from replicated tier (${BULK_TIER_CHECK})"
if [ ! -f "$BULK_TIER_CHECK" ]; then
skip "bulk tier check not present at '${BULK_TIER_CHECK}' — UNWIRED, not green"
summary "- bulk-tier: **SKIPPED** (check missing)"
if [ "$rc" -eq 0 ]; then rc=3; fi
else
bulk_rc=0
bash "$BULK_TIER_CHECK" || bulk_rc=$?
case "$bulk_rc" in
0) ok "Bulk tier distinctness + static binding passed"
summary "- bulk-tier: **passed**" ;;
3) skip "bulk tier check SKIPPED (catalog/ or apps/ absent) — not a pass"
summary "- bulk-tier: **SKIPPED** (nothing to audit)"
if [ "$rc" -eq 0 ]; then rc=3; fi ;;
*) err "Bulk tier distinctness + static binding FAILED"
summary "- bulk-tier: **FAILED**"; rc=1 ;;
esac
fi
case "$rc" in
0) ok "L0 stage complete" ;;
3) skip "L0 stage incomplete (unwired seam / missing tool) — not a pass" ;;
clusters/base/cluster-vars.yaml
+22
View File
@@ -160,6 +160,28 @@ data:
UNIFI_L4_VIP: "192.168.1.54" # UniFi controller VM: inform/STUN/adoption (apps/vms/unifi)
CRAFTY_L4_VIP: "192.168.1.55" # Crafty: Minecraft Java (TCP) + Bedrock (UDP) game ports
# --- Bulk (external NFS) storage host (docs/04 bulk tier) ---------------------------------------
# THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media
# library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive
# (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's
# `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is
# declared in the ServiceDefinition, because the path is identical in every environment and only
# the ADDRESS is not.
#
# WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense.
# The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN.
# dev nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one
# NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in
# /etc/exports), so this path is proven, not theoretical. Base default = the dev value,
# per the convention of the LAN block above.
# prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today;
# 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening
# the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give
# DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route).
# So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the
# blast radius of shipping it unset, which is NOT the usual "fails to apply".
BULK_NFS_SERVER: "192.168.1.9"
# Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
# Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
# nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port.
clusters/dev/kustomization.yaml
+9
View File
@@ -225,6 +225,15 @@ patches:
- op: replace
path: /data/CRAFTY_L4_VIP
value: "192.168.1.55"
# Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes
# (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes).
# Restated verbatim rather than inherited, per this overlay's "supply every knob" convention.
# Dev is therefore the environment in which the bulk manifests can actually be VALIDATED
# end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=`
# change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses.
- op: replace
path: /data/BULK_NFS_SERVER
value: "192.168.1.9"
# Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
# barmanObjectStore and Velero's backupStorageLocation write to.
#
clusters/prod/kustomization.yaml
+55
View File
@@ -392,6 +392,61 @@ patches:
- op: replace
path: /data/CRAFTY_L4_VIP
value: "192.168.2.22"
# ------------------------------------------------------------------------------------------
# BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. ***
# ------------------------------------------------------------------------------------------
# Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr,
# radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud).
#
# THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface,
# vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN.
# 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102;
# 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate
# outcomes give DIFFERENT values here:
# port-forward 192.168.2.102:2049 -> 192.168.1.9:2049 => "192.168.2.102"
# a WAN->LAN route to 192.168.1.0/24 => "192.168.1.9"
# Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess
# ever became reachable — mounts something unintended. So it stays a placeholder.
#
# WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and
# NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4
# over v3 was in part this decision's cost: v3 would have needed four ports and three of them
# on ephemeral numbers.
#
# WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes.
# Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward:
# the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN
# address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via
# 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node
# addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md.
#
# *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. ***
# A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this
# placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The
# failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server
# TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in
# ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at
# apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles
# a revision containing the bulk volumes; the dev overlay renders a working value, which is
# where these manifests are meant to be proven first.
- op: replace
path: /data/BULK_NFS_SERVER
# 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox
# host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot
# reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and
# keeps the base value 192.168.1.9.
#
# PROVEN from k3s-prod-1 on 2026-09-12, not inferred:
# - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp
# - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export
# carries no_root_squash, so a root write would have passed and proved nothing, while
# LSIO images default to PUID 911 and would have gone Ready then failed every import
# with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer.
# - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get
# real atomic moves instead of silently degrading to copy-then-delete on EXDEV.
# pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer),
# so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP.
value: "192.168.2.102"
# ==========================================================================================
# OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
# ==========================================================================================
conformance/tests/l0-static/bulk-tier-distinct-from-replicated/chainsaw-test.yaml
+99
View File
@@ -0,0 +1,99 @@
# =============================================================================
# L0 STATIC — the bulk tier's capacity strategy is DISTINCT from the replicated tier's
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
#
# THE BUG CLASS THIS CLOSES. The `bulk` StorageClass existed from Phase 0 as a stub:
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME`, TODO(OQ-2).
# Four ServiceDefinitions carried a comment promising to attach their media once OQ-2 was
# resolved, and the ONLY thing keeping a multi-terabyte library off the 3-replica
# `longhorn` class was those four comments. A bulk volume that lands on the replicated
# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
# three mini-PCs cannot hold. R-STATE-04 is precisely that the two tiers are distinct, so
# the distinctness is asserted rather than reviewed.
#
# It also pins the four decisions that make the STATIC form of this tier safe, each with a
# plausible "simplification" that breaks something silently: `Retain` (bulk data pre-exists
# the cluster and has consumers outside it, so anything else makes `delete pvc` a data-loss
# operation); NO dynamic provisioning (a dynamic bulk class hands out a FRESH EMPTY
# directory that mounts and Binds perfectly while the real 8 TiB is never attached — the
# most deceptive failure available here); `hard` not `soft` (a soft mount hands a server
# hiccup to the app as a short read, which for a media library is silent corruption); and
# NFSv4 (mount and lock in-protocol ⇒ ONE reachable port, tcp/2049, which is what the
# operator must open in pfSense for prod to reach this server at all).
#
# AND the EXDEV trap, which is not about storage. Sonarr/Radarr import by hardlinking or
# atomically renaming, both same-filesystem operations, and to the kernel "same filesystem"
# means SAME MOUNT. Splitting one export into two subPath mounts looks TIGHTER and breaks
# link() with EXDEV; the arrs then degrade to copy-then-delete, rewriting every import over
# NFS and destroying the seeding copy. It surfaces weeks later as "imports are slow and my
# ratio is gone". A reviewer cannot see that; this check can.
#
# GIVEN the repository as authored,
# WHEN every `class: bulk` storage entry in catalog/ is compared against the objects the
# COMMITTED apps/ goldens carry for it (what Flux reconciles — deliberately not the
# renderer, which would only prove it agrees with itself),
# THEN (1) no bulk volume is on a Longhorn-provisioned class, and its class is explicitly
# EMPTY so nothing can dynamically provision an empty stand-in;
# (2) its capacity is declared and is at least an order of magnitude above the same
# service's replicated volume — the requirement stated as arithmetic;
# (3) PV and PVC are pre-bound from BOTH ends (claimRef + volumeName) and agree;
# (4) the PV reclaim policy is Retain;
# (5) mountOptions carry `hard` and exactly one `nfsvers=4.x`, and never `soft`;
# (6) no workload mounts one export more than once (the EXDEV trap);
# (7) a readOnly entry renders nfs.readOnly AND a readOnly volumeMount AND
# ReadOnlyMany; a writable one renders neither and is ReadWriteMany; every bulk
# access mode is a `...Many`, since one export serves several namespaces and
# several nodes;
# (8) the PV's nfs.server is a BARE cluster-vars token (never a literal — dev and
# prod reach this host by different addresses), that knob has a base default, and
# EVERY overlay supplies it;
# AND (9) the check is LOAD-BEARING (ci/l0.sh runs it unconditionally, ci/gate.sh blocks
# on the l0 stage) and NON-VACUOUS (an empty bulk corpus FAILS rather than
# passing trivially, and a floor on the assertion count keeps a pass from
# degenerating into a no-op).
#
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library +
# PyYAML); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test wraps the
# SAME script.
#
# NOT proven — the runtime half, which needs a cluster: that the server is reachable, that
# the export admits these clients, that the PVC binds, that a write succeeds, or that a
# hardlink between the library and the download path actually works. The uid analysis is
# reasoning (docs/BULK-NFS-STORAGE.md §6) and the in-cluster probes that would settle it
# are written down in §9 of that document, unrun. Nor can this prove prod's address is
# CORRECT — only that it is a supplied knob; while it is a placeholder, prod's pods will
# not mount, and catching that statically needs the placeholder guard, which is absent from
# main (§7). Read-only on the media tree is a CLIENT-side guarantee only: the export serves
# one reader and two writers from the same node addresses, so the server cannot tell them
# apart (§5).
# =============================================================================
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: bulk-tier-distinct-from-replicated
labels:
conformance.home.k3s/R-STATE-04: "true"
annotations:
conformance.home.k3s/requirements: "R-STATE-04"
conformance.home.k3s/layer: "L0"
conformance.home.k3s/safety-critical: "false"
conformance.home.k3s/executable: "true"
conformance.home.k3s/intent: >-
Bulk media storage is provably NOT on the 3-replica Longhorn tier, is sized for a
different order of magnitude, cannot be silently replaced by a dynamically
provisioned empty volume, is never reclaimable by the cluster, mounts hard over
NFSv4, and is never split into two mounts of one export — so hardlinked imports keep
working. Its server is a per-environment knob, not a literal, because the two
clusters reach the same storage host by different addresses.
spec:
steps:
- name: assert-bulk-tier-is-distinct-and-statically-bound
description: >-
Read every `class: bulk` storage entry from catalog/, pair it with the
PersistentVolume, PersistentVolumeClaim and volumeMount the committed apps/ goldens
carry for it, and assert the tier separation, the static pre-binding, the reclaim
and mount-option decisions, the read-only coupling, the one-export-one-mount
invariant, and that the server address is supplied by every environment overlay.
try:
- script:
content: bash ./check.sh
conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh
Executable
+480
View File
@@ -0,0 +1,480 @@
#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated
# Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the
# replicated tier." — spec docs/04-storage-and-state.md)
#
# WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY
# ----------------------------------------------------------------------------------------
# `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2).
# Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is
# resolved", and each of them said, correctly, that adding `class: bulk` before then would
# leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing
# behind it, and the only thing preventing a multi-terabyte media library from being
# authored onto the 3-replica `longhorn` class was four hand-written comments.
#
# That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk
# storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the
# replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on
# `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the
# tier fills up. Distinctness is therefore a property worth asserting statically, on every
# gate run, rather than trusting to review.
#
# It also pins the four decisions that make the STATIC form of this tier safe, each of
# which has a plausible-looking "simplification" that silently breaks something:
#
# * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has
# consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an
# inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl
# delete pvc` a data-loss operation.
# * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The
# pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of
# existing data was never attached. This is the single most deceptive failure available
# in this tier, so `storageClassName` must be explicitly empty and binding must be
# pinned from BOTH ends (claimRef + volumeName).
# * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or
# EIO handed to the application. For a media library that is silent corruption. `hard`
# hangs instead — recoverable, where truncated is not.
# * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port
# (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is
# a FIREWALL fact as much as a storage one: prod reaches this server through pfSense,
# and the number of ports the operator must open is a consequence of this field.
#
# AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6).
# Sonarr and Radarr import by hardlinking or atomically renaming a finished download into
# the library. Both are same-filesystem operations, and to the kernel "same filesystem"
# means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`)
# looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no
# matter that the server knows they are one dataset. The arrs do not fail loudly on that:
# they degrade to copy-then-delete, rewriting every imported file over NFS and destroying
# the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is
# gone". A reviewer cannot see it, so this check does: one export, at most one mount, per
# workload.
#
# GIVEN the repository as authored,
# WHEN every `class: bulk` storage entry in catalog/ is compared against the manifests
# render.py produces for it (read from the committed apps/ goldens, i.e. what Flux
# actually reconciles — not from the renderer, which would only agree with itself),
# THEN (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC
# storageClassName is explicitly empty — never `longhorn`, never any class the
# repo declares with the Longhorn provisioner — and its PV's source is an
# EXTERNAL one (nfs), not a Longhorn volume;
# (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the
# bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own
# replicated volume. This is the requirement stated as arithmetic: the two tiers
# exist because the data has two magnitudes, so a "bulk" volume sized like a
# config PVC means the strategy was named and not actually applied;
# (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its
# PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and
# the two agree — so no other claim can win a race, and nothing can provision an
# empty replacement;
# (4) RECLAIM: every bulk PV is `Retain`;
# (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an
# `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`;
# (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above);
# (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH
# `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount,
# and its access mode is ReadOnlyMany; a writable one renders NEITHER and is
# ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed
# by several namespaces and several nodes, which ReadWriteOnce cannot express;
# (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk
# PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that
# knob has a base default; and EVERY environment overlay supplies it — so the two
# clusters, which reach this storage by different addresses, cannot be served by
# one hardcoded value;
# AND (9) LOAD-BEARING + NON-VACUOUS: ci/l0.sh runs this check unconditionally and
# ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT
# empty — a repo with zero bulk volumes fails here rather than passing trivially,
# because a check that is satisfiable by deleting the feature proves nothing.
#
# WHAT THIS DOES *NOT* PROVE (honest gaps)
# - Nothing about RUNTIME. Not that the server is reachable, not that the export permits
# these clients, not that the PVC binds, not that a write succeeds. The uid analysis in
# docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the
# in-cluster proof — they need a cluster and are not run here.
# - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While
# prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md
# §7); catching that needs the placeholder guard, which is absent from main.
# - Read-only for the media tree is a CLIENT-side guarantee, and this check can only
# assert the client side. The export serves a reader and two writers from the same node
# addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5).
# - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It
# catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a
# static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit.
# - Assertion (6) reasons over ONE workload's mounts. It cannot see two different
# services hardlinking between each other's mounts, which is not a thing that happens.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
PYTHON="${PYTHON:-python3}"
cd "${ROOT}"
if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then
echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2
exit 3
fi
"${PYTHON}" - "${ROOT}" <<'PY'
import re
import sys
from pathlib import Path
try:
import yaml
except ImportError: # pragma: no cover
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
file=sys.stderr)
raise SystemExit(1)
ROOT = Path(sys.argv[1])
FAILURES: list[str] = []
CHECKS = 0
def check(cond, msg):
global CHECKS
CHECKS += 1
if not cond:
FAILURES.append(msg)
def bail(msg):
"""A failure that makes the rest of the check meaningless: report and stop."""
print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr)
raise SystemExit(1)
def qty_to_bytes(q):
"""Kubernetes quantity -> bytes. Only the binary suffixes the schema permits."""
m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip())
if not m:
return None
return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)]
# =====================================================================================
# The replicated tier, named by the repo rather than by this file. Any StorageClass the
# repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1);
# deriving it means adding a new Longhorn class cannot create a blind spot here.
# =====================================================================================
REPLICATED_CLASSES: set[str] = set()
LONGHORN_DIR = ROOT / "platform" / "longhorn"
if LONGHORN_DIR.is_dir():
for f in sorted(LONGHORN_DIR.rglob("*.yaml")):
for doc in yaml.safe_load_all(f.read_text()):
if isinstance(doc, dict) and doc.get("kind") == "StorageClass":
if "longhorn" in str(doc.get("provisioner", "")):
REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", ""))
# `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in
# the set even if platform/longhorn/ is not present in this checkout.
REPLICATED_CLASSES.add("longhorn")
REPLICATED_CLASSES.discard("")
# =====================================================================================
# The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired
# with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling
# the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is
# a separate gate stage. Asking the renderer would only prove it agrees with itself.
# =====================================================================================
sds: dict[str, dict] = {}
for f in sorted((ROOT / "catalog").glob("*.yaml")):
doc = yaml.safe_load(f.read_text())
if isinstance(doc, dict) and doc.get("name"):
sds[doc["name"]] = doc
if not sds:
bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier")
# Every rendered app-tier document, indexed by (kind, name).
rendered: dict[tuple, dict] = {}
for f in sorted((ROOT / "apps").rglob("*.yaml")):
if f.name == "kustomization.yaml" or ".sops." in f.name:
continue
try:
docs = list(yaml.safe_load_all(f.read_text()))
except yaml.YAMLError:
continue
for doc in docs:
if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"):
rendered[(doc["kind"], doc["metadata"]["name"])] = doc
# (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2).
corpus: list[tuple[str, dict]] = []
replicated_bytes: dict[str, list[int]] = {}
for svc, sd in sds.items():
for vol in sd.get("storage") or []:
cls = vol.get("class", "longhorn")
if cls == "bulk":
corpus.append((svc, vol))
else:
b = qty_to_bytes(vol.get("size"))
if b:
replicated_bytes.setdefault(svc, []).append(b)
# ------------------------------------------------------------------- (9) NON-VACUOUS
# A corpus of zero would make every loop below trivially satisfied. The bulk tier is
# DELIVERED; a change that removes the last bulk volume must fail here and be argued for,
# not sail through on an empty iteration.
check(len(corpus) > 0,
"NO `class: bulk` storage entry exists in catalog/ — this check would then pass "
"vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME "
"stub for six roadmap phases. If the tier is genuinely being retired, retire this "
"check in the same change and say so; do not let it pass on an empty corpus")
if not corpus:
bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)")
inventory: list[str] = []
for svc, vol in corpus:
vname = vol.get("name", "<unnamed>")
label = f"{svc}/{vname}"
obj = f"{svc}-{vname}"
ro = bool(vol.get("readOnly"))
# `export` is what the schema requires of a bulk entry; without it there is nothing to
# attach and the rest of the assertions have no subject.
export = vol.get("export")
check(bool(export) and str(export).startswith("/"),
f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an "
f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)")
pv = rendered.get(("PersistentVolume", obj))
pvc = rendered.get(("PersistentVolumeClaim", obj))
check(pv is not None,
f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk "
f"entry that renders no PV has NO external storage behind it — the pod would get "
f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory")
check(pvc is not None,
f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens")
if pv is None or pvc is None:
continue
pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {}
# ------------------------------------------------------ (1) DISTINCT FROM REPLICATED
for what, spec in (("PV", pvs), ("PVC", pvcs)):
sc = spec.get("storageClassName")
check(sc not in REPLICATED_CLASSES,
f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares "
f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier "
f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not "
f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)")
check(sc == "",
f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY "
f"string. Empty is what disables dynamic provisioning AND the default-class "
f"annotation. Any named class risks a provisioner handing out a fresh EMPTY "
f"directory that mounts and binds cleanly while the real data is never attached")
check("nfs" in pvs,
f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole "
f"premise is that durability lives OUTSIDE the cluster (docs/04)")
# ------------------------------------------------------------ (4) RECLAIM POLICY
check(pvs.get("persistentVolumeReclaimPolicy") == "Retain",
f"{label}: bulk PV reclaim policy is "
f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists "
f"the cluster and has consumers outside it; anything but Retain makes `kubectl "
f"delete pvc` a data-loss operation")
# ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS
check(pvcs.get("volumeName") == obj,
f"{label}: bulk PVC does not name its PV via volumeName (got "
f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be "
f"satisfied by some OTHER volume of matching size and mode")
cref = pvs.get("claimRef") or {}
check(cref.get("name") == obj and cref.get("namespace") == svc,
f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} "
f"— pre-binding must be pinned from BOTH ends or another PVC can win the race")
# ------------------------------------------------------------- (5) MOUNT SAFETY
mopts = [str(o) for o in (pvs.get("mountOptions") or [])]
check(bool(mopts),
f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS "
f"version are whatever the client happens to default to, and both are decisions "
f"with consequences (silent short reads; how many firewall ports must open)")
check("hard" in mopts,
f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount "
f"hands a server hiccup to the application as a short read or EIO — silent "
f"corruption for a media library. `hard` hangs instead, which is recoverable")
check("soft" not in mopts,
f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the "
f"one option that turns an outage into damaged data")
vers = [o for o in mopts if o.startswith("nfsvers=")]
check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]),
f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). "
f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port "
f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the "
f"operator has to open in pfSense for prod to reach this server at all")
# ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT
want_mode = "ReadOnlyMany" if ro else "ReadWriteMany"
for what, spec in (("PV", pvs), ("PVC", pvcs)):
modes = spec.get("accessModes") or []
check(modes == [want_mode],
f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk "
f"volume is a `...Many` (one export, several namespaces, several nodes — "
f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write "
f"posture stays legible in `kubectl get pv`")
nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly"))
check(nfs_ro == ro,
f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. "
f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared "
f"posture is not the one the kernel enforces")
# The volumeMount half, read out of the rendered Deployment.
dep = rendered.get(("Deployment", svc))
check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against")
if dep is not None:
containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or []
# The app container is containers[0] by the renderer's contract (a forward-auth
# sidecar is appended AFTER it).
mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else []
check(len(mounts) == 1,
f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, "
f"expected exactly 1 — a bulk volume that is declared and never mounted is a "
f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)")
if len(mounts) == 1:
m = mounts[0]
check(bool(m.get("readOnly")) == ro,
f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the "
f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, "
f"so that dropping one still leaves the other")
check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"),
f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the "
f"declared subPath {vol.get('subPath')!r}")
# ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV
server = (pvs.get("nfs") or {}).get("server")
m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or ""))
check(m is not None,
f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars "
f"token, not a literal address. dev and prod reach this storage host by DIFFERENT "
f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct "
f"in at most one environment and silently wrong in the other (R-ENV-02)")
cap = (pvs.get("capacity") or {}).get("storage")
req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage")
check(cap is not None and qty_to_bytes(cap) is not None,
f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, "
f"and it is what makes `kubectl get pv` report something true about this tier")
check(cap == req,
f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the "
f"capacity never binds")
inventory.append(
f" {label:<28} export={export} {'ro' if ro else 'rw'} {want_mode:<14} "
f"cap={cap} server={server} opts={','.join(mopts)}"
)
# ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes
cap_b = qty_to_bytes(cap)
reps = replicated_bytes.get(svc) or []
if cap_b and reps:
biggest = max(reps)
# ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed:
# the media library is ~460x its service's config volume, and the Nextcloud archive
# ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate
# bulk volume, and this check would be teaching people to weaken it. 10x cannot be
# met by accident and still catches the regression that matters: a "bulk" volume
# sized like a config PVC, i.e. the strategy named but never applied.
check(cap_b >= 10 * biggest,
f"{label}: bulk capacity {cap} is less than 10x this service's largest "
f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the "
f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps "
f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A "
f"bulk volume sized like a config volume means the tier was named, not applied")
# ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT
# The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod.
by_service: dict[str, list[dict]] = {}
for svc, vol in corpus:
by_service.setdefault(svc, []).append(vol)
for svc, vols in sorted(by_service.items()):
seen: dict[str, str] = {}
for vol in vols:
exp = vol.get("export")
prev = seen.get(exp)
check(prev is None,
f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and "
f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the "
f"kernel, so link() between them fails EXDEV even though the server knows "
f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they "
f"degrade to copy-then-delete, rewriting every imported file over NFS and "
f"destroying the seeding copy. Mount the common ancestor ONCE instead "
f"(docs/BULK-NFS-STORAGE.md §4)")
seen[exp] = vol.get("name")
# ------------------------------------- (8b) the knob has a base default AND every overlay
tokens = set()
for _svc, vol in corpus:
pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}"))
if pv:
mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or ""))
if mm:
tokens.add(mm.group(1))
cv = ROOT / "clusters" / "base" / "cluster-vars.yaml"
base_keys: set[str] = set()
if cv.is_file():
doc = yaml.safe_load(cv.read_text()) or {}
base_keys = set((doc.get("data") or {}).keys())
overlays = sorted(
d for d in (ROOT / "clusters").iterdir()
if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file()
) if (ROOT / "clusters").is_dir() else []
for tok in sorted(tokens):
check(tok in base_keys,
f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no "
f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without "
f"it the token ships into the cluster as a literal server name")
for ov in overlays:
text = (ov / "kustomization.yaml").read_text()
check(f"/data/{tok}" in text,
f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV "
f"value by this repo's convention, so an overlay that omits it silently "
f"inherits an address on the WRONG SIDE of pfSense — a mount that either "
f"never connects or connects to something unintended (R-ENV-02)")
# --------------------------------------------------------------- (9) LOAD-BEARING
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
check("bulk-tier-distinct-from-replicated/check.sh" in L0,
"ci/l0.sh does not run bulk-tier-distinct-from-replicated/check.sh. Stage 3b scopes "
"the conformance run to TOUCHED requirements, and ci/touched-requirements.sh maps "
"neither apps/* nor platform/longhorn/* to the bulk requirement — so a change that "
"moves a bulk volume onto the replicated tier could attribute no requirement at all "
"and be gated by nothing. It must be invoked directly and unconditionally")
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
"ci/gate.sh does not set the merge block when the l0 stage fails")
# ------------------------------------------------------------------------- report
print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):")
for line in inventory:
print(line)
print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): "
f"{', '.join(sorted(REPLICATED_CLASSES))}")
print(f"bulk volumes: {len(corpus)} distinct exports: "
f"{len({v.get('export') for _s, v in corpus})} "
f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}")
# A floor on the assertion count, for the same reason the corpus may not be empty: a pass
# must not be able to degenerate into "nothing was actually asked".
MIN_CHECKS = 40
check(CHECKS >= MIN_CHECKS,
f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not "
f"evidence; the corpus or the extraction has degraded")
print()
if FAILURES:
print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)",
file=sys.stderr)
for f in FAILURES:
print(f" - {f}", file=sys.stderr)
raise SystemExit(1)
print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)")
PY
docs/BULK-NFS-STORAGE.md
+337
View File
@@ -0,0 +1,337 @@
# Bulk NFS storage — the media library and the legacy Nextcloud archive
The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`.
Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is
assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the
case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk
provisioning unimplemented, because nothing needs it.
---
## 1. The storage host, and why prod cannot reach it
The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface:
```
vmbr0 192.168.1.9/24 default via 192.168.1.1 (pfSense LAN)
```
| | nodes | reaches 192.168.1.9? |
|---|---|---|
| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) |
| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. |
Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give
different values for the `BULK_NFS_SERVER` cluster-var:
| outcome | `BULK_NFS_SERVER` for prod |
|---|---|
| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` |
| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` |
Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder
behaves differently from the repo's other ones.**
### It only needs ONE port
The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that
must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed
four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The
server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the
conservative floor.
### The export's client list is the SAME under both outcomes — measured, not assumed
**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation
(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`:
```
SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22
ss -tn -> 192.168.1.9:22 192.168.2.132:39768
```
The original WAN-side source address survives the translation (return traffic finds its way back via
the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's
**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either
way. **It does not have to wait for the routing decision.**
---
## 2. The layout, as found
### `Storage/plex` → `/mnt/Storage/plex` — 8.33 T used, 6.91 T avail
```
/mnt/Storage/plex/ drwxr-xr-x 1000:3000
├── downloads/ drwxr-xr-x 1000:1000
│ ├── complete/ drwxr-xr-x 1000:3000
│ │ ├── radarr/ drwxr-xr-x 1000:3000
│ │ └── tv-sonarr/ drwxr-xr-x 1000:3000
│ ├── incomplete/ drwxrwxr-x 1000:3000
│ ├── downloads -> /shared/downloads (symlink OUTSIDE the export — see §8)
│ └── <in-flight release dirs> drwxr-xr-x 1000:1000
├── movies/ 1165 entries drwxr-xr-x 1000:3000
├── tv/ 107 entries drwxr-xr-x 1000:3000
└── photos/ drwxr-xr-x 1000:3000
```
**The single most important fact about this layout: `downloads/` is a SIBLING of `tv/` and `movies/`
inside the same dataset.** Sonarr's import source and its library are on one filesystem on the
server, and so are Radarr's. That is what makes hardlinked / atomically-renamed imports possible at
all — and §4 is about not throwing it away on the client side.
Note the download client's completed folders are already named `radarr` and `tv-sonarr`, i.e. this
tree has been driven by an arr stack for a long time. The k3s arrs are joining an existing layout,
not defining one.
### `Storage/nextcloud` → `/mnt/Storage/nextcloud` — 270 G used
```
/mnt/Storage/nextcloud/ drwxr-xr-x root:root
└── nextcloud/ drwxr-xr-x 1000:3000
├── config/ 103 M drwxr-xr-x 1000:3000
├── config_old/ 3.4 G drwxr-xr-x 1000:3000
└── config_backup/ 267 G drwxr-xr-x 1000:3000
```
**There is no `data/` directory.** 99% of the dataset is `config_backup`. This is the old, non-k3s
Nextcloud's configuration plus a backup blob — an **archive**, not a live user-data tree. That is
why it is mounted read-only; see §6.
---
## 3. What each app mounts, and why
| app | export | mountPath | subPath | mode | access mode |
|---|---|---|---|---|---|
| plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany |
| sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
| radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
| nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany |
In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr
Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from
`/media/downloads/complete/{tv-sonarr,radarr}`.
**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually
given the same export (it is not in this change), the arrs and the download client will already agree
on absolute paths, which is the second precondition for hardlinked imports — otherwise every import
needs a Remote Path Mapping and quietly falls back to copying when one is missing.
**`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the
renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the
config volume entirely. See §8 for the separate latent bug that discovery exposed.
---
## 4. Why the arrs mount the export ROOT, not two subPaths
This is the design decision most likely to be "simplified" later by someone who has not read this.
Sonarr and Radarr import a finished download into the library by **hardlinking** it (preserving the
seeding copy) or by an **atomic rename**. Both are same-filesystem operations, and **to the kernel,
"same filesystem" means "same mount."**
The tempting narrower shape is two subPath volumes — `subPath: tv` and `subPath: downloads`. Those
are **two separate mounts** of the same export. `link()` across them returns **EXDEV** no matter that
the server knows perfectly well they are one dataset. Sonarr does not fail loudly at that: it
degrades to copy-then-delete. Every import then rewrites the entire file over NFS and destroys the
seeding copy. The symptom is "imports are slow and my ratio is gone", weeks later.
The smallest subtree containing both `tv/` and `downloads/` **is the export root**. So the root mount
is the *minimal* mount that preserves hardlinks: least privilege bounded below by a hard requirement,
not a shortcut.
The cost is stated rather than hidden: Sonarr can also write `/media/movies` and Radarr `/media/tv`.
Narrowing that would cost hardlinks, which is the worse trade.
---
## 5. The export change the operator must apply — STRICTLY ADDITIVE
**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the
exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and
192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the
live non-k3s prod stack.
Current values (`zfs get sharenfs Storage`):
```
Storage/plex [email protected]:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash
Storage/nextcloud [email protected]:@192.168.1.5,no_subtree_check,no_root_squash
```
### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list
```sh
zfs set sharenfs='[email protected]:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex
```
Every pre-existing host is preserved verbatim and no option changes; only clients are added.
### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=`
```sh
zfs set sharenfs='[email protected]:@192.168.1.5,[email protected]:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud
```
This is the one place read-only can be enforced **server-side**, and it is worth taking: only
Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a
misconfigured client cannot then damage the archive.
**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but
all three may be scheduled on any node, so the server sees the same source addresses for all of
them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a
client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an
honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a
dedicated node pool for Plex, which is a much larger change.
Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host
(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive
adding a node; it also grants the whole subnet. Per-node is the tighter default.
### Verifying, without changing anything
```sh
exportfs -v # the new client lines appear, existing ones unchanged
zfs get sharenfs Storage/plex Storage/nextcloud
```
---
## 6. uid / gid — the failure this design exists to avoid
The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through
verbatim** and the server's on-disk ownership is the entire access story. The media tree is
`0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**.
**`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS
volume plugin performs **no ownership management**, so `podSecurityContext.fsGroup` is a **no-op on
an NFS volume**. There is no recursive chown, no `g+rwX`, nothing. The container's own uid has to be
right.
| app | container uid | writes? | outcome |
|---|---|---|---|
| plex | 1000 (`PLEX_UID`/`PLEX_GID`, already set) | no | reads fine — the tree is 0755, so world-readable anyway |
| sonarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
| radarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
| nextcloud | 33 (`www-data`) | no | reads and traverses 0755 fine |
**Without the added `PUID`/`PGID`, sonarr and radarr would have failed exactly the way this tier
always fails: the mount succeeds, the pod is Ready, and every import fails EACCES.** LinuxServer
images default to uid/gid **911**, which owns nothing on that tree.
`PGID: 3000` is chosen over 1000 so files these write are group-owned by the dominant group already
on the tree, keeping them readable to the live non-k3s consumers.
`podSecurityContext.fsGroup: 3000` is also added to sonarr and radarr — **for the Longhorn config
PVC, not for the NFS mount.** fsGroup does work on a Longhorn filesystem, and it makes `/data/config`
writable by the now-uid-1000 process without depending on the image re-chowning it after `PUID`
changed.
Nextcloud needs no uid work at all, which is one more reason its mount is read-only.
---
## 7. The prod placeholder does NOT fail at apply time — read this before merging
Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation
(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.**
A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts
`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure
control-plane and never touches the server. The failure appears only when a pod mounts:
```
MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER
```
so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.**
**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a
revision that carries these bulk volumes.** Dev renders a working value and is where these manifests
are meant to be proven first.
### The static guard this convention leans on is missing from `main`
`bootstrap/standup.sh` — which carries the `*_is_placeholder` guard family — **does not exist on
`main`** (verified: `git ls-tree main bootstrap/` lists only `README.md`, `flux/`, `gitea/`, `sops/`;
the file lives on unmerged prod branches). So
`conformance/tests/l0-static/placeholder-guard-coupling/check.sh` exits **3 SKIPPED**, and **nothing
in the gate currently refuses a placeholder-shaped overlay value.**
When `standup.sh` lands, that check's assertions (2) and (3) cover this knob automatically — its
independent detector's SEED vocabulary already matches `TBD` — **provided** the guard gains an arm
for `BULK_NFS_SERVER` and a consumer that calls it. Without the arm, assertion (2) is what fails, by
design.
---
## 8. Loose ends and things deliberately not done
1. **`/data/config` is very likely the wrong home for the arrs' config PVC — a pre-existing latent
bug, not fixed here.** LinuxServer's arr images run with `-data=/config`, so the app writes
`/config`, which is the container's **ephemeral layer**, while the Longhorn PVC mounted at
`/data/config` sits unused. This is the exact failure `catalog/plex.yaml`'s `mountPath` comment
records having already been bitten by ("the PVC sat unused and Plex wrote to the container's
EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it
means those apps start from an empty config volume, so it belongs in its own change with the
operator present. **This change does not make it worse** — that is why the media mount is `/media`.
2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy
non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s
qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were
chosen so it needs no Remote Path Mapping.
3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside
the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log
it while scanning `downloads/`.
4. **`nfsvers=4.2` was not chosen**, though the server advertises it. Nothing in these workloads uses
what 4.2 adds (server-side copy, sparse-file `SEEK`/`ALLOCATE`), and 4.1 is the version the
existing `bulk` StorageClass stub already named.
5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning
change that should be measured, not guessed, and it changes client behaviour under server
failure.
6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this
workstation (browser-OIDC only), and this change is manifest-authoring under an explicit
do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and
mounting* is reasoned from the manifests.
7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.**
`ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the
in-cluster conformance test for the bulk tier is still a gap, so typing it would select a
requirement the gate cannot satisfy — the hazard `catalog/qbittorrent.yaml` already warns about.
Scaffolding that test needs a cluster and is the correct follow-up.
---
## 9. Validating on dev, once the export is extended
Dev can prove all of this before prod's network path exists. Read-only checks first:
```sh
infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv
infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive'
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide
```
Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three
of them naming the *same* export, one per namespace, which is the cross-namespace model) — each
`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`.
Then the two things that actually matter, from inside a pod:
```sh
# (a) the writer can write — this is the uid check, and it is the one that fails in practice
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK'
# (b) hardlinks work BETWEEN the library and the download path — the root-mount check
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link;
echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK'
# (c) the reader really is read-only
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \
sh -c 'touch /media/.probe 2>&1 | head -1' # expect "Read-only file system"
```
`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it
prints `EXDEV`, something has split the single mount back into two.
schemas/renderer-contract.md
+41 -4
View File
@@ -72,11 +72,48 @@ platform must render, and cites the requirement each mapping satisfies. Changing
- **Requirements:** **R-NET-01** (default-deny baseline), **R-NET-02** (explicit allow / deny
undeclared). Auditable via Hubble (R-SEC-03, R-OBS-04).
### `storage[]` (`name` / `size` / `class`) — PVCs
### `storage[]` (`name` / `size` / `class` / `mountPath` / `export` / `subPath` / `readOnly`) — PVs + PVCs
- `class: longhorn` → a **PVC** on the 3-replica Longhorn StorageClass sized by `size` (replicated
config/DB/app state). For a `vm`, the disk is a Longhorn volume via CDI.
- `class: bulk` → binding to the external **bulk** media tier (NFS/SMB indirection, sized outside the
cluster; `size` optional). Not 3× replicated.
config/DB/app state), `ReadWriteOnce`, dynamically provisioned. For a `vm`, the disk is a Longhorn
volume via CDI.
- `class: bulk` → a **static, pre-bound external NFS PersistentVolume _and_ its PVC**. The entry's
`export` is the absolute path on the bulk server; the SERVER is never named in a ServiceDefinition
— it is the `${BULK_NFS_SERVER}` cluster-vars knob, so one authored definition promotes across
environments that reach the same storage by different addresses (**R-ENV-02**). Not 3× replicated:
durability is the storage host's ZFS pool, not Longhorn.
- `storageClassName: ""` on **both** PV and PVC (explicitly empty ⇒ statically provisioned; no
provisioner is ever asked to create anything, and the default-StorageClass annotation is
suppressed). Deliberately NOT the `bulk` StorageClass: a StorageClass `provisioner` is
**immutable**, so re-pointing that stub would make Flux fail permanently on it.
- Binding is pinned from both ends — `claimRef` on the PV, `volumeName` on the PVC — so no other
claim of matching size/mode can win a race for the volume.
- `persistentVolumeReclaimPolicy: Retain`, always. Bulk data pre-exists the cluster, is far larger
than the cluster, and may have consumers outside it; nothing here may reclaim it.
- Access mode follows the read/write posture and is always `…Many`, because one export is
consumed by several namespaces and several nodes at once: `ReadOnlyMany` when `readOnly`,
`ReadWriteMany` otherwise. `ReadWriteOnce` cannot express either.
- `mountOptions` are a **platform** decision held in the renderer, not a per-service field:
`nfsvers=4.1,hard,noatime,timeo=600,retrans=2`. NFSv4 carries mount and lock in-protocol, so
the only port that must be reachable is **tcp/2049** — no rpcbind/111, mountd or statd, which is
what keeps the firewall surface to a single rule. `hard` is chosen over `soft` because a soft
mount converts a server hiccup into a short read handed to the application.
- `size` on a bulk entry is the PV's **advertised capacity and nothing more** — no component
enforces it (the ZFS pool is the real limit), so it should state the dataset's actual size so
`kubectl get pv` reports something true.
- **fsGroup does not apply.** The in-tree NFS volume plugin performs no ownership management, so
`podSecurityContext.fsGroup` is a **no-op** on a bulk volume. A bulk mount that must be written
therefore requires the container's own uid/gid to be one the export's ownership permits (e.g.
Plex's `PLEX_UID`/`PLEX_GID`, LinuxServer images' `PUID`/`PGID`) — or a server-side squash. A
mount that succeeds but cannot write is the characteristic failure of this tier.
- `mountPath` (any class) overrides the `/data/<name>` default verbatim.
- `subPath` (any class) mounts a subtree instead of the volume root. **Two `subPath` mounts of one
export are two separate mounts**, so `link()` between them fails `EXDEV`; a workload that needs
hardlinks or atomic moves between two directories (Sonarr/Radarr importing from a download path
into a library) must mount their common ancestor **once** instead.
- `readOnly` (any class) marks the volumeMount read-only, and on a bulk entry also sets the PV's
`nfs.readOnly` so the kernel mount really is `ro`. Note this is a **client-side** guarantee: one
export serving both a reader and a writer that may land on any node cannot distinguish them
server-side.
- State separation rule: anything that must survive a rebuild lives on a named volume here, never on
ephemeral fs/hostPath — so the workload can be destroyed and re-applied from git with data intact.
- **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication),
schemas/servicedefinition.schema.json
+27 -5
View File
@@ -2,7 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
"title": "ServiceDefinition",
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.3.0 (adds optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
"type": "object",
"additionalProperties": false,
"required": ["name", "source"],
@@ -153,7 +153,7 @@
},
"storage": {
"type": "array",
"description": "Persistent volumes. If it must survive a rebuild it lives here, never on ephemeral/hostPath (docs/04; R-STATE-01/02/04).",
"description": "Persistent volumes. If it must survive a rebuild it lives here, never on ephemeral/hostPath (docs/04; R-STATE-01/02/04). A `longhorn` entry renders a dynamically provisioned PVC; a `bulk` entry renders a STATIC, pre-bound NFS PersistentVolume + its PVC over data that already exists outside the cluster.",
"default": [],
"items": {
"type": "object",
@@ -167,7 +167,7 @@
},
"size": {
"type": "string",
"description": "PVC request in Kubernetes quantity form (e.g. 5Gi). Required for the longhorn tier; may be omitted for the bulk tier (external mount, sized outside the cluster - docs/04).",
"description": "PVC request in Kubernetes quantity form (e.g. 5Gi). Required for the longhorn tier. For the bulk tier it is the STATIC PV's advertised capacity: NOTHING enforces it (the ZFS pool is the real limit), so state the dataset's current size so `kubectl get pv` reports something true. Omitted => the bulk PV/PVC carry no capacity, which the API rejects, so bulk in practice sets it.",
"pattern": "^[1-9][0-9]*(Mi|Gi|Ti)$"
},
"mountPath": {
@@ -177,11 +177,33 @@
},
"class": {
"type": "string",
"description": "longhorn = 3-replica block storage for app/config/DB state (R-STATE-02); bulk = large external media tier, NOT 3x replicated (R-STATE-04).",
"description": "longhorn = 3-replica block storage for app/config/DB state (R-STATE-02); bulk = large external media tier, NOT 3x replicated, backed by a STATIC NFS PersistentVolume the renderer emits (requires `export`).",
"enum": ["longhorn", "bulk"],
"default": "longhorn"
},
"export": {
"type": "string",
"description": "bulk ONLY: the absolute NFS export path on the bulk server (e.g. /mnt/Storage/plex). The SERVER is NOT named here - it is the ${BULK_NFS_SERVER} cluster-vars knob, so the same ServiceDefinition promotes across environments (R-ENV-02). Required whenever class is bulk; forbidden otherwise.",
"pattern": "^/.*[^/]$"
},
"subPath": {
"type": "string",
"description": "Optional relative path WITHIN the volume to mount instead of its root. NOTE for hardlink/atomic-move workloads (Sonarr/Radarr): two subPath mounts of the same export are two separate mounts, so link() across them fails EXDEV. Mount the common ancestor once instead.",
"pattern": "^[^/].*[^/]$"
},
"readOnly": {
"type": "boolean",
"description": "Mount this volume read-only. For a bulk volume this also sets the PersistentVolume's nfs.readOnly (the mount really is `ro`) and makes its access mode ReadOnlyMany instead of ReadWriteMany.",
"default": false
}
}
},
"allOf": [
{
"if": { "properties": { "class": { "const": "bulk" } }, "required": ["class"] },
"then": { "required": ["export"] },
"else": { "not": { "required": ["export"] } }
}
]
}
},
"dependencies": {