diff --git a/catalog/qbittorrent.yaml b/catalog/qbittorrent.yaml index 4803b27..53f8674 100644 --- a/catalog/qbittorrent.yaml +++ b/catalog/qbittorrent.yaml @@ -359,7 +359,7 @@ storage: # The live /config contents were copied onto each PVC before this landed, # so API keys and credentials other apps reference were preserved. # THE DOWNLOADS AREA — ONE mount of the SAME export sonarr/radarr/plex already consume - # (192.168.2.102:/mnt/Storage/plex), narrowed by subPath to the `downloads` subtree and presented at + # (192.168.2.30:/mnt/Storage/plex), narrowed by subPath to the `downloads` subtree and presented at # /downloads, which is precisely where this app is ALREADY configured to write. # # WHY subPath: downloads AND NOT THE EXPORT ROOT AT /media (the shape sonarr/radarr use): @@ -434,7 +434,7 @@ dependencies: [] # default-deny: reaches only DNS; arr reach it n # for weeks and neither has any CNP naming the NFS server or port 2049 # (grep apps/wave-1/sonarr.yaml: none). The SAME fact is what keeps the VPN # egress gateway off this traffic (its selector is a podSelector), and - # 192.168.2.102 is inside the 192.168.0.0/16 excludedCIDR besides — see + # 192.168.2.30 is inside the 192.168.0.0/16 excludedCIDR besides — see # platform/vpn-egress/egress-steering.yaml, "BULK NFS". ingressRoutes: - host: qbittorrent.${BASE_DOMAIN} diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index b7dbddf..ed13791 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -234,26 +234,28 @@ data: WG_CLIENT_CIDR: "10.90.0.0/24" # remote VPN clients; .1 = server. NOT a LAN range # --- Bulk (external NFS) storage host (docs/04 bulk tier) --------------------------------------- - # THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media - # library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive + # THE address of the NAS whose ZFS datasets back every `class: bulk` volume: the media library + # (Storage/plex, 8.33 T — plex, sonarr, radarr, qbittorrent) and the legacy Nextcloud archive # (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's # `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is # declared in the ServiceDefinition, because the path is identical in every environment and only # the ADDRESS is not. # - # WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense. - # The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN. - # dev nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one - # NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in - # /etc/exports), so this path is proven, not theoretical. Base default = the dev value, - # per the convention of the LAN block above. - # prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today; - # 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening - # the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give - # DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route). - # So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the - # blast radius of shipping it unset, which is NOT the usual "fails to apply". - BULK_NFS_SERVER: "192.168.1.9" + # IT IS NO LONGER A CROSS-NETWORK KNOB (2026-10-07). This used to read "the two clusters are on + # OPPOSITE SIDES of pfSense": the datasets lived on a Proxmox host whose only interface was + # vmbr0 192.168.1.9/24 on pfSense's LAN, which prod (192.168.2.0/24, pfSense's WAN) could reach + # only through a DNAT at 192.168.2.102. Both pools were physically moved onto a standalone + # TrueNAS at 192.168.2.30 — prod's OWN /24 — so there is no crossing to arrange and no candidate + # values to choose between. Measured 2026-10-07: 192.168.2.102:2049 no longer answers; + # 192.168.2.30:2049 and :111 do. There is now exactly ONE storage host and one correct value; + # this stays a knob only because the renderer reads it per-environment. + # + # *** CHANGING THIS VALUE IS NOT A PLAIN RECONCILE. *** A bound PersistentVolume's spec.nfs is + # immutable — measured by server dry-run 2026-10-07: + # spec.persistentvolumesource: Forbidden: spec.persistentvolumesource is immutable after creation + # so Flux's apply FAILS rather than updating, and the pods keep their old mount. The PVs must be + # deleted and recreated; the operator sequence is docs/BULK-NFS-STORAGE.md §10. + BULK_NFS_SERVER: "192.168.2.30" # Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style. # Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db, diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 2ed0b54..3e4fc87 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -279,15 +279,15 @@ patches: - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.1.55" - # Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes - # (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes). - # Restated verbatim rather than inherited, per this overlay's "supply every knob" convention. - # Dev is therefore the environment in which the bulk manifests can actually be VALIDATED - # end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=` - # change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses. + # Bulk (external NFS) storage host. THIS VALUE IS INERT — dev is decommissioned, its nodes + # (192.168.1.40/.41/.42) no longer exist, and the Proxmox host this used to name (192.168.1.9) + # went with the pools it held. It is restated per this overlay's "supply every knob" + # convention, not because anything consumes it. It carries the real address so that no file + # in the tree renders a host that is gone; it has NOT been proven from a dev node, because + # there is no dev node to prove it from. - op: replace path: /data/BULK_NFS_SERVER - value: "192.168.1.9" + value: "192.168.2.30" # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG # barmanObjectStore and Velero's backupStorageLocation write to. # diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index dab3d63..ad562df 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -540,60 +540,48 @@ patches: path: /data/LAN_CIDR value: "192.168.2.0/24" # ------------------------------------------------------------------------------------------ - # BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. *** + # BULK (EXTERNAL NFS) STORAGE HOST — the TrueNAS at 192.168.2.30. # ------------------------------------------------------------------------------------------ # Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr, - # radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud). + # radarr, qbittorrent) and the 270 G legacy Nextcloud archive (Storage/nextcloud). # - # THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface, - # vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN. - # 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102; - # 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate - # outcomes give DIFFERENT values here: - # port-forward 192.168.2.102:2049 -> 192.168.1.9:2049 => "192.168.2.102" - # a WAN->LAN route to 192.168.1.0/24 => "192.168.1.9" - # Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess - # ever became reachable — mounts something unintended. So it stays a placeholder. + # THIS WAS THE REPO'S ONE LOAD-BEARING PLACEHOLDER, AND THE REASON IT COULD BE IS NOW GONE. + # The datasets lived on a Proxmox host with a single interface on pfSense's LAN + # (192.168.1.9), which prod could reach only through a DNAT at 192.168.2.102 — so the correct + # value here depended on an undecided operator choice, and guessing it would have authored a + # cluster that mounts nothing. Both pools were physically moved onto a standalone TrueNAS at + # 192.168.2.30, prod's OWN /24. No crossing to arrange, no candidates to choose between. + # Measured 2026-10-07: 192.168.2.102:2049 no longer answers; 192.168.2.30:2049 and :111 do. # - # WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and - # NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4 - # over v3 was in part this decision's cost: v3 would have needed four ports and three of them - # on ephemeral numbers. - # - # WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes. - # Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward: - # the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN - # address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via - # 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node - # addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md. - # - # *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. *** - # A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this - # placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The - # failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server - # TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in - # ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at - # apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles - # a revision containing the bulk volumes; the dev overlay renders a working value, which is - # where these manifests are meant to be proven first. + # ONE PORT IS STILL ALL IT NEEDS: tcp/2049. The PVs mount nfsvers=4.1 and NFSv4 carries mount + # and lock in-protocol — no rpcbind/111, no mountd, no statd. The NAS advertises + # `+3 +4 +4.1 +4.2` (/proc/fs/nfsd/versions, measured), so 4.1 negotiates. - op: replace path: /data/BULK_NFS_SERVER - # 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox - # host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot - # reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and - # keeps the base value 192.168.1.9. + # VERIFIED ON THE NAS 2026-10-07, before any cluster touched it (`exportfs -v`): + # /mnt/Storage/plex 192.168.2.10/.11/.12 rw,sec=sys,secure,root_squash,no_all_squash + # /mnt/Storage/nextcloud 192.168.2.10/.11/.12 ro,sec=sys,secure,root_squash,no_all_squash # - # PROVEN from k3s-prod-1 on 2026-09-12, not inferred: - # - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp - # - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export - # carries no_root_squash, so a root write would have passed and proved nothing, while - # LSIO images default to PUID 911 and would have gone Ready then failed every import - # with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer. - # - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get - # real atomic moves instead of silently degrading to copy-then-delete on EXDEV. - # pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer), - # so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP. - value: "192.168.2.102" + # no_all_squash is the flag the entire permission model rests on: client uids pass through + # untouched, so the arrs' PUID=1000 arrives as the owner of a 0755 1000:3000 tree. sec=sys + # keeps that numeric rather than principal-mapped. The export deliberately does NOT carry + # no_root_squash, which the Proxmox one did: nothing here needs it, because every write to + # the tree happens after the LSIO drop to uid 1000 and the images only chown their Longhorn + # /config. Both subPath directories (plex/downloads, nextcloud/nextcloud) already exist at + # 0755 1000:3000, so a squashed root can still traverse them for subPath resolution. + # + # The ro= posture on the archive is now SERVER-enforced, which the old export also managed. + # The media tree's read-only half (plex) stays a CLIENT-side guarantee — all four media pods + # can land on any node, so the server sees the same three addresses for reader and writers + # alike and cannot tell them apart. Unchanged by the move; see docs/BULK-NFS-STORAGE.md §5. + # + # *** CHANGING THIS VALUE REQUIRES DELETING AND RECREATING THE PVs. *** A bound PV's + # spec.nfs is immutable — measured by server dry-run 2026-10-07: + # spec.persistentvolumesource: Forbidden: spec.persistentvolumesource is immutable after creation + # so Flux's apply FAILS rather than updating and the pods keep their old, dead mount. The + # operator sequence is docs/BULK-NFS-STORAGE.md §10. Retain + the renderer's pre-set + # claimRef make it safe: no data is touched and each recreated PV rebinds its own PVC. + value: "192.168.2.30" # ========================================================================================== # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07). # ========================================================================================== diff --git a/decisions/2026-10-07-bulk-nfs-truenas.md b/decisions/2026-10-07-bulk-nfs-truenas.md new file mode 100644 index 0000000..a133bd6 --- /dev/null +++ b/decisions/2026-10-07-bulk-nfs-truenas.md @@ -0,0 +1,85 @@ +--- +date: 2026-10-07 +branch: fix/bulk-nfs-truenas +title: Repoint bulk NFS at the TrueNAS (192.168.2.30); the PVs must be recreated, not reconciled +--- + +## What changed + +Both ZFS pools were physically moved off the Proxmox host onto a standalone TrueNAS Community +Edition at **192.168.2.30**, which sits on prod's own `192.168.2.0/24`. `BULK_NFS_SERVER` moves from +`192.168.2.102` (the pfSense DNAT prod used to mount through) to `192.168.2.30`. + +- `clusters/prod/kustomization.yaml` — `BULK_NFS_SERVER` → `192.168.2.30`. The comment block that + justified the DNAT, and the "NOT YET DECIDED / THIS PLACEHOLDER IS LOAD-BEARING" banner, are + replaced with the measured export configuration and the recreate requirement below. +- `clusters/base/cluster-vars.yaml` — base default `192.168.1.9` → `192.168.2.30`. That host no + longer exists. +- `clusters/dev/kustomization.yaml` — same; marked explicitly **inert**, since dev is decommissioned + and its nodes are gone. It carries the real address only so nothing in the tree renders a dead one. +- `docs/BULK-NFS-STORAGE.md` — §1 and §5 rewritten for TrueNAS (exports, service settings, the ZFS + `sharenfs` properties that had to be cleared, and the ACL hazard); §7 rewritten now that the + placeholder is gone; §9 retargeted from dev to prod; **new §10** documents the PV + delete-and-recreate. §2/§3/§4/§6 are unchanged by the move. +- `catalog/qbittorrent.yaml`, `platform/monitoring/node-pressure-dashboard.yaml` — two comment/prose + references to `192.168.2.102` as the live export address. + +**The headline operational fact: repointing this knob is NOT a plain reconcile.** A bound PV's NFS +source is immutable, so Flux's apply fails rather than updating and the pods keep their old, dead +mount. The five PVs must be deleted and recreated. That sequence is §10 and is an operator action; +it is **not** performed by this PR. + +## What I verified + +**On the NAS (192.168.2.30), measured 2026-10-07 by the operator at my request:** + +- `zfs list` — `Storage` → `/mnt/Storage`, `Storage/plex` → `/mnt/Storage/plex`, + `Storage/nextcloud` → `/mnt/Storage/nextcloud`. The PV paths resolve unchanged. +- Ownership intact: `/mnt/Storage/plex`, `tv/`, `downloads/`, `movies/`, + `downloads/complete/`, `nextcloud/nextcloud/` all `drwxr-xr-x 1000:3000`. `movies/` 1171 entries + and `tv/` 107, against 1165 / 107 measured on Proxmox. +- `acltype=nfsv4` (SCALE's built-in default, `SOURCE: default`), `aclinherit=passthrough`, and no + non-trivial ACLs — mode bits still govern. +- Three stale `sharenfs` dataset properties (`plex`, `nextcloud`, `docker-volumes`) carried over from + Proxmox; cleared with `zfs inherit`, re-checked empty. +- `exportfs -v`: both shares to `192.168.2.10/.11/.12` with `sec=sys,secure,root_squash,no_all_squash`, + `rw` on plex and `ro` on nextcloud. +- `/proc/fs/nfsd/versions` → `+3 +4 +4.1 +4.2`, so the PVs' `nfsvers=4.1` negotiates. +- **A defect was found and fixed during setup:** the plex share listed `192.168.2.13` — an address no + node holds — and so omitted `192.168.2.12` (k3s-prod-3). Corrected and re-verified. + +**From the cluster:** + +- Node InternalIPs are `192.168.2.10/.11/.12` (`kubectl get nodes`), confirming `.13` was wrong. +- All five NFS PVs currently name `192.168.2.102`. +- `192.168.2.30:2049` and `:111` accept connections; `192.168.2.102:2049` does not. +- **`spec.nfs` immutability, by server-side dry-run** — this is the finding that shapes the change: + `spec.persistentvolumesource: Forbidden: spec.persistentvolumesource is immutable after creation`. +- The pods report `Ready` while their mounts are dead: a `stat /media` answers from attribute cache, + but a `readdir` blocks indefinitely (the PVs mount `hard`). Hence "scale to zero first" in §10. + +**Gate:** see the PR; run in the pinned toolbox from this worktree. + +## What I did NOT verify + +- **Nothing has mounted 192.168.2.30.** No pod has touched the new server — the PVs still point at + the old address and cannot be updated in place. Every §9 probe (numeric-id passthrough, the uid-1000 + write, the hardlink/EXDEV check, Plex's read-only posture) is **outstanding**, not passing. The + equivalents passed against the old host on 2026-09-12; that is not evidence about this one. +- **The §10 sequence is reasoned, not executed.** Retain + the renderer's pre-set `claimRef` are read + from the manifests and from `catalog/render.py`; I have not watched a PVC rebind after a PV delete + in this cluster. +- **`root_squash` is a behaviour change from the Proxmox export**, which carried `no_root_squash`. The + argument that nothing needs root on the tree (LSIO drops to 1000; the images chown only their + Longhorn `/config`; both subPath directories already exist at 0755) is reasoned from the images and + the measured permissions. It has not been exercised. If a mount or subPath resolution fails with + EACCES, Maproot `root:root` reproduces the old export and is the first thing to try. +- **Whether NFSv3 can be disabled.** Enabled Protocols is `NFSv3, NFSv4`; the PVs only need v4.1, so + narrowing would close the rpcbind/mountd/statd surface. Not done — it is a change to make after the + cutover is proven, not during it. +- **Whether pfSense / the legacy stack is actually retired.** I measured only that + `192.168.2.102:2049` stopped answering. The other legacy consumers of these datasets + (`192.168.1.6`, `10.123.1.10`, `192.168.1.18`) were dropped from the new exports; if any is still + live, it has lost access. Not checked. +- **L2/L3 conformance.** `vars.CLUSTER_AVAILABLE` is unset, so CI proves the L0/L1 offline subset + only. Nothing in the gate exercises an NFS mount. diff --git a/docs/BULK-NFS-STORAGE.md b/docs/BULK-NFS-STORAGE.md index 3d99879..8974a6d 100644 --- a/docs/BULK-NFS-STORAGE.md +++ b/docs/BULK-NFS-STORAGE.md @@ -1,61 +1,43 @@ # Bulk NFS storage — the media library and the legacy Nextcloud archive -The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`. -Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is -assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the +The operator-facing companion to the `class: bulk` storage entries in +`catalog/{plex,sonarr,radarr,qbittorrent,nextcloud}.yaml`. + +**Rewritten 2026-10-07 for the move off Proxmox onto a standalone TrueNAS at 192.168.2.30.** The +layout (§2), the mount design (§3, §4) and the uid model (§6) are unchanged by the move and were +measured on the old host on 2026-09-07. Everything about the storage host itself — §1, §5, §10 — +was re-measured on the NAS on 2026-10-07. Nothing here is assumed. It resolves the long-standing +**OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk provisioning unimplemented, because nothing needs it. --- -## 1. The storage host, and why prod cannot reach it +## 1. The storage host -The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface: +The bulk data lives on a **standalone TrueNAS** (Community Edition) at **192.168.2.30** — the same +`192.168.2.0/24` as prod's nodes (`192.168.2.10/.11/.12`), so it is reached directly, with no NAT +and no routing decision. -``` -vmbr0 192.168.1.9/24 default via 192.168.1.1 (pfSense LAN) -``` - -| | nodes | reaches 192.168.1.9? | -|---|---|---| -| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) | -| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. | - -Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give -different values for the `BULK_NFS_SERVER` cluster-var: - -| outcome | `BULK_NFS_SERVER` for prod | -|---|---| -| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` | -| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` | - -Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder -behaves differently from the repo's other ones.** +**This section used to be the hard part of this page.** The datasets were on the Proxmox host, whose +only interface was `vmbr0 192.168.1.9/24` on pfSense's **LAN**, while prod sits on pfSense's **WAN**. +tcp/2049 did not traverse that, so prod mounted through a DNAT at `192.168.2.102`, and which address +belonged in `BULK_NFS_SERVER` depended on an undecided operator choice. Both pools were physically +moved on 2026-10-07 (`docs/plans/proxmox-to-n5-air-migration.md`, Phase C/D) and all of that is +simply gone. Measured 2026-10-07: **192.168.2.102:2049 no longer answers; 192.168.2.30:2049 and +:111 do.** ### It only needs ONE port The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed -four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The -server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the -conservative floor. +four ports, three of them on ephemeral numbers. The NAS advertises `+3 +4 +4.1 +4.2` +(`/proc/fs/nfsd/versions`, measured 2026-10-07), so 4.1 is available and is the conservative floor. -### The export's client list is the SAME under both outcomes — measured, not assumed - -**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation -(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`: - -``` -SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22 -ss -tn -> 192.168.1.9:22 192.168.2.132:39768 -``` - -The original WAN-side source address survives the translation (return traffic finds its way back via -the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's -**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either -way. **It does not have to wait for the routing decision.** - ---- +**NFSv4 is not on by default and its absence does not look like a storage problem.** TrueNAS ships +NFSv3-only until NFSv4 is added to *Services → NFS → Enabled Protocols*. Without it the mount never +completes and the pods sit in `ContainerCreating` — no permission error, nothing in the app logs, +just no volume. ## 2. The layout, as found @@ -107,16 +89,18 @@ why it is mounted read-only; see §6. | plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany | | sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany | | radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany | +| qbittorrent | `/mnt/Storage/plex` | `/downloads` | `downloads` | rw | ReadWriteMany | | nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany | In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from `/media/downloads/complete/{tv-sonarr,radarr}`. -**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually -given the same export (it is not in this change), the arrs and the download client will already agree -on absolute paths, which is the second precondition for hardlinked imports — otherwise every import -needs a Remote Path Mapping and quietly falls back to copying when one is missing. +**The three library consumers use the identical mountPath on purpose**, and qbittorrent has since +been given the same export (narrowed by `subPath: downloads`, presented at `/downloads` to match its +existing `Session\DefaultSavePath`). The arrs and the download client therefore agree on absolute +paths, which is the second precondition for hardlinked imports — otherwise every import needs a +Remote Path Mapping and quietly falls back to copying when one is missing. **`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the @@ -147,62 +131,123 @@ Narrowing that would cost hardlinks, which is the worse trade. --- -## 5. The export change the operator must apply — STRICTLY ADDITIVE +## 5. The exports, as configured on TrueNAS — and the settings that decide permissions -**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the -exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and -192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the -live non-k3s prod stack. +**Do not apply this from the repo.** It is operator configuration on the NAS, recorded here so the +exact values are reviewable. Everything below was **measured on 192.168.2.30 on 2026-10-07**, after +the move. -Current values (`zfs get sharenfs Storage`): +### The exports ``` -Storage/plex rw=@192.168.1.6:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash -Storage/nextcloud rw=@192.168.1.6:@192.168.1.5,no_subtree_check,no_root_squash +/mnt/Storage/plex 192.168.2.10 192.168.2.11 192.168.2.12 + (sync,wdelay,hide,no_subtree_check,sec=sys,rw,secure,root_squash,no_all_squash) +/mnt/Storage/nextcloud 192.168.2.10 192.168.2.11 192.168.2.12 + (sync,wdelay,hide,no_subtree_check,sec=sys,ro,secure,root_squash,no_all_squash) ``` -### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list +Per-node addresses rather than `@192.168.2.0/24`, following the precedent of the old host. A `/24` +would work and would survive adding a node; it also grants the whole subnet. + +**All three nodes must appear on BOTH shares.** Any media pod can be scheduled on any node, so a +missing node is a latent failure, not an immediate one: it surfaces the first time a pod reschedules +there, as `access denied by server while mounting`, with the cause weeks stale. One was caught +exactly this way during setup — the plex share listed `192.168.2.13`, an address no node holds, and +so was missing `192.168.2.12` (k3s-prod-3). Nothing was failing at the time. + +### The option flags ARE the permission contract + +| flag | why it matters here | +|---|---| +| **`no_all_squash`** | The one the whole model rests on. Client uids pass through untouched, so the arrs' `PUID=1000` arrives as the owner of a `0755 1000:3000` tree. In the UI this is **Mapall User/Group left EMPTY** — setting either squashes every client to one identity and erases the reader/writer distinction entirely. | +| **`sec=sys`** | AUTH_SYS. Numeric uid/gid, not Kerberos principals. §6 is only true because of this. | +| **`root_squash`** | Maproot left empty. The Proxmox export carried `no_root_squash`; **nothing here needs it** — every write to the tree happens after the LSIO drop to uid 1000, and the images only chown their Longhorn `/config`. This is a deliberate tightening over the old export, not parity. | +| **`rw` / `ro`** | The archive's read-only posture is **server-enforced**, as it was before. The media tree cannot get the same treatment — see below. | +| **`secure`** | Reserved source ports required; "Allow non-root mount" off. The kubelet mounts as root from a reserved port and does not need the exception. | + +**Plex's read-only posture is still a CLIENT-side guarantee.** Plex is ro and the arrs are rw, but +all four can land on any node, so the server sees the same three source addresses for all of them +and cannot distinguish the reader from the writers. `spec.nfs.readOnly: true` on `plex-media` plus +the volumeMount flag is the whole enforcement. Unchanged by the move; an honest limitation, not an +oversight. If it ever needs to be server-enforced, the lever is a dedicated node pool for Plex. + +### Service settings that are NOT obvious from the export list + +- **Enabled Protocols must include NFSv4** (above). +- **Require Kerberos for NFSv4: off.** The numeric AUTH_SYS pass-through is the model. +- **Manage Groups Server-side: off.** This is `rpc.mountd --manage-gids`: the server *discards the + client's supplied group list* and looks that uid's groups up in TrueNAS's own user database + instead. With no local uid 1000 defined, the arrs' `PGID=3000` would simply vanish. Writes would + survive anyway — uid 1000 **owns** the tree, so the owner bits grant access without the group — + but it is a free way to lose group semantics. +- **NFSv4 DNS Domain: blank.** It only feeds `idmapd.conf`, which numeric mapping bypasses. +- **Bind IP 192.168.2.30** means loopback does not answer: check with `showmount -e 192.168.2.30`, + not `localhost`. + +There is **no** "NFSv3 ownership model for NFSv4" control on current TrueNAS — it was an older +TrueNAS/CORE option, and no replacement is needed: on a Linux client against a Linux server over +AUTH_SYS, both the `nfs` and `nfsd` modules default to `nfs4_disable_idmapping=Y`, so NFSv4 carries +raw numeric ids rather than `user@domain` principals. §9's `ls -n` probe is how you confirm it. + +### The ZFS `sharenfs` properties must be CLEARED, not edited + +The pools arrived with the old host's exports still set as dataset properties: + +``` +Storage/plex rw=@192.168.1.6:@10.123.1.10:@192.168.1.18:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash +Storage/nextcloud rw=@192.168.1.6:@192.168.1.5,ro=@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash +Storage/docker-volumes rw=@192.168.1.6:@192.168.1.5:@192.168.1.10:@192.168.1.18,no_subtree_check,no_root_squash +``` + +TrueNAS generates `/etc/exports` from its own middleware database and will fight a property it did +not set — the failure mode is two differently-configured exports of one path, with no indication +which is in force. Cleared 2026-10-07: ```sh -zfs set sharenfs='rw=@192.168.1.6:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex +zfs inherit sharenfs Storage/plex +zfs inherit sharenfs Storage/nextcloud +zfs inherit sharenfs Storage/docker-volumes ``` -Every pre-existing host is preserved verbatim and no option changes; only clients are added. +`Storage` is `off`, so each resolves to `off`. -### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=` +### Do NOT touch the dataset's permissions or ACLs -```sh -zfs set sharenfs='rw=@192.168.1.6:@192.168.1.5,ro=@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud -``` +The tree's ownership **is** the access story (§6), and TrueNAS offers two ways to destroy it +irreversibly across 8.33 T. -This is the one place read-only can be enforced **server-side**, and it is worth taking: only -Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a -misconfigured client cannot then damage the archive. +`Storage/plex` carries `acltype=nfsv4` (SCALE's built-in default, `SOURCE: default`) and +`aclinherit=passthrough` (inherited). Measured 2026-10-07: the directories carry **trivial ACLs +only** — `nfs4xdr_getfacl` shows just `owner@`/`group@`/`everyone@` matching the mode — so `0755 +1000:3000` governs exactly as it did under Proxmox. -**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but -all three may be scheduled on any node, so the server sees the same source addresses for all of -them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a -client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an -honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a -dedicated node pool for Plex, which is a much larger change. +- **Stay out of the dataset's Edit Permissions / Edit ACL screen.** On an `acltype=nfsv4` dataset + that is a full ACL editor, and its presets write *real* NFSv4 ACLs, recursively if permitted. + There is no undo. +- `aclinherit=passthrough` means a single ACL written at `/mnt/Storage/plex` would be **inherited by + everything created under it afterwards** — the blast radius is the whole future library, not one + directory. +- Note GNU `ls`'s `+` marker is **not** a reliable ACL indicator here: it looks for a POSIX ACL + xattr, and this dataset stores NFSv4 ACLs elsewhere. Use `nfs4xdr_getfacl`. -Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host -(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive -adding a node; it also grants the whole subnet. Per-node is the tighter default. +`xattr=on` rather than `sa` is a performance detail, not a permissions one, and changing it would +not affect existing files. ### Verifying, without changing anything ```sh -exportfs -v # the new client lines appear, existing ones unchanged -zfs get sharenfs Storage/plex Storage/nextcloud +cat /proc/fs/nfsd/versions # +4.1 must be present +exportfs -v # the two shares, three nodes each, flags as tabulated above +showmount -e 192.168.2.30 # NOT localhost -- the service is bound to one address +zfs get -r -o name,property,value,source sharenfs Storage # every row off or - ``` ---- - ## 6. uid / gid — the failure this design exists to avoid -The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through -verbatim** and the server's on-disk ownership is the entire access story. The media tree is +The export uses `sec=sys` and `no_all_squash`, so **the client's numeric uid is passed through +verbatim** and the server's on-disk ownership is the entire access story. (It no longer carries +`no_root_squash`, which the Proxmox export did — see §5. Nothing here needs it, and the uid that +matters is 1000, not 0.) The media tree is `0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**. **`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS @@ -233,24 +278,27 @@ Nextcloud needs no uid work at all, which is one more reason its mount is read-o --- -## 7. The prod placeholder does NOT fail at apply time — read this before merging +## 7. Why this knob is unvalidated at create AND immutable at update + +**The `TBD-PROD-BULK-NFS-SERVER` placeholder is gone** (2026-10-07) — prod now carries a real +address, because the NAS is on prod's own subnet and there is no longer a decision to defer. The +hazard that made it load-bearing is worth keeping on the page, because it is the half that explains +why §10 exists. Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation -(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.** +(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one was not.** -A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts -`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure -control-plane and never touches the server. The failure appears only when a pod mounts: +A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepted +`TBD-PROD-BULK-NFS-SERVER`, the PV was created, and the PVC **bound** to it — binding is pure +control-plane and never touches the server. The failure appeared only when a pod mounted: ``` MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER ``` -so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.** - -**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a -revision that carries these bulk volumes.** Dev renders a working value and is where these manifests -are meant to be proven first. +**The same field is immutable on UPDATE** (§10), which is the other half of the same awkwardness: a +wrong value is accepted silently at create, and then cannot be corrected in place. Together they are +why repointing this knob is a delete-and-recreate rather than a reconcile. ### The static guard this convention leans on is missing from `main` — but the gate now refuses the value anyway @@ -287,10 +335,9 @@ Without the arm, HALF A is what holds the line, by design. EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it means those apps start from an empty config volume, so it belongs in its own change with the operator present. **This change does not make it worse** — that is why the media mount is `/media`. -2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy - non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s - qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were - chosen so it needs no Remote Path Mapping. +2. **RESOLVED — qbittorrent now has the bulk mount** (`subPath: downloads` at `/downloads`), so the + import source is written by the k3s download client rather than the legacy non-k3s one. See + `catalog/qbittorrent.yaml`. 3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log it while scanning `downloads/`. @@ -300,10 +347,10 @@ Without the arm, HALF A is what holds the line, by design. 5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning change that should be measured, not guessed, and it changes client behaviour under server failure. -6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this - workstation (browser-OIDC only), and this change is manifest-authoring under an explicit - do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and - mounting* is reasoned from the manifests. +6. **Every claim about the NAS in §5 is measured (2026-10-07); the post-move MOUNT is not.** The PVs + still have to be recreated (§10) before any pod mounts 192.168.2.30, so the §9 probes are the + outstanding verification, not a formality. The pre-move equivalents did pass against the old host + on 2026-09-12. 7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.** `ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the in-cluster conformance test for the bulk tier is still a gap, so typing it would select a @@ -312,36 +359,95 @@ Without the arm, HALF A is what holds the line, by design. --- -## 9. Validating on dev, once the export is extended +## 9. Validating, after the PVs are recreated -Dev can prove all of this before prod's network path exists. Read-only checks first: +These run against **prod**; dev is decommissioned. They are the outstanding verification for the +move — nothing below has been run against 192.168.2.30. Read-only checks first: ```sh -infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv -infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive' -infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide +./infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get pv -o custom-columns=\ +NAME:.metadata.name,SERVER:.spec.nfs.server,PATH:.spec.nfs.path,STATUS:.status.phase' ``` -Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three -of them naming the *same* export, one per namespace, which is the cross-namespace model) — each -`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`. +Expect **five** NFS PVs — `plex-media`, `sonarr-media`, `radarr-media`, `qbittorrent-downloads`, +`nextcloud-legacy-archive` (four of them naming the *same* export, one per namespace, which is the +cross-namespace model) — each `Bound`, `RECLAIM POLICY Retain`, empty `STORAGECLASS`, and +**`SERVER 192.168.2.30`**. A PV still showing `192.168.2.102` means the recreate in §10 did not +happen; Flux will NOT have updated it in place. -Then the two things that actually matter, from inside a pod: +Then the four things that actually matter, from inside a pod: ```sh -# (a) the writer can write — this is the uid check, and it is the one that fails in practice -infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \ - sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK' +# (a) numeric ids survived the wire -- the idmapping check, new since the TrueNAS move +kubectl --context k3s-prod -n sonarr exec deploy/sonarr -c sonarr -- sh -c 'ls -n /media | head -3' +# expect uid 1000 / gid 3000 on the entries. 65534 / nobody means ids are being mapped, not +# passed through, and every import will fail EACCES while the pod still reports Ready. -# (b) hardlinks work BETWEEN the library and the download path — the root-mount check -infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \ +# (b) the writer can write -- the uid check, and the one that fails in practice +kubectl --context k3s-prod -n sonarr exec deploy/sonarr -c sonarr -- \ + sh -c 'touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK' + +# (c) hardlinks work BETWEEN the library and the download path -- the root-mount check +kubectl --context k3s-prod -n sonarr exec deploy/sonarr -c sonarr -- \ sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link; echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK' -# (c) the reader really is read-only -infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \ - sh -c 'touch /media/.probe 2>&1 | head -1' # expect "Read-only file system" +# (d) the reader really is read-only +kubectl --context k3s-prod -n plex exec deploy/plex -- sh -c 'touch /media/.probe 2>&1 | head -1' +# expect "Read-only file system" ``` -`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it +`(c)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it prints `EXDEV`, something has split the single mount back into two. + +Reading the failures together is what localises the fault: **(b) EACCES with (a) showing `nobody`** +is idmapping (§5 service settings). **(b) EACCES with (a) showing `1000`** is on-disk ownership — +stop and check `ls -ldn` on the NAS before changing anything else. + +--- + +## 10. Repointing BULK_NFS_SERVER means DELETING AND RECREATING THE PVs + +**A bound PersistentVolume's NFS source is immutable.** Measured by server-side dry-run on +2026-10-07: + +``` +$ kubectl patch pv sonarr-media --type=json --dry-run=server \ + -p '[{"op":"replace","path":"/spec/nfs/server","value":"192.168.2.30"}]' +The PersistentVolume "sonarr-media" is invalid: spec.persistentvolumesource: Forbidden: + spec.persistentvolumesource is immutable after creation +``` + +Note the pairing with §7: `spec.nfs.server` is **unvalidated at create** (any string is accepted and +the PVC binds) and **immutable at update**. So editing `BULK_NFS_SERVER` alone does not fail loudly +and does not work either — Flux's apply errors on the five PVs and the pods keep their previous, +now-dead mount. + +This is safe to do because `persistentVolumeReclaimPolicy: Retain` means deleting a PV never touches +the data, and `catalog/render.py` emits a **pre-set `claimRef`** on each one, so a recreated PV +rebinds its own PVC with no manual step. + +**Operator sequence** (mutating `kubectl` is an operator action, not an agent action): + +```sh +# 1. Stop the consumers. This also releases the hung hard-mounts against the old server: +# the pods report Ready, but any real I/O on /media blocks indefinitely once 2049 stops +# answering, because the PVs mount `hard`. +for ns in plex sonarr radarr qbittorrent nextcloud; do + kubectl --context k3s-prod -n $ns scale deploy --all --replicas=0 +done + +# 2. Delete the five PVs. Retain + no live claim means nothing is reclaimed or erased. +kubectl --context k3s-prod delete pv plex-media sonarr-media radarr-media \ + qbittorrent-downloads nextcloud-legacy-archive + +# 3. Let Flux reconcile the revision carrying BULK_NFS_SERVER=192.168.2.30, which recreates them. +# Confirm .status.lastAppliedRevision matches the GitRepository artifact before continuing. + +# 4. Bring the consumers back, then run every probe in §9. +``` + +The PVCs pass through `Lost` between steps 2 and 3; that is expected and resolves when the PV with +the matching `claimRef` reappears. If a PVC is still `Lost` after Flux reconciles, the PV name or +its `claimRef` namespace does not match — do **not** delete the PVC to "fix" it. + diff --git a/platform/monitoring/node-pressure-dashboard.yaml b/platform/monitoring/node-pressure-dashboard.yaml index d724d71..63aef5c 100644 --- a/platform/monitoring/node-pressure-dashboard.yaml +++ b/platform/monitoring/node-pressure-dashboard.yaml @@ -199,7 +199,7 @@ data: "id": 15, "type": "timeseries", "title": "CAUSE 4 — NFS RPC retransmission ratio, per node", - "description": "node_nfs_rpc_retransmissions_total / node_nfs_rpcs_total, from /proc/net/rpc/nfs. Alert line at 1% (NodeNFSRetransmissionRateHigh). MEASURED BASELINE IS EXACTLY ZERO on all three nodes over 48h, including throughout the 2026-09-14 spike — so any visible line here means the export at 192.168.2.102, or the path to it, is dropping requests. A flat zero does NOT clear NFS: it only clears the TRANSPORT. A merely SLOW export needs per-mount RPC round-trip time, which arrives with the mountstats collector (see kube-prometheus-stack.yaml) and is the known gap recorded in runbooks/node-io.md.", + "description": "node_nfs_rpc_retransmissions_total / node_nfs_rpcs_total, from /proc/net/rpc/nfs. Alert line at 1% (NodeNFSRetransmissionRateHigh). MEASURED BASELINE IS EXACTLY ZERO on all three nodes over 48h, including throughout the 2026-09-14 spike — so any visible line here means the export at 192.168.2.30, or the path to it, is dropping requests. A flat zero does NOT clear NFS: it only clears the TRANSPORT. A merely SLOW export needs per-mount RPC round-trip time, which arrives with the mountstats collector (see kube-prometheus-stack.yaml) and is the known gap recorded in runbooks/node-io.md.", "gridPos": {"h": 8, "w": 12, "x": 12, "y": 39}, "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "red", "value": 0.01}