diff --git a/catalog/render.py b/catalog/render.py index 3533e46..e75141d 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -53,6 +53,30 @@ LBL = "homelab.keithjackson.dev" # Hardcoding the literal here is what let prod be authored to serve dev's public hostnames, # and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10). ZONE = "${BASE_DOMAIN}" +# The bulk (external NFS) tier's SERVER, emitted as a Flux post-build TOKEN for exactly the same +# reason as ZONE: dev and prod reach the same Proxmox ZFS host by DIFFERENT addresses (dev's nodes +# are on the host's own LAN and dial it directly; prod's nodes are on the far side of pfSense and +# must be given whichever address the still-pending routing decision produces). Hardcoding either +# one authors a manifest that is silently wrong in the other environment. Declared in +# clusters/base/cluster-vars.yaml and patched by BOTH overlays; written BARE (it is a real knob, +# not a shell expansion), so StrictPostBuildSubstitutions fails the tier if an overlay forgets it. +BULK_NFS_SERVER = "${BULK_NFS_SERVER}" +# Mount options for every bulk NFS volume. These are a PLATFORM decision, not a per-service one, +# so they live here rather than on the ServiceDefinition surface: +# nfsvers=4.1 NFSv4 carries mount+lock IN the protocol, so it needs ONE port (tcp/2049) and no +# rpcbind/111, mountd or statd. That is what makes the prod path a single pfSense +# rule instead of four, and it is why v4 is chosen over v3. The server advertises +# +3 +4 +4.1 +4.2; 4.1 is the conservative floor every node kernel already speaks. +# hard A `soft` mount turns a server hiccup into a short read / EIO handed to the +# application, which for a media library and an archive is silent corruption. +# `hard` blocks instead. The cost is explicit: an NFS outage HANGS I/O rather than +# failing it, so an outage shows up as stuck pods, not as damaged files. Chosen +# deliberately -- hung is recoverable, truncated is not. +# noatime The datasets are relatime, so every read would otherwise generate a metadata +# WRITE back to the pool. Plex walking an 8 T library must not do that. +# timeo=600/retrans=2 the TCP defaults, stated explicitly so a future reader knows the retry +# budget was considered rather than inherited. +BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retrans=2"] GATEWAY_NAME = "shared" GATEWAY_NS = "gateway" KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label @@ -111,8 +135,10 @@ def apply_defaults(sd: dict) -> dict: obs.setdefault("slis", []) s["observability"] = obs - # storage entry class default is longhorn (replicated tier). - s["storage"] = [{**{"class": "longhorn"}, **v} for v in s["storage"]] + # storage entry class default is longhorn (replicated tier); readOnly default is False + # (grant-nothing-extra is expressed by SETTING it, so the default stays back-compatible and + # every pre-existing golden renders byte-identical). + s["storage"] = [{**{"class": "longhorn", "readOnly": False}, **v} for v in s["storage"]] # dependency protocol default is tcp. s["dependencies"] = [{**{"protocol": "tcp"}, **d} for d in s["dependencies"]] return s @@ -200,18 +226,125 @@ def _service_ports(s: dict) -> list: return [{"name": f"port-{p}", "port": p, "targetPort": p} for p in ports] +def _bulk_access_mode(vol: dict) -> str: + """A bulk volume's access mode follows its read/write posture. + + Every bulk volume is `...Many`: the SAME export is consumed by several namespaces at once + (plex, sonarr and radarr all sit on /mnt/Storage/plex) and by several nodes at once, which is + precisely what ReadWriteOnce cannot express. WHICH `Many` it is says whether this consumer + writes -- so `kubectl get pv` reports the intent, not merely the topology. + """ + return "ReadOnlyMany" if vol.get("readOnly") else "ReadWriteMany" + + +def r_bulk_pvs(s: dict) -> list: + """One STATIC PersistentVolume per `class: bulk` storage entry (docs/04 bulk tier). + + WHY A PV AT ALL, RATHER THAN AN INLINE `nfs:` POD VOLUME + ------------------------------------------------------- + An inline volume is fewer objects and it would work. It is rejected for four reasons, in + descending order of what they would cost to discover later: + + 1. RECLAIM POLICY. `persistentVolumeReclaimPolicy: Retain` is a declared, reviewable + property of a PV. There is no equivalent for an inline volume -- and the data behind + these mounts is 8.33 TiB of media and a 267 GiB archive that ALSO has live non-k3s + consumers. A statement in git that nothing in this cluster may reclaim it is worth an + object. + 2. READ-ONLY LIVES IN ONE PLACE. `spec.nfs.readOnly: true` becomes a real `ro` mount option + for every pod that ever binds this volume. The same intent expressed on an inline volume + is one Deployment edit away from being dropped silently, with no diff anywhere near the + storage. (It is still a CLIENT-side guarantee -- the export cannot distinguish plex from + sonarr, since both may run on any node -- but it is one that lives in one reviewable + place.) + 3. VISIBILITY. `kubectl get pv,pvc` is where an operator looks for "what external storage is + attached, and who holds it". An inline volume is invisible there, and invisible to + anything that reasons over PVCs. + 4. IT IS THE CONTRACT THE REPO ALREADY HAS. Every other storage entry is a PVC; the + renderer, the Kyverno ServiceDefinition policy and the backup wiring all key on that. An + inline volume would be a second, parallel storage model inside the same renderer. + + CROSS-NAMESPACE. A PV binds 1:1 to a PVC, so "plex, sonarr and radarr share one export" is NOT + one shared PV -- it is three PVs, one per namespace, each naming the same server+path. That is + the correct model rather than a workaround: the sharing happens on the NFS server, which is + built for concurrent clients, and each namespace keeps its own independently reviewable access + mode (plex ReadOnlyMany, the arrs ReadWriteMany). One shared PV would have forced a single + posture on all three. + + BINDING IS PINNED FROM BOTH ENDS -- `claimRef` here and `volumeName` on the PVC. Either alone + leaves a race in which some other PVC of matching size and mode grabs the volume first; + together the binding is deterministic. `storageClassName: ""` (explicitly empty, NOT the `bulk` + StorageClass) is what tells the binder this volume is statically provisioned: it suppresses the + default-StorageClass annotation and guarantees no provisioner is ever asked to create anything. + It also sidesteps a trap -- a StorageClass's `provisioner` is IMMUTABLE, so re-pointing the + existing `bulk` class (whose stub names an NFS CSI driver that is not installed) at this data + would make Flux fail forever on that one object. + """ + out = [] + for vol in s["storage"]: + if vol.get("class") != "bulk": + continue + name = f"{s['name']}-{vol['name']}" + nfs = {"server": BULK_NFS_SERVER, "path": vol["export"]} + if vol.get("readOnly"): + nfs["readOnly"] = True + spec = { + "capacity": {"storage": vol["size"]}, + "accessModes": [_bulk_access_mode(vol)], + # Retain: deleting the PVC must NEVER be able to reach the data. There is nothing to + # "reclaim" here in any case -- the volume was not provisioned by the cluster. + "persistentVolumeReclaimPolicy": "Retain", + "storageClassName": "", + "mountOptions": list(BULK_NFS_MOUNT_OPTIONS), + "nfs": nfs, + "claimRef": {"namespace": s["name"], "name": name}, + } + out.append( + { + "apiVersion": "v1", + "kind": "PersistentVolume", + "metadata": { + "name": name, + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": spec, + } + ) + return out + + def r_pvcs(s: dict) -> list: """One PVC per storage entry (R-STATE-01/02/04). - longhorn -> 3-replica replicated tier; bulk -> external media tier (not 3x replicated). + longhorn -> 3-replica replicated tier, dynamically provisioned, ReadWriteOnce. + bulk -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName. """ out = [] for vol in s["storage"]: cls = vol.get("class", "longhorn") - storage_class = "longhorn" if cls == "longhorn" else "bulk" + if cls == "bulk": + # Mirror image of the PV: the same explicitly-empty class (so nothing dynamically + # provisions), the same access mode, and volumeName completing the two-sided binding. + out.append( + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": f"{s['name']}-{vol['name']}", + "namespace": s["name"], + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": { + "accessModes": [_bulk_access_mode(vol)], + "storageClassName": "", + "volumeName": f"{s['name']}-{vol['name']}", + "resources": {"requests": {"storage": vol["size"]}}, + }, + } + ) + continue spec = { "accessModes": ["ReadWriteOnce"], - "storageClassName": storage_class, + "storageClassName": "longhorn", } if vol.get("size"): spec["resources"] = {"requests": {"storage": vol["size"]}} @@ -257,7 +390,17 @@ def r_deployment(s: dict) -> dict: # it verbatim for apps that need a fixed on-disk layout (e.g. Gitea keeps all state # directly under /data). Omitting it preserves the /data/ default (back-compat). mount_path = vol.get("mountPath", f"/data/{vol['name']}") - volume_mounts.append({"name": vol["name"], "mountPath": mount_path}) + mount = {"name": vol["name"], "mountPath": mount_path} + # subPath mounts a subtree of the volume instead of its root. Emitted only when declared, + # so every pre-existing golden stays byte-identical. + if vol.get("subPath"): + mount["subPath"] = vol["subPath"] + # readOnly on the mount as well as on the PV's nfs source. Belt and braces on purpose: the + # PV field is what makes the kernel mount `ro`; this one is what a reviewer reading the + # Deployment sees. An edit that drops one still leaves the other. + if vol.get("readOnly"): + mount["readOnly"] = True + volume_mounts.append(mount) volumes.append( { "name": vol["name"], @@ -1095,6 +1238,14 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None) if s["kind"] == "container": # Container PVCs. (For a vm, the CDI DataVolume owns/creates the disk PVC, so we # never render a plain PVC for the VM root disk — it would collide with CDI.) + # The static bulk PV is rendered BEFORE the PVC that names it. Document order inside one + # kustomize build is not an apply-ordering guarantee, but a PVC whose PV does not exist + # yet merely stays Pending and binds on the next reconcile, so this is a readability + # convention rather than a correctness requirement. + for pv in r_bulk_pvs(s): + docs.append( + (f"PersistentVolume {pv['metadata']['name']} - static external NFS (bulk tier)", pv) + ) for pvc in r_pvcs(s): docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc)) docs.append(("Deployment (R-PLAT-02)", r_deployment(s))) diff --git a/schemas/renderer-contract.md b/schemas/renderer-contract.md index 8e647a6..01d53a7 100644 --- a/schemas/renderer-contract.md +++ b/schemas/renderer-contract.md @@ -72,11 +72,48 @@ platform must render, and cites the requirement each mapping satisfies. Changing - **Requirements:** **R-NET-01** (default-deny baseline), **R-NET-02** (explicit allow / deny undeclared). Auditable via Hubble (R-SEC-03, R-OBS-04). -### `storage[]` (`name` / `size` / `class`) — PVCs +### `storage[]` (`name` / `size` / `class` / `mountPath` / `export` / `subPath` / `readOnly`) — PVs + PVCs - `class: longhorn` → a **PVC** on the 3-replica Longhorn StorageClass sized by `size` (replicated - config/DB/app state). For a `vm`, the disk is a Longhorn volume via CDI. -- `class: bulk` → binding to the external **bulk** media tier (NFS/SMB indirection, sized outside the - cluster; `size` optional). Not 3× replicated. + config/DB/app state), `ReadWriteOnce`, dynamically provisioned. For a `vm`, the disk is a Longhorn + volume via CDI. +- `class: bulk` → a **static, pre-bound external NFS PersistentVolume _and_ its PVC**. The entry's + `export` is the absolute path on the bulk server; the SERVER is never named in a ServiceDefinition + — it is the `${BULK_NFS_SERVER}` cluster-vars knob, so one authored definition promotes across + environments that reach the same storage by different addresses (**R-ENV-02**). Not 3× replicated: + durability is the storage host's ZFS pool, not Longhorn. + - `storageClassName: ""` on **both** PV and PVC (explicitly empty ⇒ statically provisioned; no + provisioner is ever asked to create anything, and the default-StorageClass annotation is + suppressed). Deliberately NOT the `bulk` StorageClass: a StorageClass `provisioner` is + **immutable**, so re-pointing that stub would make Flux fail permanently on it. + - Binding is pinned from both ends — `claimRef` on the PV, `volumeName` on the PVC — so no other + claim of matching size/mode can win a race for the volume. + - `persistentVolumeReclaimPolicy: Retain`, always. Bulk data pre-exists the cluster, is far larger + than the cluster, and may have consumers outside it; nothing here may reclaim it. + - Access mode follows the read/write posture and is always `…Many`, because one export is + consumed by several namespaces and several nodes at once: `ReadOnlyMany` when `readOnly`, + `ReadWriteMany` otherwise. `ReadWriteOnce` cannot express either. + - `mountOptions` are a **platform** decision held in the renderer, not a per-service field: + `nfsvers=4.1,hard,noatime,timeo=600,retrans=2`. NFSv4 carries mount and lock in-protocol, so + the only port that must be reachable is **tcp/2049** — no rpcbind/111, mountd or statd, which is + what keeps the firewall surface to a single rule. `hard` is chosen over `soft` because a soft + mount converts a server hiccup into a short read handed to the application. + - `size` on a bulk entry is the PV's **advertised capacity and nothing more** — no component + enforces it (the ZFS pool is the real limit), so it should state the dataset's actual size so + `kubectl get pv` reports something true. + - **fsGroup does not apply.** The in-tree NFS volume plugin performs no ownership management, so + `podSecurityContext.fsGroup` is a **no-op** on a bulk volume. A bulk mount that must be written + therefore requires the container's own uid/gid to be one the export's ownership permits (e.g. + Plex's `PLEX_UID`/`PLEX_GID`, LinuxServer images' `PUID`/`PGID`) — or a server-side squash. A + mount that succeeds but cannot write is the characteristic failure of this tier. +- `mountPath` (any class) overrides the `/data/` default verbatim. +- `subPath` (any class) mounts a subtree instead of the volume root. **Two `subPath` mounts of one + export are two separate mounts**, so `link()` between them fails `EXDEV`; a workload that needs + hardlinks or atomic moves between two directories (Sonarr/Radarr importing from a download path + into a library) must mount their common ancestor **once** instead. +- `readOnly` (any class) marks the volumeMount read-only, and on a bulk entry also sets the PV's + `nfs.readOnly` so the kernel mount really is `ro`. Note this is a **client-side** guarantee: one + export serving both a reader and a writer that may land on any node cannot distinguish them + server-side. - State separation rule: anything that must survive a rebuild lives on a named volume here, never on ephemeral fs/hostPath — so the workload can be destroyed and re-applied from git with data intact. - **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication), diff --git a/schemas/servicedefinition.schema.json b/schemas/servicedefinition.schema.json index 449e980..20ce03b 100644 --- a/schemas/servicedefinition.schema.json +++ b/schemas/servicedefinition.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json", "title": "ServiceDefinition", - "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.3.0 (adds optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", + "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", "type": "object", "additionalProperties": false, "required": ["name", "source"], @@ -153,7 +153,7 @@ }, "storage": { "type": "array", - "description": "Persistent volumes. If it must survive a rebuild it lives here, never on ephemeral/hostPath (docs/04; R-STATE-01/02/04).", + "description": "Persistent volumes. If it must survive a rebuild it lives here, never on ephemeral/hostPath (docs/04; R-STATE-01/02/04). A `longhorn` entry renders a dynamically provisioned PVC; a `bulk` entry renders a STATIC, pre-bound NFS PersistentVolume + its PVC over data that already exists outside the cluster.", "default": [], "items": { "type": "object", @@ -167,7 +167,7 @@ }, "size": { "type": "string", - "description": "PVC request in Kubernetes quantity form (e.g. 5Gi). Required for the longhorn tier; may be omitted for the bulk tier (external mount, sized outside the cluster - docs/04).", + "description": "PVC request in Kubernetes quantity form (e.g. 5Gi). Required for the longhorn tier. For the bulk tier it is the STATIC PV's advertised capacity: NOTHING enforces it (the ZFS pool is the real limit), so state the dataset's current size so `kubectl get pv` reports something true. Omitted => the bulk PV/PVC carry no capacity, which the API rejects, so bulk in practice sets it.", "pattern": "^[1-9][0-9]*(Mi|Gi|Ti)$" }, "mountPath": { @@ -177,11 +177,33 @@ }, "class": { "type": "string", - "description": "longhorn = 3-replica block storage for app/config/DB state (R-STATE-02); bulk = large external media tier, NOT 3x replicated (R-STATE-04).", + "description": "longhorn = 3-replica block storage for app/config/DB state (R-STATE-02); bulk = large external media tier, NOT 3x replicated, backed by a STATIC NFS PersistentVolume the renderer emits (requires `export`).", "enum": ["longhorn", "bulk"], "default": "longhorn" + }, + "export": { + "type": "string", + "description": "bulk ONLY: the absolute NFS export path on the bulk server (e.g. /mnt/Storage/plex). The SERVER is NOT named here - it is the ${BULK_NFS_SERVER} cluster-vars knob, so the same ServiceDefinition promotes across environments (R-ENV-02). Required whenever class is bulk; forbidden otherwise.", + "pattern": "^/.*[^/]$" + }, + "subPath": { + "type": "string", + "description": "Optional relative path WITHIN the volume to mount instead of its root. NOTE for hardlink/atomic-move workloads (Sonarr/Radarr): two subPath mounts of the same export are two separate mounts, so link() across them fails EXDEV. Mount the common ancestor once instead.", + "pattern": "^[^/].*[^/]$" + }, + "readOnly": { + "type": "boolean", + "description": "Mount this volume read-only. For a bulk volume this also sets the PersistentVolume's nfs.readOnly (the mount really is `ro`) and makes its access mode ReadOnlyMany instead of ReadWriteMany.", + "default": false } - } + }, + "allOf": [ + { + "if": { "properties": { "class": { "const": "bulk" } }, "required": ["class"] }, + "then": { "required": ["export"] }, + "else": { "not": { "required": ["export"] } } + } + ] } }, "dependencies": {