diff --git a/apps/wave-0/pihole.yaml b/apps/wave-0/pihole.yaml index d609790..adeaa8c 100644 --- a/apps/wave-0/pihole.yaml +++ b/apps/wave-0/pihole.yaml @@ -35,34 +35,18 @@ spec: - protocol: TCP port: 53 --- -# PVC pihole-config (R-STATE) -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pihole-config - namespace: pihole - labels: - app.kubernetes.io/name: pihole -spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn - resources: - requests: - storage: 2Gi ---- -# Deployment (R-PLAT-02) +# StatefulSet (R-PLAT-02) apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: pihole namespace: pihole labels: app.kubernetes.io/name: pihole spec: - replicas: 1 - strategy: - type: Recreate + replicas: 2 + serviceName: pihole + podManagementPolicy: Parallel selector: matchLabels: app.kubernetes.io/name: pihole @@ -97,7 +81,7 @@ spec: key: webui-password volumeMounts: - name: config - mountPath: /data/config + mountPath: /etc/pihole lifecycle: postStart: exec: @@ -175,10 +159,39 @@ spec: port: http initialDelaySeconds: 5 periodSeconds: 15 - volumes: - - name: config - persistentVolumeClaim: - claimName: pihole-config + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: pihole + topologyKey: kubernetes.io/hostname + volumeClaimTemplates: + - metadata: + name: config + labels: + app.kubernetes.io/name: pihole + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 2Gi +--- +# PodDisruptionBudget — keep one replica through a drain (availability.replicas) +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: pihole + namespace: pihole + labels: + app.kubernetes.io/name: pihole +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: pihole --- # Service apiVersion: v1 @@ -281,11 +294,11 @@ spec: - name: pihole.slis rules: - alert: pihole-down - expr: kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"} == 0 or absent(kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}) + expr: kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"} == 0 or absent(kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}) for: 5m labels: severity: critical service: pihole annotations: summary: pihole is down - description: 'pihole: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' + description: 'pihole: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready).' diff --git a/apps/wave-0/tests/app-slis.test.yaml b/apps/wave-0/tests/app-slis.test.yaml index c95052d..e062cca 100644 --- a/apps/wave-0/tests/app-slis.test.yaml +++ b/apps/wave-0/tests/app-slis.test.yaml @@ -1,6 +1,8 @@ # promtool rule-unit tests for the wave-0 per-app `-down` SLI rules. # Manifests under test: every PrometheusRule in apps/wave-0/ — gitea, jackett, pihole, plex, -# qbittorrent (container/Deployment) and vm-demo (KubeVirt VMI). The harness discovers this file by +# qbittorrent (container/Deployment), pihole (container/StatefulSet — `availability.replicas: 2`, so +# its SLI reads kube_statefulset_status_replicas_ready; see schemas/renderer-contract.md +# `### availability`) and vm-demo (KubeVirt VMI). The harness discovers this file by # the SUITE convention (`/tests/*.test.yaml`) and flattens those RENDERED manifests, so these # tests run against the exact expressions Flux applies, never a copy. # @@ -41,7 +43,7 @@ tests: values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' values: '1+0x20' - - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}' + - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' values: '1+0x20' @@ -81,7 +83,7 @@ tests: values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' values: '1+0x20' - - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}' + - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}' values: '1+0x20' - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' values: '0+0x20' @@ -132,7 +134,7 @@ tests: values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' values: '0+0x20' - - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}' + - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}' values: '0+0x20' - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' values: '1+0x20' @@ -170,10 +172,10 @@ tests: severity: critical service: pihole namespace: pihole - deployment: pihole + statefulset: pihole exp_annotations: summary: "pihole is down" - description: "pihole: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + description: "pihole: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)." - eval_time: 10m alertname: qbittorrent-down exp_alerts: diff --git a/catalog/pihole.yaml b/catalog/pihole.yaml index 6052687..8ebfdc8 100644 --- a/catalog/pihole.yaml +++ b/catalog/pihole.yaml @@ -12,6 +12,42 @@ storage: - name: config size: 2Gi class: longhorn # replicated config state; DR-critical (docs/04 line 118; R-STATE-02/03) + # THE PVC WAS MOUNTED SOMEWHERE PIHOLE NEVER WRITES, AND SO HELD NOTHING AT ALL. Without an + # explicit mountPath the renderer defaults to /data/, i.e. /data/config — but PiHole keeps + # every piece of its state in /etc/pihole. MEASURED on live prod 2026-10-04: `/etc/pihole` (7.1 M + # — gravity.db, pihole.toml, the query DB) was on the **overlay** filesystem, i.e. the + # container's EPHEMERAL layer, while `/dev/longhorn/pvc-849edb86...` sat mounted at + # /data/config holding `lost+found` and 24 K of nothing. Three consequences, all live until now: + # * every restart silently discarded PiHole's whole state, re-downloading 72 k gravity domains + # and regenerating the admin password hash; + # * `backup.policy: daily-offcluster` below has been faithfully backing up an EMPTY volume, + # which is worse than no backup because it reports success; + # * it is also why phase 1's password/adlist declaration mattered more than it first appeared — + # that state was not merely undeclared, it was ephemeral. + # This is the same class of bug catalog/plex.yaml's `config` entry documents ("Without this the + # PVC sat unused and Plex wrote to the container's EPHEMERAL layer"). + # + # /etc/pihole IS THE IMAGE'S INTENDED MOUNT POINT, and an EMPTY volume there self-initialises. + # Verified in the running image, not assumed: + # * `/usr/bin/start.sh` (PID 1) calls `migrate_gravity`, whose `migrate_to_database()` creates + # the gravity database when `[ ! -e "$gravityDBfile" ]` — the same branch the postStart hook + # below relies on, and the branch that produced the current ephemeral copy. + # * FTL writes `pihole.toml` itself from the `FTLCONF_*` env above (that is what the + # `### CHANGED (env)` markers in it are). + # * NO `podSecurityContext.fsGroup` is needed and adding one would be inert: `start.sh` line 5 + # is `chown -R pihole:pihole /etc/pihole/ /var/log/pihole/` and PID 1 runs as **uid 0**, so + # the image fixes ownership of a fresh root-owned volume before anything else starts. Same + # arrangement as plex's image managing its own /config ownership. + # * the image's own default for `files.macvendor` was moved OUT of this directory (to + # `/macvendor.db`), which is what a maintainer does to a path they expect to be a volume. + # + # WHY IT IS FIXED IN THIS BRANCH rather than left for its own: `availability.replicas: 2` renders + # per-replica volumeClaimTemplates, and with the wrong mountPath those would be two 2 Gi Longhorn + # volumes (times three Longhorn replicas each) holding nothing but `lost+found` while all state + # stayed ephemeral — paying the cost of persistence and getting none of it. It also REMOVES this + # change's main operational risk: with gravity.db persisted, a pod restart is cheap, instead of + # every restart re-downloading 72 k domains inside a wave that has a 5 m timeout. + mountPath: /etc/pihole dependencies: [] # default-deny: reaches only DNS; it PROVIDES DNS (docs/03; R-NET-01/02) # ------------------------------------------------------------------------------------------------ # LAN SPLIT-HORIZON DNS — the half of R-DNS-03 that platform/coredns/coredns-custom.yaml defers to. @@ -114,6 +150,39 @@ env: secretKeyRef: name: pihole-bootstrap key: webui-password +availability: + # TWO REPLICAS, ON DIFFERENT NODES. LAN DNS is a critical network service and it was ONE pod on + # ONE node. Measured 2026-10-04, that cost more than "until it reschedules": + # * the node running the pod dies -> ~40s to mark it NotReady, then `tolerationSeconds: 300` + # before the pod is even evicted, then reschedule, then Longhorn must force-detach the RWO + # volume from a dead node and reattach it. Roughly 6-10 MINUTES with no LAN resolution. + # * the node holding the `pihole-dns-*` L2 lease dies -> Cilium re-elects an announcer, tens of + # seconds. (These were DIFFERENT nodes: lease on k3s-prod-2, pod on k3s-prod-1. Two of three + # nodes were single points of failure for LAN DNS, with wildly different blast radii.) + # The Services already tolerate this: `externalTrafficPolicy: Cluster` on the DNS LoadBalancers + # (apps/wave-0/pihole-dns.yaml) means the L2 lease holder forwards to ANY ready backend, so a + # second replica is picked up with no change there. + # + # WHAT THIS DOES NOT FIX, and it is the bigger risk: LAN DNS still depends on the CLUSTER. etcd + # quorum loss, a Cilium regression, a Longhorn problem or a bad reconcile takes DNS down + # everywhere regardless of replica count. The only thing that removes that dependency is a + # resolver outside the cluster carrying the same `address=` lines. Considered and not chosen + # (operator decision, 2026-10-04); see decisions/2026-10-04-pihole-multi-replica.md. + # + # THE PREREQUISITE WAS DONE FIRST, and this field would have been actively harmful without it. A + # replica gets a FRESH volume, and until 2026-10-04 PiHole's admin password and adlists lived only + # on the PVC — so replica 1 would have come up SERVING DNS AND BLOCKING NOTHING, with a different + # password, and answers would have varied by which pod replied. Both are now declared (the + # password as a SOPS Secret -> FTLCONF_webserver_api_password; the adlists reconciled into + # gravity.db by the postStart hook below), which is what makes a fresh replica identical to its + # sibling. See decisions/2026-10-04-pihole-declare-state.md. + # + # WHY 2 AND NOT 3. Two tolerates the loss of any single node, which is the stated goal, and leaves + # a spare node for the evicted replica to land on under the HARD anti-affinity the renderer emits. + # Three would pin one replica per node with no spare, so a single node failure would leave a + # replica permanently Pending and — `apps-wave-0` being `wait: true` — stall the app tier for as + # long as the node was down. Two is the count that degrades gracefully here. + replicas: 2 ingressRoutes: - host: pihole.${BASE_DOMAIN} port: 80 # admin web UI, protected at the Gateway by forward-auth (docs/09) diff --git a/catalog/render.py b/catalog/render.py index 0b27f8b..e8b668d 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -272,7 +272,15 @@ def apply_defaults(sd: dict) -> dict: # object that actually runs on the cluster is not the one this SD renders (keycloak: the # Keycloak operator runs a StatefulSet, so a Deployment-shaped rule would `absent()` forever). lw = dict(obs.get("livenessWorkload") or {}) - lw.setdefault("kind", "VirtualMachineInstance" if s["kind"] == "vm" else "Deployment") + # The default must track what is ACTUALLY rendered, or the `up` SLI silently dies: a + # Deployment-shaped rule against a StatefulSet `absent()`s forever, which is an alert that can + # never fire — the exact failure mode the keycloak override was added for. Deriving it from + # `availability.replicas` means bumping the replica count cannot leave a stale rule behind. + lw.setdefault( + "kind", + "VirtualMachineInstance" if s["kind"] == "vm" + else ("StatefulSet" if _is_statefulset(s) else "Deployment"), + ) lw.setdefault("name", s["name"]) obs["livenessWorkload"] = lw s["observability"] = obs @@ -475,6 +483,37 @@ def r_bulk_pvs(s: dict) -> list: return out +def _replicas(s: dict) -> int: + """Declared replica count (schema 1.10.0 `availability.replicas`). Default 1.""" + return int((s.get("availability") or {}).get("replicas", 1) or 1) + + +def _is_statefulset(s: dict) -> bool: + """True when this container workload must render a StatefulSet rather than a Deployment. + + THE DECIDING CONSTRAINT IS THE VOLUME, not a preference. A longhorn storage entry renders + ReadWriteOnce (r_pvcs), and one RWO PVC cannot be attached by pods on two different nodes -- + which is exactly what `availability.replicas > 1` plus the node anti-affinity below asks for. + A 2-replica Deployment over a shared RWO claim does not degrade, it simply never schedules the + second pod. A StatefulSet's volumeClaimTemplates give each replica its OWN claim, which is the + only shape that satisfies both at once. + + A replicas-1 service keeps rendering a Deployment so every pre-existing golden is byte-identical. + """ + return s["kind"] == "container" and _replicas(s) > 1 + + +def _sts_claim_volumes(s: dict) -> list: + """Storage entries that become per-replica volumeClaimTemplates on a StatefulSet. + + Only the dynamically-provisioned (longhorn) entries. A `bulk` entry is a STATIC, pre-bound NFS + PV over data that already exists outside the cluster and is shared read-only by every consumer + (r_bulk_pvs) -- per-replica copies of it are neither possible nor wanted, so it stays a + standalone PVC and an ordinary pod volume. + """ + return [v for v in s["storage"] if v.get("class", "longhorn") != "bulk"] + + def r_pvcs(s: dict) -> list: """One PVC per storage entry (R-STATE-01/02/04). @@ -482,8 +521,15 @@ def r_pvcs(s: dict) -> list: bulk -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName. """ out = [] + # On a StatefulSet the longhorn claims are NOT standalone objects — they are + # volumeClaimTemplates, so each replica gets its own (see _is_statefulset). Emitting both would + # leave an orphan PVC that nothing mounts and that `prune` would later delete out from under + # nobody. `bulk` is unaffected and still rendered here. + sts = _is_statefulset(s) for vol in s["storage"]: cls = vol.get("class", "longhorn") + if sts and cls != "bulk": + continue if cls == "bulk": # Mirror image of the PV: the same explicitly-empty class (so nothing dynamically # provisions), the same access mode, and volumeName completing the two-sided binding. @@ -564,6 +610,12 @@ def r_deployment(s: dict) -> dict: if vol.get("readOnly"): mount["readOnly"] = True volume_mounts.append(mount) + # On a StatefulSet a longhorn volume is supplied by its volumeClaimTemplate, whose + # metadata.name IS the pod volume name. Adding it to pod.volumes as well is a conflict + # (duplicate volume name), so it is skipped here and emitted below. The MOUNT is unchanged — + # the container cannot tell the difference, which is what keeps this transparent to the app. + if _is_statefulset(s) and vol.get("class", "longhorn") != "bulk": + continue volumes.append( { "name": vol["name"], @@ -661,20 +713,80 @@ def r_deployment(s: dict) -> dict: if ann: tmpl_meta["annotations"] = ann - spec = {"replicas": 1} - # An app with persistent storage mounts a ReadWriteOnce Longhorn PVC. The default - # RollingUpdate strategy tries to start the NEW pod before terminating the old one, but - # the old pod still holds the RWO volume -> the new pod is stuck ContainerCreating and - # the rollout deadlocks (single-replica app can never hand the volume over). Recreate - # terminates the old pod first, freeing the PVC for the new one (R-STATE-01/02). - if s["storage"]: + sts = _is_statefulset(s) + + # NODE ANTI-AFFINITY, and it is REQUIRED rather than preferred. `availability.replicas > 1` + # exists to survive the loss of one node; `preferred` would let both replicas land on the same + # node and quietly deliver none of that, with every object reporting healthy. The cost is real + # and is accepted: with N replicas the workload needs N schedulable nodes, so on a 3-node + # cluster losing TWO nodes leaves a replica Pending and — because `apps-wave-0` is `wait: true` + # with `apps-wave-1` dependsOn it — STALLS THE APP TIER. That coincides with having already lost + # etcd quorum on a 3-node control plane, i.e. the cluster is not reconciling anything at that + # point anyway, so the stall is a symptom rather than a new failure. + if sts: + pod_spec["affinity"] = { + "podAntiAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchLabels": {"app.kubernetes.io/name": s["name"]} + }, + "topologyKey": "kubernetes.io/hostname", + } + ] + } + } + + spec = {"replicas": _replicas(s)} + if sts: + # serviceName is a required StatefulSet field. It points at the service rendered by + # r_service; that Service is a normal ClusterIP rather than headless, so per-pod DNS + # (.) does NOT resolve — and nothing here needs it. These replicas are + # interchangeable resolvers behind one VIP, not peers that address each other. + spec["serviceName"] = s["name"] + # Parallel, not the OrderedReady default. OrderedReady starts replica N+1 only after N is + # Ready, so a cold start costs the sum of every replica's first-boot work instead of the + # max — and `apps-wave-0` has `timeout: 5m`. There is no ordering requirement between + # replicas to protect (no leader, no migration, no shared volume), so serialising buys + # nothing and risks the wave timing out on a rebuild. + spec["podManagementPolicy"] = "Parallel" + elif s["storage"]: + # An app with persistent storage mounts a ReadWriteOnce Longhorn PVC. The default + # RollingUpdate strategy tries to start the NEW pod before terminating the old one, but + # the old pod still holds the RWO volume -> the new pod is stuck ContainerCreating and + # the rollout deadlocks (single-replica app can never hand the volume over). Recreate + # terminates the old pod first, freeing the PVC for the new one (R-STATE-01/02). + # + # A StatefulSet needs none of this and cannot express it: each replica owns its own claim, + # so a rolling update hands nothing over, and `strategy` is not a StatefulSet field at all. spec["strategy"] = {"type": "Recreate"} spec["selector"] = {"matchLabels": {"app.kubernetes.io/name": s["name"]}} spec["template"] = {"metadata": tmpl_meta, "spec": pod_spec} + if sts: + # Per-replica claims. metadata.name MUST equal the pod volume name the container mounts + # (see the volumes loop above); the resulting PVCs are --, e.g. + # config-pihole-0. Same tier and access mode a standalone longhorn PVC gets from r_pvcs — + # this changes WHO owns the claim, never what it is. + tmpls = [] + for vol in _sts_claim_volumes(s): + vspec = {"accessModes": ["ReadWriteOnce"], "storageClassName": "longhorn"} + if vol.get("size"): + vspec["resources"] = {"requests": {"storage": vol["size"]}} + tmpls.append( + { + "metadata": { + "name": vol["name"], + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": vspec, + } + ) + if tmpls: + spec["volumeClaimTemplates"] = tmpls return { "apiVersion": "apps/v1", - "kind": "Deployment", + "kind": "StatefulSet" if sts else "Deployment", "metadata": { "name": s["name"], "namespace": s["name"], @@ -684,6 +796,35 @@ def r_deployment(s: dict) -> dict: } +def r_pdb(s: dict): + """PodDisruptionBudget for a multi-replica workload (availability.replicas > 1). + + minAvailable 1, not `maxUnavailable: 1`. The budget's job here is to stop a VOLUNTARY + disruption — a node drain for maintenance, a cluster upgrade — from taking the last replica + with it; `minAvailable: 1` says that whatever else happens, one must keep serving. It does + NOT protect against involuntary disruption (a node dying), which is what the anti-affinity + and the replica count are for. + + Rendered only when replicas > 1: a PDB over a single-replica workload is actively harmful, as + `minAvailable: 1` would then block every drain of its node forever. + """ + if not _is_statefulset(s): + return None + return { + "apiVersion": "policy/v1", + "kind": "PodDisruptionBudget", + "metadata": { + "name": s["name"], + "namespace": s["name"], + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": { + "minAvailable": 1, + "selector": {"matchLabels": {"app.kubernetes.io/name": s["name"]}}, + }, + } + + def r_service(s: dict): ports = _service_ports(s) # A declared metrics endpoint becomes a NAMED Service port, because that name is the only @@ -1817,7 +1958,13 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None) ) for pvc in r_pvcs(s): docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc)) - docs.append(("Deployment (R-PLAT-02)", r_deployment(s))) + wl = r_deployment(s) + docs.append((f"{wl['kind']} (R-PLAT-02)", wl)) + pdb = r_pdb(s) + if pdb: + docs.append( + ("PodDisruptionBudget — keep one replica through a drain (availability.replicas)", pdb) + ) svc = r_service(s) if svc: docs.append(("Service", svc)) @@ -2266,6 +2413,132 @@ def _selftest() -> int: check("Namespace" in kinds, "container: missing Namespace") check("NetworkPolicy" in kinds, "container: missing default-deny NetworkPolicy") check("Deployment" in kinds, "container: missing Deployment") + check( + "PodDisruptionBudget" not in kinds, + "replicas-1 container must render NO PodDisruptionBudget (minAvailable 1 over a single " + "replica blocks every drain of its node forever)", + ) + + # 1b) availability.replicas (schema 1.10.0). The DEFAULT path must be untouched, and the >1 + # path must change workload kind, claim ownership, spreading, budget AND the SLI metric + # together — each of these has a silent-failure mode if it is the one that gets forgotten. + base_av = { + "name": "av", + "source": {"image": "nginx:1"}, + "storage": [{"name": "config", "size": "1Gi"}], + "observability": {"metrics": True, "slis": ["up"]}, + } + one = render({**base_av, "availability": {"replicas": 1}}) + one_kinds = [d[1]["kind"] for d in one] + check( + "Deployment" in one_kinds and "StatefulSet" not in one_kinds, + "availability.replicas=1 must still render a Deployment (back-compat)", + ) + check( + "PersistentVolumeClaim" in one_kinds, + "availability.replicas=1 must still render a standalone PVC", + ) + check( + "PodDisruptionBudget" not in one_kinds, + "availability.replicas=1 must render no PodDisruptionBudget", + ) + + many = render({**base_av, "availability": {"replicas": 2}}) + many_kinds = [d[1]["kind"] for d in many] + check( + "StatefulSet" in many_kinds and "Deployment" not in many_kinds, + "availability.replicas>1 must render a StatefulSet, not a Deployment (a shared RWO claim " + "cannot be attached on two nodes, so a multi-replica Deployment never schedules)", + ) + check( + "PersistentVolumeClaim" not in many_kinds, + "availability.replicas>1 must NOT also render a standalone longhorn PVC — it would be an " + "orphan that nothing mounts", + ) + sts = next(d[1] for d in many if d[1]["kind"] == "StatefulSet") + check(sts["spec"].get("replicas") == 2, "StatefulSet must carry the declared replica count") + check( + "strategy" not in sts["spec"], + "StatefulSet must not carry `strategy` (not a StatefulSet field; Recreate is a " + "Deployment-only workaround for handing over one RWO volume)", + ) + check( + sts["spec"].get("podManagementPolicy") == "Parallel", + "StatefulSet must set podManagementPolicy=Parallel — OrderedReady serialises first-boot " + "work across replicas and apps-wave-0 has a 5m timeout", + ) + check(sts["spec"].get("serviceName") == "av", "StatefulSet must set serviceName") + tmpls = sts["spec"].get("volumeClaimTemplates") or [] + check( + [t["metadata"]["name"] for t in tmpls] == ["config"], + "StatefulSet volumeClaimTemplate name must equal the pod volume name the container mounts, " + "or the pod gets a duplicate/unresolvable volume", + ) + check( + tmpls and tmpls[0]["spec"]["accessModes"] == ["ReadWriteOnce"] + and tmpls[0]["spec"]["storageClassName"] == "longhorn", + "StatefulSet claim template must keep the same tier/access mode as the standalone PVC", + ) + sts_vols = {v["name"] for v in sts["spec"]["template"]["spec"].get("volumes", [])} + check( + "config" not in sts_vols, + "a claim-template volume must NOT also appear in pod.volumes (duplicate volume name)", + ) + mounts = sts["spec"]["template"]["spec"]["containers"][0].get("volumeMounts") or [] + check( + any(m["name"] == "config" for m in mounts), + "the container must still mount the volume by name on a StatefulSet (transparent to the app)", + ) + aff = (sts["spec"]["template"]["spec"].get("affinity") or {}).get("podAntiAffinity") or {} + req = aff.get("requiredDuringSchedulingIgnoredDuringExecution") or [] + check( + req and req[0].get("topologyKey") == "kubernetes.io/hostname", + "availability.replicas>1 must render REQUIRED per-node anti-affinity — `preferred` can put " + "every replica on one node and deliver none of the intended node-failure tolerance", + ) + pdbs = [d[1] for d in many if d[1]["kind"] == "PodDisruptionBudget"] + check(len(pdbs) == 1, "availability.replicas>1 must render exactly one PodDisruptionBudget") + check( + pdbs and pdbs[0]["spec"].get("minAvailable") == 1, + "PodDisruptionBudget must be minAvailable=1 (keep one replica serving through a drain)", + ) + pr_many = next((d[1] for d in many if d[1]["kind"] == "PrometheusRule"), None) + check( + pr_many is not None + and "kube_statefulset_status_replicas_ready" + in pr_many["spec"]["groups"][0]["rules"][0]["expr"], + "the up SLI must follow the rendered workload kind — a Deployment-shaped rule against a " + "StatefulSet absent()s forever and can never fire", + ) + + # A bulk volume is shared external NFS: it must stay a standalone PVC even on a StatefulSet. + mixed = render( + { + **base_av, + "availability": {"replicas": 2}, + "storage": [ + {"name": "config", "size": "1Gi"}, + {"name": "media", "class": "bulk", "export": "/mnt/x", "size": "1Ti"}, + ], + } + ) + m_sts = next(d[1] for d in mixed if d[1]["kind"] == "StatefulSet") + check( + [t["metadata"]["name"] for t in m_sts["spec"]["volumeClaimTemplates"]] == ["config"], + "only longhorn volumes become claim templates; a bulk share must not be per-replica", + ) + check( + any( + d[1]["kind"] == "PersistentVolumeClaim" + and d[1]["metadata"]["name"] == "av-media" + for d in mixed + ), + "a bulk PVC must still be rendered standalone on a StatefulSet", + ) + check( + "media" in {v["name"] for v in m_sts["spec"]["template"]["spec"].get("volumes", [])}, + "a bulk volume must still appear in pod.volumes on a StatefulSet", + ) ns = next(d[1] for d in docs if d[1]["kind"] == "Namespace") check( diff --git a/decisions/2026-10-04-pihole-multi-replica.md b/decisions/2026-10-04-pihole-multi-replica.md new file mode 100644 index 0000000..bceffa6 --- /dev/null +++ b/decisions/2026-10-04-pihole-multi-replica.md @@ -0,0 +1,145 @@ +--- +date: 2026-10-04 +branch: feat/pihole-multi-replica +title: PiHole runs two replicas on different nodes — and its config PVC was mounted where PiHole never writes +--- + +## What changed + +Phase 2 of making LAN DNS survive a node failure (phase 1: +`decisions/2026-10-04-pihole-declare-state.md`). PiHole was one pod on one node, which for a +whole-house resolver cost more than "until it reschedules" — measured 2026-10-04: + +| Failure | Recovery | +|---|---| +| the node running the pod dies | ~40s to mark NotReady + `tolerationSeconds: 300` before eviction + reschedule + a Longhorn RWO force-detach → **roughly 6-10 minutes with no LAN DNS** | +| the node holding the `pihole-dns-*` L2 lease dies | Cilium re-elects an announcer, tens of seconds | + +Those were **different** nodes (lease on k3s-prod-2, pod on k3s-prod-1), so two of three nodes were +single points of failure for LAN DNS with wildly different blast radii. + +**A new schema field, because this is intent and not a primitive.** `availability.replicas` +(schema **1.10.0**, mechanism in `schemas/renderer-contract.md` `### availability`). `1` or absent +renders a Deployment exactly as before — every pre-existing golden is byte-identical, verified. `>1` +renders, together: + +- a **StatefulSet** instead of a Deployment. This is forced, not preferred: a longhorn PVC is + **ReadWriteOnce**, and one RWO claim cannot attach on two nodes, so a 2-replica Deployment does not + degrade — the second pod never schedules. +- longhorn storage as **per-replica `volumeClaimTemplates`** (`config-pihole-0`, `config-pihole-1`); + the standalone PVC is no longer emitted, since it would be an orphan nothing mounts. A `bulk` + entry stays a standalone PVC — it is a shared static NFS export, not per-replica state. +- **required** `podAntiAffinity` on `kubernetes.io/hostname`. `preferred` would let both replicas + land on one node and deliver none of the intended tolerance while reporting healthy. +- `podManagementPolicy: Parallel`, so a cold start costs the max of the replicas' first-boot work + rather than the sum — `apps-wave-0` has `timeout: 5m`. +- a **PodDisruptionBudget**, `minAvailable: 1`, so a node drain cannot take the last replica. +- `observability.livenessWorkload.kind` defaulting to `StatefulSet`, so the `up` SLI reads + `kube_statefulset_status_replicas_ready`. A Deployment-shaped rule against a StatefulSet + `absent()`s forever — an alert that can never fire. + +`catalog/pihole.yaml` sets `replicas: 2`, not 3: two tolerates any single node loss and leaves a +spare node for the evicted replica under the hard anti-affinity. Three would pin one per node with no +spare, so a single node failure would leave a replica permanently `Pending` and — `apps-wave-0` being +`wait: true` — stall the app tier for as long as the node was down. + +### The bug found on the way, and fixed here + +**PiHole's config PVC was mounted at a path PiHole never writes, so it held nothing and all of its +state was ephemeral.** With no explicit `mountPath` the renderer defaults to `/data/`; PiHole +keeps everything in `/etc/pihole`. Measured on live prod 2026-10-04: `/etc/pihole` (7.1 M — +`gravity.db`, `pihole.toml`, the query DB) was on the **overlay** filesystem, while the 2 Gi Longhorn +volume sat at `/data/config` holding `lost+found` and 24 K of nothing. Consequences, all live until +now: + +- every restart silently discarded PiHole's entire state and re-downloaded 72 k gravity domains; +- `backup.policy: daily-offcluster` has been faithfully backing up an **empty** volume, which is + worse than no backup because it reports success; +- it also means phase 1 was understated: that state was not merely undeclared, it was **ephemeral**. + +Same class of bug `catalog/plex.yaml` documents for its own `config` entry. Fixed in this branch +rather than its own because with the wrong path the new per-replica volumes would be two 2 Gi +Longhorn volumes (×3 Longhorn replicas each) holding only `lost+found` while all state stayed +ephemeral — paying for persistence and getting none. It also removes this change's main operational +risk: with `gravity.db` persisted, a restart is cheap instead of re-downloading 72 k domains inside a +wave with a 5 m timeout. + +## What I verified + +- **Back-compatibility, the thing most likely to break quietly:** after the renderer change and + before `catalog/pihole.yaml` declared anything, the canonical + `render.py --all catalog -o apps --wave-layout --no-image-automation` produced **zero diff** across + the whole `apps/` tree. `render-golden` passes. +- **The new behaviour is pinned by ~20 new `render.py --selftest` assertions** (so `ci/render-golden.sh` + hard-fails on a regression, and no `conformance/**` change was needed): `replicas: 1` still renders a + Deployment + standalone PVC + no PDB; `replicas: 2` renders a StatefulSet and **no** standalone + longhorn PVC, carries the replica count, omits `strategy`, sets `Parallel` and `serviceName`, names + the claim template exactly as the mounted volume, keeps RWO/longhorn on it, does **not** duplicate + that volume in `pod.volumes`, still mounts it on the container, renders required hostname + anti-affinity, renders exactly one `minAvailable: 1` PDB, and switches the SLI to + `kube_statefulset_status_replicas_ready`. A mixed longhorn+bulk case asserts the bulk share stays a + standalone PVC and an ordinary pod volume. +- **Rendered pihole output, read back:** `replicas: 2`, `serviceName: pihole`, + `podManagementPolicy: Parallel`, no `strategy`, claim template `config` (RWO/longhorn/2Gi), + anti-affinity `kubernetes.io/hostname`, `pod.volumes: []`, mount `config -> /etc/pihole`, + PDB `minAvailable: 1`, SLI `kube_statefulset_status_replicas_ready`. +- **The `/etc/pihole` mount is safe, evidenced from the image's own init rather than assumed:** + `/usr/bin/start.sh` is PID 1 and runs as **uid 0**; its line 5 is + `chown -R pihole:pihole /etc/pihole/ /var/log/pihole/`, so a fresh root-owned volume has its + ownership fixed before anything else starts (**no `fsGroup` needed, and one would be inert**); it + calls `migrate_gravity`, whose `migrate_to_database()` creates the gravity database when + `[ ! -e "$gravityDBfile" ]`; FTL writes `pihole.toml` itself from the `FTLCONF_*` env; and the + image's own default for `files.macvendor` was moved **out** of that directory, which is what a + maintainer does to a path they expect to be a volume. +- **The PVC really is unused:** `df` showed `/etc/pihole` on `overlay` and + `/dev/longhorn/pvc-849edb86-...` on `/data/config`; `ls -A /data/config` → `lost+found` only. +- **The gate caught a real regression and it is fixed.** The first run came back **rc 1**: + `alert-rules-unit-tested` failed because `apps/wave-0/tests/app-slis.test.yaml` fed pihole a + `kube_deployment_status_replicas_available` series and expected a `deployment: pihole` label. Three + input series, the firing expectation and the file header were updated; + `platform/monitoring/tests/run.sh apps/wave-0/tests/app-slis.test.yaml` → **SUCCESS** (6 rules). +- **Schema:** `check-jsonschema` passes over all of `catalog/*.yaml`. +- **Toolbox gate:** rc **3**, with `spec-immutability`, `decisions-log`, `lint`, `l0`, + `render-golden`, `l1`, `approval-policy` all **pass**. The rc 3 is the two environmental skips — + `gitea-settings` (no local `SETTINGS_API_TOKEN`) and `conformance` L2/L3 (`CLUSTER_AVAILABLE` + unset). **Not green; reported as rc 3.** +- **No sensitive paths touched**, so unlike the previous two branches this one needs no + `/approve-override` — confirmed against `infra/gitea/approval-policy.json`. + +## What I did NOT verify + +- **Nothing has been reconciled.** No StatefulSet exists, no second replica has ever run, the + anti-affinity has never been evaluated by the scheduler, the PDB has never been consulted, and the + `/etc/pihole` mount has never been exercised on a real pod. Everything above is the rendered + artifact plus the image's own source. +- **THE ROLLOUT WILL BRIEFLY DROP LAN DNS, and I have not measured for how long.** `postStart` must + complete before a container is reported Running, so a pod is not Ready — and the Service has no + endpoints — until phase 1's adlist hook finishes. On this transition both replicas start with an + empty `/etc/pihole`, so the hook finds a fresh `gravity.db` with zero adlists and runs a real + `pihole -g` (72 k domains, network download) before either pod becomes Ready. **Do this at a quiet + time.** Two mitigating facts: every pihole restart has *already* paid this cost (state was + ephemeral), and after this change `gravity.db` persists so later restarts should not. If it turns + out slow, backgrounding the gravity rebuild inside the hook would decouple readiness from it, at + the cost of a window where DNS resolves but does not block — not done here. +- **Whether `pihole -g` finishes inside `apps-wave-0`'s 5 m `timeout`.** `Parallel` was chosen to + make this the max rather than the sum of two first boots, but the real duration is unmeasured (it + was stubbed during phase 1's testing). If the wave times out, the Kustomization reports failed even + though the pods may come up seconds later. +- **The old PVC's deletion.** `apps-wave-0` has `prune: true`, so removing `pihole-config` from the + rendered output should delete it. That destroys nothing (it holds `lost+found`), but I have not + observed the prune, and I did not check whether Longhorn retains the backing volume afterwards. +- **The transition overlap.** For a moment the pruned Deployment's pod and the new StatefulSet pods + may both match the Service selector. Both are configured identically from git so this should be + benign — unobserved. +- **Longhorn capacity.** Two 2 Gi claims at 3 Longhorn replicas each is ~12 Gi of real storage where + there was ~6 Gi. Not checked against free space. +- **Whether two independent query databases are acceptable to the operator.** Stats and query history + now split per replica, and the admin UI shows only the pod that answered. Adlists and password are + identical (phase 1), so blocking and auth are consistent; the *reporting* is not. +- **The residual risk is unchanged and is the larger one.** Replicas address node failure. LAN DNS + still depends on the cluster — etcd quorum, Cilium, Longhorn, a bad reconcile — and only a resolver + outside the cluster carrying the same `address=` lines removes that. Considered and not chosen + (operator decision, 2026-10-04). +- **`tolerationSeconds: 300` is untouched.** With two replicas the surviving one serves immediately, so + the 5-minute eviction delay no longer gates recovery — but it still delays the *replacement* replica, + and during that window there is no redundancy left. diff --git a/schemas/renderer-contract.md b/schemas/renderer-contract.md index ca52833..0b3827a 100644 --- a/schemas/renderer-contract.md +++ b/schemas/renderer-contract.md @@ -161,6 +161,41 @@ platform must render, and cites the requirement each mapping satisfies. Changing - **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication), **R-STATE-04** (bulk tier). (docs/04) +### `availability` (`replicas`) — workload kind, per-replica volumes, spreading, budget + +**Schema 1.10.0.** Availability INTENT. It names no Kubernetes object, no topology key and no +budget — exactly like `hardware.gpu`, and for the same reason: the mechanism below should be +replaceable without a catalog edit. + +- **Absent, or `replicas: 1`** → a **Deployment**, standalone PVC(s), `strategy: Recreate` when there + is storage, no PodDisruptionBudget. Byte-identical to pre-1.10.0 output, so every existing app is + untouched. +- **`replicas: N > 1`** → all of the following, together: + | Rendered | Why | + |---|---| + | **StatefulSet** instead of Deployment | A longhorn PVC is **ReadWriteOnce**. One RWO claim cannot attach on two nodes, so a multi-replica Deployment does not degrade — the second pod never schedules. | + | longhorn storage becomes **`volumeClaimTemplates`** | Gives each replica its own claim (`--`, e.g. `config-pihole-0`). The template's `metadata.name` **must** equal the pod volume name the container mounts. The standalone PVC is then **not** emitted — it would be an orphan nothing mounts. | + | `bulk` storage stays a **standalone PVC** | A bulk entry is a static, pre-bound NFS PV over data that already exists outside the cluster and is shared read-only. Per-replica copies are neither possible nor wanted. | + | **required** `podAntiAffinity` on `kubernetes.io/hostname` | `preferred` would let every replica land on one node and deliver none of the intended node-failure tolerance, while every object still reported healthy. | + | `podManagementPolicy: Parallel` | `OrderedReady` serialises first-boot work across replicas; `apps-wave-0` has `timeout: 5m`. There is no ordering requirement between replicas to protect. | + | **PodDisruptionBudget**, `minAvailable: 1` | Stops a *voluntary* disruption (node drain, upgrade) taking the last replica. It does nothing about a node dying — that is what the replica count and anti-affinity are for. Never rendered at `replicas: 1`, where `minAvailable: 1` would block every drain of that node forever. | + | `observability.livenessWorkload.kind` defaults to **`StatefulSet`** | The `up` SLI reads kube-state-metrics by workload kind. A Deployment-shaped rule against a StatefulSet `absent()`s forever — an alert that can never fire. Deriving the default means bumping the replica count cannot leave a stale rule behind. | + +**Two costs, both deliberate, neither hidden:** + +1. **N replicas need N schedulable nodes.** On a 3-node cluster, losing two nodes leaves a replica + `Pending`; since `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, that **stalls the + app tier**. That coincides with having lost etcd quorum on a 3-node control plane, so nothing is + reconciling at that point anyway — the stall is a symptom, not a new failure mode. +2. **Only declare `> 1` for a service whose on-disk state is reproducible from git.** A replica with + a fresh volume comes up serving whatever it can bootstrap itself and nothing else. For pihole that + prerequisite was done first — the admin password and the adlists were declared in + `decisions/2026-10-04-pihole-declare-state.md` — because otherwise a second replica would have + served DNS while blocking nothing, with a different password. + +**Requirements:** R-PLAT-02 (the workload), R-STATE-01/02 (per-replica replicated volumes), +R-OBS-01 (the SLI must follow the rendered kind). + ### `hardware` (`gpu`) — host device capability - **The field is INTENT, never MECHANISM.** `hardware.gpu` is an **enum of capability classes** (today: `intel-quicksync`), not a device path, not a hostPath, not a resource request. The diff --git a/schemas/servicedefinition.schema.json b/schemas/servicedefinition.schema.json index f1a0542..34f5543 100644 --- a/schemas/servicedefinition.schema.json +++ b/schemas/servicedefinition.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json", "title": "ServiceDefinition", - "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", + "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.10.0 (adds optional `availability` for the container kind, one member: `availability.replicas`. It is AVAILABILITY INTENT, not a scheduling primitive -- it names no Kubernetes object, no topology key and no budget. 1 (the default) renders a Deployment exactly as before, so every pre-existing app stays byte-identical; >1 means the service must survive the loss of a single node, and the renderer satisfies that with a StatefulSet whose longhorn volumes become per-replica volumeClaimTemplates (a longhorn PVC is ReadWriteOnce and cannot attach on two nodes, so a multi-replica Deployment is UNSCHEDULABLE, not merely suboptimal), REQUIRED podAntiAffinity on kubernetes.io/hostname (preferred would let every replica land on one node and deliver none of the intended tolerance while reporting healthy), podManagementPolicy Parallel, a PodDisruptionBudget of minAvailable 1, and an `up` SLI that follows the rendered kind (a Deployment-shaped rule against a StatefulSet absent()s forever and can never fire). Added for pihole, where LAN DNS -- a critical network service -- was one pod on one node: measured 2026-10-04, losing the node running it meant roughly 6-10 minutes with no LAN resolution (tolerationSeconds 300 plus node-NotReady detection plus a Longhorn RWO force-detach). Deliberately NOT usable until a service's state is reproducible from git, because a replica with a fresh volume otherwise comes up serving but unconfigured -- for pihole that prerequisite was done first (decisions/2026-10-04-pihole-declare-state.md). Capped at 5: a count above the node count can never schedule under the anti-affinity this implies. Mechanism documented in schemas/renderer-contract.md `### availability`). 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", "type": "object", "additionalProperties": false, "required": ["name", "source"], @@ -139,6 +139,21 @@ "supplementalGroups": { "type": "array", "items": { "type": "integer" }, "description": "Additional GIDs added to the first process in each container." } } }, + "availability": { + "type": "object", + "description": "Optional availability intent for the container kind: how many replicas of this workload must be running. INTENT ONLY -- it names no Kubernetes object and no scheduling primitive. How the platform satisfies it (workload kind, per-replica volumes, node spreading, disruption budget) is catalog/render.py's business, documented in schemas/renderer-contract.md `### availability`. Absent => replicas 1 and a Deployment, so every pre-existing app renders byte-identical. Ignored for kind=vm (a VM's availability is a different problem with different primitives).", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "replicas": { + "type": "integer", + "description": "How many replicas must be running. 1 (default) renders a Deployment exactly as before. >1 declares that the service must survive the loss of a single node; the renderer then emits a StatefulSet with per-replica volumeClaimTemplates, required per-node anti-affinity, podManagementPolicy Parallel and a PodDisruptionBudget. Only declare >1 for a service whose on-disk state is reproducible from git -- a replica with a fresh volume comes up serving whatever it can bootstrap itself, and nothing else.", + "minimum": 1, + "maximum": 5, + "default": 1 + } + } + }, "hardware": { "type": "object", "description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume and a matching mount on the app container, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",