Merge pull request 'feat(pihole): two replicas on different nodes, and fix a PVC mounted where pihole never writes' (#45) from feat/pihole-multi-replica into main

Reviewed-on: #45

apps/wave-0/pihole.yaml

@@ -35,34 +35,18 @@ spec:
     - protocol: TCP
       port: 53
 ---
-# PVC pihole-config (R-STATE)
+# StatefulSet (R-PLAT-02)
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: pihole-config
-  namespace: pihole
-  labels:
-    app.kubernetes.io/name: pihole
-spec:
-  accessModes:
-  - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 2Gi
----
-# Deployment (R-PLAT-02)
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: pihole
   namespace: pihole
   labels:
     app.kubernetes.io/name: pihole
 spec:
-  replicas: 1
+  replicas: 2
-  strategy:
+  serviceName: pihole
-    type: Recreate
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app.kubernetes.io/name: pihole
@@ -97,7 +81,7 @@ spec:
               key: webui-password
         volumeMounts:
         - name: config
-          mountPath: /data/config
+          mountPath: /etc/pihole
         lifecycle:
           postStart:
             exec:
@@ -175,10 +159,39 @@ spec:
             port: http
           initialDelaySeconds: 5
           periodSeconds: 15
-      volumes:
+      affinity:
-      - name: config
+        podAntiAffinity:
-        persistentVolumeClaim:
+          requiredDuringSchedulingIgnoredDuringExecution:
-          claimName: pihole-config
+          - labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: pihole
+            topologyKey: kubernetes.io/hostname
+  volumeClaimTemplates:
+  - metadata:
+      name: config
+      labels:
+        app.kubernetes.io/name: pihole
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: longhorn
+      resources:
+        requests:
+          storage: 2Gi
+---
+# PodDisruptionBudget — keep one replica through a drain (availability.replicas)
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: pihole
+  namespace: pihole
+  labels:
+    app.kubernetes.io/name: pihole
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: pihole
 ---
 # Service
 apiVersion: v1
@@ -281,11 +294,11 @@ spec:
   - name: pihole.slis
     rules:
     - alert: pihole-down
-      expr: kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"} == 0 or absent(kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"})
+      expr: kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"} == 0 or absent(kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"})
       for: 5m
       labels:
         severity: critical
         service: pihole
       annotations:
         summary: pihole is down
-        description: 'pihole: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).'
+        description: 'pihole: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready).'

apps/wave-0/tests/app-slis.test.yaml

@@ -1,6 +1,8 @@
 # promtool rule-unit tests for the wave-0 per-app `<app>-down` SLI rules.
 # Manifests under test: every PrometheusRule in apps/wave-0/ — gitea, jackett, pihole, plex,
-# qbittorrent (container/Deployment) and vm-demo (KubeVirt VMI). The harness discovers this file by
+# qbittorrent (container/Deployment), pihole (container/StatefulSet — `availability.replicas: 2`, so
+# its SLI reads kube_statefulset_status_replicas_ready; see schemas/renderer-contract.md
+# `### availability`) and vm-demo (KubeVirt VMI). The harness discovers this file by
 # the SUITE convention (`<dir>/tests/*.test.yaml`) and flattens those RENDERED manifests, so these
 # tests run against the exact expressions Flux applies, never a copy.
 #
@@ -41,7 +43,7 @@ tests:
         values: '1+0x20'
       - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}'
         values: '1+0x20'
-      - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}'
+      - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}'
         values: '1+0x20'
       - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}'
         values: '1+0x20'
@@ -81,7 +83,7 @@ tests:
         values: '1+0x20'
       - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}'
         values: '1+0x20'
-      - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}'
+      - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}'
         values: '1+0x20'
       - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}'
         values: '0+0x20'
@@ -132,7 +134,7 @@ tests:
         values: '0+0x20'
       - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}'
         values: '0+0x20'
-      - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}'
+      - series: 'kube_statefulset_status_replicas_ready{namespace="pihole", statefulset="pihole"}'
         values: '0+0x20'
       - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}'
         values: '1+0x20'
@@ -170,10 +172,10 @@ tests:
               severity: critical
               service: pihole
               namespace: pihole
-              deployment: pihole
+              statefulset: pihole
             exp_annotations:
               summary: "pihole is down"
-              description: "pihole: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)."
+              description: "pihole: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)."
       - eval_time: 10m
         alertname: qbittorrent-down
         exp_alerts:

catalog/pihole.yaml

@@ -12,6 +12,42 @@ storage:
   - name: config
     size: 2Gi
     class: longhorn           # replicated config state; DR-critical (docs/04 line 118; R-STATE-02/03)
+    # THE PVC WAS MOUNTED SOMEWHERE PIHOLE NEVER WRITES, AND SO HELD NOTHING AT ALL. Without an
+    # explicit mountPath the renderer defaults to /data/<name>, i.e. /data/config — but PiHole keeps
+    # every piece of its state in /etc/pihole. MEASURED on live prod 2026-10-04: `/etc/pihole` (7.1 M
+    # — gravity.db, pihole.toml, the query DB) was on the **overlay** filesystem, i.e. the
+    # container's EPHEMERAL layer, while `/dev/longhorn/pvc-849edb86...` sat mounted at
+    # /data/config holding `lost+found` and 24 K of nothing. Three consequences, all live until now:
+    #   * every restart silently discarded PiHole's whole state, re-downloading 72 k gravity domains
+    #     and regenerating the admin password hash;
+    #   * `backup.policy: daily-offcluster` below has been faithfully backing up an EMPTY volume,
+    #     which is worse than no backup because it reports success;
+    #   * it is also why phase 1's password/adlist declaration mattered more than it first appeared —
+    #     that state was not merely undeclared, it was ephemeral.
+    # This is the same class of bug catalog/plex.yaml's `config` entry documents ("Without this the
+    # PVC sat unused and Plex wrote to the container's EPHEMERAL layer").
+    #
+    # /etc/pihole IS THE IMAGE'S INTENDED MOUNT POINT, and an EMPTY volume there self-initialises.
+    # Verified in the running image, not assumed:
+    #   * `/usr/bin/start.sh` (PID 1) calls `migrate_gravity`, whose `migrate_to_database()` creates
+    #     the gravity database when `[ ! -e "$gravityDBfile" ]` — the same branch the postStart hook
+    #     below relies on, and the branch that produced the current ephemeral copy.
+    #   * FTL writes `pihole.toml` itself from the `FTLCONF_*` env above (that is what the
+    #     `### CHANGED (env)` markers in it are).
+    #   * NO `podSecurityContext.fsGroup` is needed and adding one would be inert: `start.sh` line 5
+    #     is `chown -R pihole:pihole /etc/pihole/ /var/log/pihole/` and PID 1 runs as **uid 0**, so
+    #     the image fixes ownership of a fresh root-owned volume before anything else starts. Same
+    #     arrangement as plex's image managing its own /config ownership.
+    #   * the image's own default for `files.macvendor` was moved OUT of this directory (to
+    #     `/macvendor.db`), which is what a maintainer does to a path they expect to be a volume.
+    #
+    # WHY IT IS FIXED IN THIS BRANCH rather than left for its own: `availability.replicas: 2` renders
+    # per-replica volumeClaimTemplates, and with the wrong mountPath those would be two 2 Gi Longhorn
+    # volumes (times three Longhorn replicas each) holding nothing but `lost+found` while all state
+    # stayed ephemeral — paying the cost of persistence and getting none of it. It also REMOVES this
+    # change's main operational risk: with gravity.db persisted, a pod restart is cheap, instead of
+    # every restart re-downloading 72 k domains inside a wave that has a 5 m timeout.
+    mountPath: /etc/pihole
 dependencies: []              # default-deny: reaches only DNS; it PROVIDES DNS (docs/03; R-NET-01/02)
 # ------------------------------------------------------------------------------------------------
 # LAN SPLIT-HORIZON DNS — the half of R-DNS-03 that platform/coredns/coredns-custom.yaml defers to.
@@ -114,6 +150,39 @@ env:
       secretKeyRef:
         name: pihole-bootstrap
         key: webui-password
+availability:
+  # TWO REPLICAS, ON DIFFERENT NODES. LAN DNS is a critical network service and it was ONE pod on
+  # ONE node. Measured 2026-10-04, that cost more than "until it reschedules":
+  #   * the node running the pod dies -> ~40s to mark it NotReady, then `tolerationSeconds: 300`
+  #     before the pod is even evicted, then reschedule, then Longhorn must force-detach the RWO
+  #     volume from a dead node and reattach it. Roughly 6-10 MINUTES with no LAN resolution.
+  #   * the node holding the `pihole-dns-*` L2 lease dies -> Cilium re-elects an announcer, tens of
+  #     seconds. (These were DIFFERENT nodes: lease on k3s-prod-2, pod on k3s-prod-1. Two of three
+  #     nodes were single points of failure for LAN DNS, with wildly different blast radii.)
+  # The Services already tolerate this: `externalTrafficPolicy: Cluster` on the DNS LoadBalancers
+  # (apps/wave-0/pihole-dns.yaml) means the L2 lease holder forwards to ANY ready backend, so a
+  # second replica is picked up with no change there.
+  #
+  # WHAT THIS DOES NOT FIX, and it is the bigger risk: LAN DNS still depends on the CLUSTER. etcd
+  # quorum loss, a Cilium regression, a Longhorn problem or a bad reconcile takes DNS down
+  # everywhere regardless of replica count. The only thing that removes that dependency is a
+  # resolver outside the cluster carrying the same `address=` lines. Considered and not chosen
+  # (operator decision, 2026-10-04); see decisions/2026-10-04-pihole-multi-replica.md.
+  #
+  # THE PREREQUISITE WAS DONE FIRST, and this field would have been actively harmful without it. A
+  # replica gets a FRESH volume, and until 2026-10-04 PiHole's admin password and adlists lived only
+  # on the PVC — so replica 1 would have come up SERVING DNS AND BLOCKING NOTHING, with a different
+  # password, and answers would have varied by which pod replied. Both are now declared (the
+  # password as a SOPS Secret -> FTLCONF_webserver_api_password; the adlists reconciled into
+  # gravity.db by the postStart hook below), which is what makes a fresh replica identical to its
+  # sibling. See decisions/2026-10-04-pihole-declare-state.md.
+  #
+  # WHY 2 AND NOT 3. Two tolerates the loss of any single node, which is the stated goal, and leaves
+  # a spare node for the evicted replica to land on under the HARD anti-affinity the renderer emits.
+  # Three would pin one replica per node with no spare, so a single node failure would leave a
+  # replica permanently Pending and — `apps-wave-0` being `wait: true` — stall the app tier for as
+  # long as the node was down. Two is the count that degrades gracefully here.
+  replicas: 2
 ingressRoutes:
   - host: pihole.${BASE_DOMAIN}
     port: 80                  # admin web UI, protected at the Gateway by forward-auth (docs/09)

catalog/render.py

@@ -272,7 +272,15 @@ def apply_defaults(sd: dict) -> dict:
     # object that actually runs on the cluster is not the one this SD renders (keycloak: the
     # Keycloak operator runs a StatefulSet, so a Deployment-shaped rule would `absent()` forever).
     lw = dict(obs.get("livenessWorkload") or {})
-    lw.setdefault("kind", "VirtualMachineInstance" if s["kind"] == "vm" else "Deployment")
+    # The default must track what is ACTUALLY rendered, or the `up` SLI silently dies: a
+    # Deployment-shaped rule against a StatefulSet `absent()`s forever, which is an alert that can
+    # never fire — the exact failure mode the keycloak override was added for. Deriving it from
+    # `availability.replicas` means bumping the replica count cannot leave a stale rule behind.
+    lw.setdefault(
+        "kind",
+        "VirtualMachineInstance" if s["kind"] == "vm"
+        else ("StatefulSet" if _is_statefulset(s) else "Deployment"),
+    )
     lw.setdefault("name", s["name"])
     obs["livenessWorkload"] = lw
     s["observability"] = obs
@@ -475,6 +483,37 @@ def r_bulk_pvs(s: dict) -> list:
     return out
 
 
+def _replicas(s: dict) -> int:
+    """Declared replica count (schema 1.10.0 `availability.replicas`). Default 1."""
+    return int((s.get("availability") or {}).get("replicas", 1) or 1)
+
+
+def _is_statefulset(s: dict) -> bool:
+    """True when this container workload must render a StatefulSet rather than a Deployment.
+
+    THE DECIDING CONSTRAINT IS THE VOLUME, not a preference. A longhorn storage entry renders
+    ReadWriteOnce (r_pvcs), and one RWO PVC cannot be attached by pods on two different nodes --
+    which is exactly what `availability.replicas > 1` plus the node anti-affinity below asks for.
+    A 2-replica Deployment over a shared RWO claim does not degrade, it simply never schedules the
+    second pod. A StatefulSet's volumeClaimTemplates give each replica its OWN claim, which is the
+    only shape that satisfies both at once.
+
+    A replicas-1 service keeps rendering a Deployment so every pre-existing golden is byte-identical.
+    """
+    return s["kind"] == "container" and _replicas(s) > 1
+
+
+def _sts_claim_volumes(s: dict) -> list:
+    """Storage entries that become per-replica volumeClaimTemplates on a StatefulSet.
+
+    Only the dynamically-provisioned (longhorn) entries. A `bulk` entry is a STATIC, pre-bound NFS
+    PV over data that already exists outside the cluster and is shared read-only by every consumer
+    (r_bulk_pvs) -- per-replica copies of it are neither possible nor wanted, so it stays a
+    standalone PVC and an ordinary pod volume.
+    """
+    return [v for v in s["storage"] if v.get("class", "longhorn") != "bulk"]
+
+
 def r_pvcs(s: dict) -> list:
     """One PVC per storage entry (R-STATE-01/02/04).
 
@@ -482,8 +521,15 @@ def r_pvcs(s: dict) -> list:
     bulk     -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName.
     """
     out = []
+    # On a StatefulSet the longhorn claims are NOT standalone objects — they are
+    # volumeClaimTemplates, so each replica gets its own (see _is_statefulset). Emitting both would
+    # leave an orphan PVC that nothing mounts and that `prune` would later delete out from under
+    # nobody. `bulk` is unaffected and still rendered here.
+    sts = _is_statefulset(s)
     for vol in s["storage"]:
         cls = vol.get("class", "longhorn")
+        if sts and cls != "bulk":
+            continue
         if cls == "bulk":
             # Mirror image of the PV: the same explicitly-empty class (so nothing dynamically
             # provisions), the same access mode, and volumeName completing the two-sided binding.
@@ -564,6 +610,12 @@ def r_deployment(s: dict) -> dict:
         if vol.get("readOnly"):
             mount["readOnly"] = True
         volume_mounts.append(mount)
+        # On a StatefulSet a longhorn volume is supplied by its volumeClaimTemplate, whose
+        # metadata.name IS the pod volume name. Adding it to pod.volumes as well is a conflict
+        # (duplicate volume name), so it is skipped here and emitted below. The MOUNT is unchanged —
+        # the container cannot tell the difference, which is what keeps this transparent to the app.
+        if _is_statefulset(s) and vol.get("class", "longhorn") != "bulk":
+            continue
         volumes.append(
             {
                 "name": vol["name"],
@@ -661,20 +713,80 @@ def r_deployment(s: dict) -> dict:
     if ann:
         tmpl_meta["annotations"] = ann
 
-    spec = {"replicas": 1}
+    sts = _is_statefulset(s)
+
+    # NODE ANTI-AFFINITY, and it is REQUIRED rather than preferred. `availability.replicas > 1`
+    # exists to survive the loss of one node; `preferred` would let both replicas land on the same
+    # node and quietly deliver none of that, with every object reporting healthy. The cost is real
+    # and is accepted: with N replicas the workload needs N schedulable nodes, so on a 3-node
+    # cluster losing TWO nodes leaves a replica Pending and — because `apps-wave-0` is `wait: true`
+    # with `apps-wave-1` dependsOn it — STALLS THE APP TIER. That coincides with having already lost
+    # etcd quorum on a 3-node control plane, i.e. the cluster is not reconciling anything at that
+    # point anyway, so the stall is a symptom rather than a new failure.
+    if sts:
+        pod_spec["affinity"] = {
+            "podAntiAffinity": {
+                "requiredDuringSchedulingIgnoredDuringExecution": [
+                    {
+                        "labelSelector": {
+                            "matchLabels": {"app.kubernetes.io/name": s["name"]}
+                        },
+                        "topologyKey": "kubernetes.io/hostname",
+                    }
+                ]
+            }
+        }
+
+    spec = {"replicas": _replicas(s)}
+    if sts:
+        # serviceName is a required StatefulSet field. It points at the service rendered by
+        # r_service; that Service is a normal ClusterIP rather than headless, so per-pod DNS
+        # (<pod>.<svc>) does NOT resolve — and nothing here needs it. These replicas are
+        # interchangeable resolvers behind one VIP, not peers that address each other.
+        spec["serviceName"] = s["name"]
+        # Parallel, not the OrderedReady default. OrderedReady starts replica N+1 only after N is
+        # Ready, so a cold start costs the sum of every replica's first-boot work instead of the
+        # max — and `apps-wave-0` has `timeout: 5m`. There is no ordering requirement between
+        # replicas to protect (no leader, no migration, no shared volume), so serialising buys
+        # nothing and risks the wave timing out on a rebuild.
+        spec["podManagementPolicy"] = "Parallel"
+    elif s["storage"]:
         # An app with persistent storage mounts a ReadWriteOnce Longhorn PVC. The default
         # RollingUpdate strategy tries to start the NEW pod before terminating the old one, but
         # the old pod still holds the RWO volume -> the new pod is stuck ContainerCreating and
         # the rollout deadlocks (single-replica app can never hand the volume over). Recreate
         # terminates the old pod first, freeing the PVC for the new one (R-STATE-01/02).
-    if s["storage"]:
+        #
+        # A StatefulSet needs none of this and cannot express it: each replica owns its own claim,
+        # so a rolling update hands nothing over, and `strategy` is not a StatefulSet field at all.
         spec["strategy"] = {"type": "Recreate"}
     spec["selector"] = {"matchLabels": {"app.kubernetes.io/name": s["name"]}}
     spec["template"] = {"metadata": tmpl_meta, "spec": pod_spec}
+    if sts:
+        # Per-replica claims. metadata.name MUST equal the pod volume name the container mounts
+        # (see the volumes loop above); the resulting PVCs are <name>-<sts>-<ordinal>, e.g.
+        # config-pihole-0. Same tier and access mode a standalone longhorn PVC gets from r_pvcs —
+        # this changes WHO owns the claim, never what it is.
+        tmpls = []
+        for vol in _sts_claim_volumes(s):
+            vspec = {"accessModes": ["ReadWriteOnce"], "storageClassName": "longhorn"}
+            if vol.get("size"):
+                vspec["resources"] = {"requests": {"storage": vol["size"]}}
+            tmpls.append(
+                {
+                    "metadata": {
+                        "name": vol["name"],
+                        "labels": {"app.kubernetes.io/name": s["name"]},
+                    },
+                    "spec": vspec,
+                }
+            )
+        if tmpls:
+            spec["volumeClaimTemplates"] = tmpls
 
     return {
         "apiVersion": "apps/v1",
-        "kind": "Deployment",
+        "kind": "StatefulSet" if sts else "Deployment",
         "metadata": {
             "name": s["name"],
             "namespace": s["name"],
@@ -684,6 +796,35 @@ def r_deployment(s: dict) -> dict:
     }
 
 
+def r_pdb(s: dict):
+    """PodDisruptionBudget for a multi-replica workload (availability.replicas > 1).
+
+    minAvailable 1, not `maxUnavailable: 1`. The budget's job here is to stop a VOLUNTARY
+    disruption — a node drain for maintenance, a cluster upgrade — from taking the last replica
+    with it; `minAvailable: 1` says that whatever else happens, one must keep serving. It does
+    NOT protect against involuntary disruption (a node dying), which is what the anti-affinity
+    and the replica count are for.
+
+    Rendered only when replicas > 1: a PDB over a single-replica workload is actively harmful, as
+    `minAvailable: 1` would then block every drain of its node forever.
+    """
+    if not _is_statefulset(s):
+        return None
+    return {
+        "apiVersion": "policy/v1",
+        "kind": "PodDisruptionBudget",
+        "metadata": {
+            "name": s["name"],
+            "namespace": s["name"],
+            "labels": {"app.kubernetes.io/name": s["name"]},
+        },
+        "spec": {
+            "minAvailable": 1,
+            "selector": {"matchLabels": {"app.kubernetes.io/name": s["name"]}},
+        },
+    }
+
+
 def r_service(s: dict):
     ports = _service_ports(s)
     # A declared metrics endpoint becomes a NAMED Service port, because that name is the only
@@ -1817,7 +1958,13 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
             )
         for pvc in r_pvcs(s):
             docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc))
-        docs.append(("Deployment (R-PLAT-02)", r_deployment(s)))
+        wl = r_deployment(s)
+        docs.append((f"{wl['kind']} (R-PLAT-02)", wl))
+        pdb = r_pdb(s)
+        if pdb:
+            docs.append(
+                ("PodDisruptionBudget — keep one replica through a drain (availability.replicas)", pdb)
+            )
         svc = r_service(s)
         if svc:
             docs.append(("Service", svc))
@@ -2266,6 +2413,132 @@ def _selftest() -> int:
     check("Namespace" in kinds, "container: missing Namespace")
     check("NetworkPolicy" in kinds, "container: missing default-deny NetworkPolicy")
     check("Deployment" in kinds, "container: missing Deployment")
+    check(
+        "PodDisruptionBudget" not in kinds,
+        "replicas-1 container must render NO PodDisruptionBudget (minAvailable 1 over a single "
+        "replica blocks every drain of its node forever)",
+    )
+
+    # 1b) availability.replicas (schema 1.10.0). The DEFAULT path must be untouched, and the >1
+    # path must change workload kind, claim ownership, spreading, budget AND the SLI metric
+    # together — each of these has a silent-failure mode if it is the one that gets forgotten.
+    base_av = {
+        "name": "av",
+        "source": {"image": "nginx:1"},
+        "storage": [{"name": "config", "size": "1Gi"}],
+        "observability": {"metrics": True, "slis": ["up"]},
+    }
+    one = render({**base_av, "availability": {"replicas": 1}})
+    one_kinds = [d[1]["kind"] for d in one]
+    check(
+        "Deployment" in one_kinds and "StatefulSet" not in one_kinds,
+        "availability.replicas=1 must still render a Deployment (back-compat)",
+    )
+    check(
+        "PersistentVolumeClaim" in one_kinds,
+        "availability.replicas=1 must still render a standalone PVC",
+    )
+    check(
+        "PodDisruptionBudget" not in one_kinds,
+        "availability.replicas=1 must render no PodDisruptionBudget",
+    )
+
+    many = render({**base_av, "availability": {"replicas": 2}})
+    many_kinds = [d[1]["kind"] for d in many]
+    check(
+        "StatefulSet" in many_kinds and "Deployment" not in many_kinds,
+        "availability.replicas>1 must render a StatefulSet, not a Deployment (a shared RWO claim "
+        "cannot be attached on two nodes, so a multi-replica Deployment never schedules)",
+    )
+    check(
+        "PersistentVolumeClaim" not in many_kinds,
+        "availability.replicas>1 must NOT also render a standalone longhorn PVC — it would be an "
+        "orphan that nothing mounts",
+    )
+    sts = next(d[1] for d in many if d[1]["kind"] == "StatefulSet")
+    check(sts["spec"].get("replicas") == 2, "StatefulSet must carry the declared replica count")
+    check(
+        "strategy" not in sts["spec"],
+        "StatefulSet must not carry `strategy` (not a StatefulSet field; Recreate is a "
+        "Deployment-only workaround for handing over one RWO volume)",
+    )
+    check(
+        sts["spec"].get("podManagementPolicy") == "Parallel",
+        "StatefulSet must set podManagementPolicy=Parallel — OrderedReady serialises first-boot "
+        "work across replicas and apps-wave-0 has a 5m timeout",
+    )
+    check(sts["spec"].get("serviceName") == "av", "StatefulSet must set serviceName")
+    tmpls = sts["spec"].get("volumeClaimTemplates") or []
+    check(
+        [t["metadata"]["name"] for t in tmpls] == ["config"],
+        "StatefulSet volumeClaimTemplate name must equal the pod volume name the container mounts, "
+        "or the pod gets a duplicate/unresolvable volume",
+    )
+    check(
+        tmpls and tmpls[0]["spec"]["accessModes"] == ["ReadWriteOnce"]
+        and tmpls[0]["spec"]["storageClassName"] == "longhorn",
+        "StatefulSet claim template must keep the same tier/access mode as the standalone PVC",
+    )
+    sts_vols = {v["name"] for v in sts["spec"]["template"]["spec"].get("volumes", [])}
+    check(
+        "config" not in sts_vols,
+        "a claim-template volume must NOT also appear in pod.volumes (duplicate volume name)",
+    )
+    mounts = sts["spec"]["template"]["spec"]["containers"][0].get("volumeMounts") or []
+    check(
+        any(m["name"] == "config" for m in mounts),
+        "the container must still mount the volume by name on a StatefulSet (transparent to the app)",
+    )
+    aff = (sts["spec"]["template"]["spec"].get("affinity") or {}).get("podAntiAffinity") or {}
+    req = aff.get("requiredDuringSchedulingIgnoredDuringExecution") or []
+    check(
+        req and req[0].get("topologyKey") == "kubernetes.io/hostname",
+        "availability.replicas>1 must render REQUIRED per-node anti-affinity — `preferred` can put "
+        "every replica on one node and deliver none of the intended node-failure tolerance",
+    )
+    pdbs = [d[1] for d in many if d[1]["kind"] == "PodDisruptionBudget"]
+    check(len(pdbs) == 1, "availability.replicas>1 must render exactly one PodDisruptionBudget")
+    check(
+        pdbs and pdbs[0]["spec"].get("minAvailable") == 1,
+        "PodDisruptionBudget must be minAvailable=1 (keep one replica serving through a drain)",
+    )
+    pr_many = next((d[1] for d in many if d[1]["kind"] == "PrometheusRule"), None)
+    check(
+        pr_many is not None
+        and "kube_statefulset_status_replicas_ready"
+        in pr_many["spec"]["groups"][0]["rules"][0]["expr"],
+        "the up SLI must follow the rendered workload kind — a Deployment-shaped rule against a "
+        "StatefulSet absent()s forever and can never fire",
+    )
+
+    # A bulk volume is shared external NFS: it must stay a standalone PVC even on a StatefulSet.
+    mixed = render(
+        {
+            **base_av,
+            "availability": {"replicas": 2},
+            "storage": [
+                {"name": "config", "size": "1Gi"},
+                {"name": "media", "class": "bulk", "export": "/mnt/x", "size": "1Ti"},
+            ],
+        }
+    )
+    m_sts = next(d[1] for d in mixed if d[1]["kind"] == "StatefulSet")
+    check(
+        [t["metadata"]["name"] for t in m_sts["spec"]["volumeClaimTemplates"]] == ["config"],
+        "only longhorn volumes become claim templates; a bulk share must not be per-replica",
+    )
+    check(
+        any(
+            d[1]["kind"] == "PersistentVolumeClaim"
+            and d[1]["metadata"]["name"] == "av-media"
+            for d in mixed
+        ),
+        "a bulk PVC must still be rendered standalone on a StatefulSet",
+    )
+    check(
+        "media" in {v["name"] for v in m_sts["spec"]["template"]["spec"].get("volumes", [])},
+        "a bulk volume must still appear in pod.volumes on a StatefulSet",
+    )
 
     ns = next(d[1] for d in docs if d[1]["kind"] == "Namespace")
     check(

decisions/2026-10-04-pihole-multi-replica.md

@@ -0,0 +1,145 @@
+---
+date: 2026-10-04
+branch: feat/pihole-multi-replica
+title: PiHole runs two replicas on different nodes — and its config PVC was mounted where PiHole never writes
+---
+
+## What changed
+
+Phase 2 of making LAN DNS survive a node failure (phase 1:
+`decisions/2026-10-04-pihole-declare-state.md`). PiHole was one pod on one node, which for a
+whole-house resolver cost more than "until it reschedules" — measured 2026-10-04:
+
+| Failure | Recovery |
+|---|---|
+| the node running the pod dies | ~40s to mark NotReady + `tolerationSeconds: 300` before eviction + reschedule + a Longhorn RWO force-detach → **roughly 6-10 minutes with no LAN DNS** |
+| the node holding the `pihole-dns-*` L2 lease dies | Cilium re-elects an announcer, tens of seconds |
+
+Those were **different** nodes (lease on k3s-prod-2, pod on k3s-prod-1), so two of three nodes were
+single points of failure for LAN DNS with wildly different blast radii.
+
+**A new schema field, because this is intent and not a primitive.** `availability.replicas`
+(schema **1.10.0**, mechanism in `schemas/renderer-contract.md` `### availability`). `1` or absent
+renders a Deployment exactly as before — every pre-existing golden is byte-identical, verified. `>1`
+renders, together:
+
+- a **StatefulSet** instead of a Deployment. This is forced, not preferred: a longhorn PVC is
+  **ReadWriteOnce**, and one RWO claim cannot attach on two nodes, so a 2-replica Deployment does not
+  degrade — the second pod never schedules.
+- longhorn storage as **per-replica `volumeClaimTemplates`** (`config-pihole-0`, `config-pihole-1`);
+  the standalone PVC is no longer emitted, since it would be an orphan nothing mounts. A `bulk`
+  entry stays a standalone PVC — it is a shared static NFS export, not per-replica state.
+- **required** `podAntiAffinity` on `kubernetes.io/hostname`. `preferred` would let both replicas
+  land on one node and deliver none of the intended tolerance while reporting healthy.
+- `podManagementPolicy: Parallel`, so a cold start costs the max of the replicas' first-boot work
+  rather than the sum — `apps-wave-0` has `timeout: 5m`.
+- a **PodDisruptionBudget**, `minAvailable: 1`, so a node drain cannot take the last replica.
+- `observability.livenessWorkload.kind` defaulting to `StatefulSet`, so the `up` SLI reads
+  `kube_statefulset_status_replicas_ready`. A Deployment-shaped rule against a StatefulSet
+  `absent()`s forever — an alert that can never fire.
+
+`catalog/pihole.yaml` sets `replicas: 2`, not 3: two tolerates any single node loss and leaves a
+spare node for the evicted replica under the hard anti-affinity. Three would pin one per node with no
+spare, so a single node failure would leave a replica permanently `Pending` and — `apps-wave-0` being
+`wait: true` — stall the app tier for as long as the node was down.
+
+### The bug found on the way, and fixed here
+
+**PiHole's config PVC was mounted at a path PiHole never writes, so it held nothing and all of its
+state was ephemeral.** With no explicit `mountPath` the renderer defaults to `/data/<name>`; PiHole
+keeps everything in `/etc/pihole`. Measured on live prod 2026-10-04: `/etc/pihole` (7.1 M —
+`gravity.db`, `pihole.toml`, the query DB) was on the **overlay** filesystem, while the 2 Gi Longhorn
+volume sat at `/data/config` holding `lost+found` and 24 K of nothing. Consequences, all live until
+now:
+
+- every restart silently discarded PiHole's entire state and re-downloaded 72 k gravity domains;
+- `backup.policy: daily-offcluster` has been faithfully backing up an **empty** volume, which is
+  worse than no backup because it reports success;
+- it also means phase 1 was understated: that state was not merely undeclared, it was **ephemeral**.
+
+Same class of bug `catalog/plex.yaml` documents for its own `config` entry. Fixed in this branch
+rather than its own because with the wrong path the new per-replica volumes would be two 2 Gi
+Longhorn volumes (×3 Longhorn replicas each) holding only `lost+found` while all state stayed
+ephemeral — paying for persistence and getting none. It also removes this change's main operational
+risk: with `gravity.db` persisted, a restart is cheap instead of re-downloading 72 k domains inside a
+wave with a 5 m timeout.
+
+## What I verified
+
+- **Back-compatibility, the thing most likely to break quietly:** after the renderer change and
+  before `catalog/pihole.yaml` declared anything, the canonical
+  `render.py --all catalog -o apps --wave-layout --no-image-automation` produced **zero diff** across
+  the whole `apps/` tree. `render-golden` passes.
+- **The new behaviour is pinned by ~20 new `render.py --selftest` assertions** (so `ci/render-golden.sh`
+  hard-fails on a regression, and no `conformance/**` change was needed): `replicas: 1` still renders a
+  Deployment + standalone PVC + no PDB; `replicas: 2` renders a StatefulSet and **no** standalone
+  longhorn PVC, carries the replica count, omits `strategy`, sets `Parallel` and `serviceName`, names
+  the claim template exactly as the mounted volume, keeps RWO/longhorn on it, does **not** duplicate
+  that volume in `pod.volumes`, still mounts it on the container, renders required hostname
+  anti-affinity, renders exactly one `minAvailable: 1` PDB, and switches the SLI to
+  `kube_statefulset_status_replicas_ready`. A mixed longhorn+bulk case asserts the bulk share stays a
+  standalone PVC and an ordinary pod volume.
+- **Rendered pihole output, read back:** `replicas: 2`, `serviceName: pihole`,
+  `podManagementPolicy: Parallel`, no `strategy`, claim template `config` (RWO/longhorn/2Gi),
+  anti-affinity `kubernetes.io/hostname`, `pod.volumes: []`, mount `config -> /etc/pihole`,
+  PDB `minAvailable: 1`, SLI `kube_statefulset_status_replicas_ready`.
+- **The `/etc/pihole` mount is safe, evidenced from the image's own init rather than assumed:**
+  `/usr/bin/start.sh` is PID 1 and runs as **uid 0**; its line 5 is
+  `chown -R pihole:pihole /etc/pihole/ /var/log/pihole/`, so a fresh root-owned volume has its
+  ownership fixed before anything else starts (**no `fsGroup` needed, and one would be inert**); it
+  calls `migrate_gravity`, whose `migrate_to_database()` creates the gravity database when
+  `[ ! -e "$gravityDBfile" ]`; FTL writes `pihole.toml` itself from the `FTLCONF_*` env; and the
+  image's own default for `files.macvendor` was moved **out** of that directory, which is what a
+  maintainer does to a path they expect to be a volume.
+- **The PVC really is unused:** `df` showed `/etc/pihole` on `overlay` and
+  `/dev/longhorn/pvc-849edb86-...` on `/data/config`; `ls -A /data/config` → `lost+found` only.
+- **The gate caught a real regression and it is fixed.** The first run came back **rc 1**:
+  `alert-rules-unit-tested` failed because `apps/wave-0/tests/app-slis.test.yaml` fed pihole a
+  `kube_deployment_status_replicas_available` series and expected a `deployment: pihole` label. Three
+  input series, the firing expectation and the file header were updated;
+  `platform/monitoring/tests/run.sh apps/wave-0/tests/app-slis.test.yaml` → **SUCCESS** (6 rules).
+- **Schema:** `check-jsonschema` passes over all of `catalog/*.yaml`.
+- **Toolbox gate:** rc **3**, with `spec-immutability`, `decisions-log`, `lint`, `l0`,
+  `render-golden`, `l1`, `approval-policy` all **pass**. The rc 3 is the two environmental skips —
+  `gitea-settings` (no local `SETTINGS_API_TOKEN`) and `conformance` L2/L3 (`CLUSTER_AVAILABLE`
+  unset). **Not green; reported as rc 3.**
+- **No sensitive paths touched**, so unlike the previous two branches this one needs no
+  `/approve-override` — confirmed against `infra/gitea/approval-policy.json`.
+
+## What I did NOT verify
+
+- **Nothing has been reconciled.** No StatefulSet exists, no second replica has ever run, the
+  anti-affinity has never been evaluated by the scheduler, the PDB has never been consulted, and the
+  `/etc/pihole` mount has never been exercised on a real pod. Everything above is the rendered
+  artifact plus the image's own source.
+- **THE ROLLOUT WILL BRIEFLY DROP LAN DNS, and I have not measured for how long.** `postStart` must
+  complete before a container is reported Running, so a pod is not Ready — and the Service has no
+  endpoints — until phase 1's adlist hook finishes. On this transition both replicas start with an
+  empty `/etc/pihole`, so the hook finds a fresh `gravity.db` with zero adlists and runs a real
+  `pihole -g` (72 k domains, network download) before either pod becomes Ready. **Do this at a quiet
+  time.** Two mitigating facts: every pihole restart has *already* paid this cost (state was
+  ephemeral), and after this change `gravity.db` persists so later restarts should not. If it turns
+  out slow, backgrounding the gravity rebuild inside the hook would decouple readiness from it, at
+  the cost of a window where DNS resolves but does not block — not done here.
+- **Whether `pihole -g` finishes inside `apps-wave-0`'s 5 m `timeout`.** `Parallel` was chosen to
+  make this the max rather than the sum of two first boots, but the real duration is unmeasured (it
+  was stubbed during phase 1's testing). If the wave times out, the Kustomization reports failed even
+  though the pods may come up seconds later.
+- **The old PVC's deletion.** `apps-wave-0` has `prune: true`, so removing `pihole-config` from the
+  rendered output should delete it. That destroys nothing (it holds `lost+found`), but I have not
+  observed the prune, and I did not check whether Longhorn retains the backing volume afterwards.
+- **The transition overlap.** For a moment the pruned Deployment's pod and the new StatefulSet pods
+  may both match the Service selector. Both are configured identically from git so this should be
+  benign — unobserved.
+- **Longhorn capacity.** Two 2 Gi claims at 3 Longhorn replicas each is ~12 Gi of real storage where
+  there was ~6 Gi. Not checked against free space.
+- **Whether two independent query databases are acceptable to the operator.** Stats and query history
+  now split per replica, and the admin UI shows only the pod that answered. Adlists and password are
+  identical (phase 1), so blocking and auth are consistent; the *reporting* is not.
+- **The residual risk is unchanged and is the larger one.** Replicas address node failure. LAN DNS
+  still depends on the cluster — etcd quorum, Cilium, Longhorn, a bad reconcile — and only a resolver
+  outside the cluster carrying the same `address=` lines removes that. Considered and not chosen
+  (operator decision, 2026-10-04).
+- **`tolerationSeconds: 300` is untouched.** With two replicas the surviving one serves immediately, so
+  the 5-minute eviction delay no longer gates recovery — but it still delays the *replacement* replica,
+  and during that window there is no redundancy left.

schemas/renderer-contract.md

@@ -161,6 +161,41 @@ platform must render, and cites the requirement each mapping satisfies. Changing
 - **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication),
   **R-STATE-04** (bulk tier). (docs/04)
 
+### `availability` (`replicas`) — workload kind, per-replica volumes, spreading, budget
+
+**Schema 1.10.0.** Availability INTENT. It names no Kubernetes object, no topology key and no
+budget — exactly like `hardware.gpu`, and for the same reason: the mechanism below should be
+replaceable without a catalog edit.
+
+- **Absent, or `replicas: 1`** → a **Deployment**, standalone PVC(s), `strategy: Recreate` when there
+  is storage, no PodDisruptionBudget. Byte-identical to pre-1.10.0 output, so every existing app is
+  untouched.
+- **`replicas: N > 1`** → all of the following, together:
+  | Rendered | Why |
+  |---|---|
+  | **StatefulSet** instead of Deployment | A longhorn PVC is **ReadWriteOnce**. One RWO claim cannot attach on two nodes, so a multi-replica Deployment does not degrade — the second pod never schedules. |
+  | longhorn storage becomes **`volumeClaimTemplates`** | Gives each replica its own claim (`<vol>-<svc>-<ordinal>`, e.g. `config-pihole-0`). The template's `metadata.name` **must** equal the pod volume name the container mounts. The standalone PVC is then **not** emitted — it would be an orphan nothing mounts. |
+  | `bulk` storage stays a **standalone PVC** | A bulk entry is a static, pre-bound NFS PV over data that already exists outside the cluster and is shared read-only. Per-replica copies are neither possible nor wanted. |
+  | **required** `podAntiAffinity` on `kubernetes.io/hostname` | `preferred` would let every replica land on one node and deliver none of the intended node-failure tolerance, while every object still reported healthy. |
+  | `podManagementPolicy: Parallel` | `OrderedReady` serialises first-boot work across replicas; `apps-wave-0` has `timeout: 5m`. There is no ordering requirement between replicas to protect. |
+  | **PodDisruptionBudget**, `minAvailable: 1` | Stops a *voluntary* disruption (node drain, upgrade) taking the last replica. It does nothing about a node dying — that is what the replica count and anti-affinity are for. Never rendered at `replicas: 1`, where `minAvailable: 1` would block every drain of that node forever. |
+  | `observability.livenessWorkload.kind` defaults to **`StatefulSet`** | The `up` SLI reads kube-state-metrics by workload kind. A Deployment-shaped rule against a StatefulSet `absent()`s forever — an alert that can never fire. Deriving the default means bumping the replica count cannot leave a stale rule behind. |
+
+**Two costs, both deliberate, neither hidden:**
+
+1. **N replicas need N schedulable nodes.** On a 3-node cluster, losing two nodes leaves a replica
+   `Pending`; since `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, that **stalls the
+   app tier**. That coincides with having lost etcd quorum on a 3-node control plane, so nothing is
+   reconciling at that point anyway — the stall is a symptom, not a new failure mode.
+2. **Only declare `> 1` for a service whose on-disk state is reproducible from git.** A replica with
+   a fresh volume comes up serving whatever it can bootstrap itself and nothing else. For pihole that
+   prerequisite was done first — the admin password and the adlists were declared in
+   `decisions/2026-10-04-pihole-declare-state.md` — because otherwise a second replica would have
+   served DNS while blocking nothing, with a different password.
+
+**Requirements:** R-PLAT-02 (the workload), R-STATE-01/02 (per-replica replicated volumes),
+R-OBS-01 (the SLI must follow the rendered kind).
+
 ### `hardware` (`gpu`) — host device capability
 - **The field is INTENT, never MECHANISM.** `hardware.gpu` is an **enum of capability classes**
   (today: `intel-quicksync`), not a device path, not a hostPath, not a resource request. The

schemas/servicedefinition.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
   "title": "ServiceDefinition",
-  "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked proLine truncated
+  "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.10.0 (adds optional `availability` for the container kind, one member: `availability.replicas`. It is AVAILABILITY INTENT, not a scheduling primitive -- it names no Kubernetes object, no topology key and no budget. 1 (the default) renders a Deployment exactly as before, so every pre-existing app stays byte-identical; >1 means the service must survive the loss of a single node, and the renderer satisfies that with a StatefulSet whose longhorn volumes become per-replica volumeClaimTemplates (a longhorn PVC is ReadWriteOnce and cannot attach on two nodes, so a multi-replica Deployment is UNSCHEDULABLE, not merely suboptimal), REQUIRED podAntiAffinity on kubernetes.io/hostname (preferred would let every replica land on one node and deliver none of the intended tolerance while reporting healthy), podManagementPolicy Parallel, a PodDisruptionBudget of minAvailable 1, and an `up` SLI that follows the rendered kind (a Deployment-shaped rule against a StatefulSet absent()s forever and can never fire). Added for pihole, where LAN DNS -- a critical network service -- was one pod on one node: measured 2026-10-04, losing the node running it meant roughly 6-10 minutes with no LAN resolution (tolerationSeconds 300 plus node-NotReady detection plus a Longhorn RWO force-detach). Deliberately NOT usable until a service's state is reproducible from git, because a replica with a fresh volume otherwise comes up serving but unconfigured -- for pihole that prerequisite was done first (decisions/2026-10-04-pihole-declare-state.md). Capped at 5: a count above the node count can never schedule under the anti-affinity this implies. Mechanism documented in schemas/renderer-contract.md `### availability`). 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.mdLine truncated
   "type": "object",
   "additionalProperties": false,
   "required": ["name", "source"],
@@ -139,6 +139,21 @@
         "supplementalGroups": { "type": "array", "items": { "type": "integer" }, "description": "Additional GIDs added to the first process in each container." }
       }
     },
+    "availability": {
+      "type": "object",
+      "description": "Optional availability intent for the container kind: how many replicas of this workload must be running. INTENT ONLY -- it names no Kubernetes object and no scheduling primitive. How the platform satisfies it (workload kind, per-replica volumes, node spreading, disruption budget) is catalog/render.py's business, documented in schemas/renderer-contract.md `### availability`. Absent => replicas 1 and a Deployment, so every pre-existing app renders byte-identical. Ignored for kind=vm (a VM's availability is a different problem with different primitives).",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "replicas": {
+          "type": "integer",
+          "description": "How many replicas must be running. 1 (default) renders a Deployment exactly as before. >1 declares that the service must survive the loss of a single node; the renderer then emits a StatefulSet with per-replica volumeClaimTemplates, required per-node anti-affinity, podManagementPolicy Parallel and a PodDisruptionBudget. Only declare >1 for a service whose on-disk state is reproducible from git -- a replica with a fresh volume comes up serving whatever it can bootstrap itself, and nothing else.",
+          "minimum": 1,
+          "maximum": 5,
+          "default": 1
+        }
+      }
+    },
     "hardware": {
       "type": "object",
       "description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume and a matching mount on the app container, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",