Merge pull request 'feat(gpu): satisfy hardware.gpu: intel-quicksync with gpu.intel.com/i915 instead of a /dev/dri hostPath (the hostPath was measured not to work: device-cgroup EPERM)' (#37) from feat/quicksync-via-device-plugin into main

Reviewed-on: #37
Reviewed-by: Keith Jackson <[email protected]>

apps/wave-0/jackett.yaml

@@ -100,11 +100,15 @@ spec:
         - --allowed-group=platform-admins
         - --skip-provider-button=true
         - --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --api-route=^/api/
+        - --cookie-name=_oauth2_proxy_jackett
+        - --cookie-domain=jackett.${BASE_DOMAIN}
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -125,6 +129,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

apps/wave-0/pihole.yaml

@@ -108,11 +108,14 @@ spec:
         - --allowed-group=platform-admins
         - --skip-provider-button=true
         - --redirect-url=https://pihole.${BASE_DOMAIN}/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --cookie-name=_oauth2_proxy_pihole
+        - --cookie-domain=pihole.${BASE_DOMAIN}
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -133,6 +136,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

apps/wave-0/plex.yaml

@@ -137,6 +137,9 @@ spec:
             secretKeyRef:
               name: plex-claim
               key: PLEX_CLAIM
+        resources:
+          limits:
+            gpu.intel.com/i915: 1
         volumeMounts:
         - name: config
           mountPath: /config

apps/wave-0/qbittorrent.yaml

@@ -171,11 +171,14 @@ spec:
         - --allowed-group=platform-admins
         - --skip-provider-button=true
         - --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --cookie-name=_oauth2_proxy_qbittorrent
+        - --cookie-domain=qbittorrent.${BASE_DOMAIN}
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -196,6 +199,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

apps/wave-1/radarr.yaml

@@ -153,11 +153,16 @@ spec:
         - --allowed-group=platform-admins
         - --skip-provider-button=true
         - --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --api-route=^/api/
+        - --api-route=^/signalr/
+        - --cookie-name=_oauth2_proxy_radarr
+        - --cookie-domain=radarr.${BASE_DOMAIN}
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -178,6 +183,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

apps/wave-1/sonarr.yaml

@@ -153,11 +153,14 @@ spec:
         - --allowed-group=platform-admins
         - --skip-provider-button=true
         - --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --cookie-name=_oauth2_proxy_sonarr
+        - --cookie-domain=sonarr.${BASE_DOMAIN}
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -178,6 +181,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

catalog/jackett.yaml

@@ -51,6 +51,23 @@ auth:
     # This grants no real privilege: platform-admins already maps to cluster-admin via
     # k8s-oidc-rbac.yaml (oidc:platform-admins), so a member can already exec into these pods.
     - platform-admins           # operators of these services must not be locked out of them
+  # Answer Jackett's OWN API with 401 instead of a 302 to Keycloak (schema 1.9.0; renders
+  # --api-route, singular and repeatable, onto the oauth2-proxy sidecar).
+  #
+  # MEASURED on this pod's access log, 2026-09-30: on /api/ paths, 33 x 302 against 2 x 401 —
+  # and 4 of 6 POSTs were answered BY THE PROXY and never reached Jackett (access-log upstream
+  # field `-`, not `/`), including both `/api/v2.0/server/update` calls. Jackett's jQuery does
+  # not send the exact `Accept: application/json` oauth2-proxy's AJAX guard requires, so the
+  # 302 was followed invisibly, satisfied silently by the live Keycloak SSO session, and
+  # resolved 200 carrying the dashboard's HTML where the JavaScript expected JSON. A browser
+  # following a 302 also re-issues a POST as a GET and DROPS THE BODY, which is why "adding an
+  # indexer silently does nothing" was the user-visible symptom and nothing logged a failure.
+  #
+  # SCOPE: `/api/` only. Jackett serves its UI at /UI/ and its static assets elsewhere, and a
+  # browser never NAVIGATES to /api/ — so a plain page load is still 302'd into a silent SSO
+  # sign-in, which is the behaviour that must not change.
+  apiRoutes:
+    - ^/api/
 update:
   channel: stable               # R-UPD-01
   range: ">=0.22.0"             # stay on the Jackett 0.22.x stable line

catalog/pihole.yaml

@@ -107,6 +107,13 @@ auth:
   mode: forward-auth          # oauth2-proxy -> Keycloak; bare UI, no native OIDC (docs/15; R-IDP-02)
   groups:
     - platform-admins         # sensitive admin UI: authenticated AND in group (R-IDP-06)
+  # NO `apiRoutes` HERE, DELIBERATELY. PiHole's admin UI serves no /api/ path, so the `^/api/`
+  # that jackett and radarr need would match nothing on a good day — and a BROADER guess would
+  # answer PiHole's own page loads with 401, leaving a logged-out operator on a bare error page
+  # with no sign-in offered. This app is the reason auth.apiRoutes is per-service (schema 1.9.0)
+  # rather than a renderer constant: the measurement that justifies the flag on two apps is
+  # evidence about those two apps. Its oauth2-proxy log holds 0 unauthenticated requests across
+  # the current pod's whole 23h lifetime (measured 2026-10-01), so there is nothing to act on.
 update:
   channel: stable             # R-UPD-01
   range: ">=2024.0.0"

catalog/plex.yaml

@@ -80,6 +80,60 @@ env:
   # remote access are UNVERIFIED. Advertising a wrong/internal-only URL breaks remote access for ALL
   # clients, so per "prefer safety over completeness" it is left unset. FOLLOW-UP: once remote
   # reachability through the Gateway is confirmed, set ADVERTISE_IP: "https://plex.home.keithjackson.dev:443".
+# HARDWARE TRANSCODING — the host iGPU's render device (schema 1.8.0 `hardware`; R-PLAT-02).
+# INTENT ONLY, AND THE MECHANISM DELIBERATELY DOES NOT LIVE HERE. This says "this workload needs an
+# Intel Quick Sync render device"; it does NOT name a device path and it does NOT name a resource.
+# How the platform hands the device over is `catalog/render.py`'s business and is documented in
+# `schemas/renderer-contract.md` `### hardware` — read that for the mechanism, never this comment.
+# The payoff is proven, not theoretical: on 2026-10-01 the mechanism was swapped outright (hostPath
+# -> Intel GPU device plugin extended resource) and THIS BLOCK DID NOT MOVE — a `render.py` change
+# plus a re-render, zero catalog edits. Do not "simplify" it into a raw `/dev/...` path; that is the
+# whole point of the enum.
+#
+# THE DEVICE IS PRESENT ON EVERY PROD NODE, so nothing had to be installed and Plex can land
+# anywhere. Measured on all three nodes 2026-09-30: `/dev/dri/renderD128` present, `root:render`,
+# GID 992, mode 0660, driver `i915` bound (NOT `xe`), UHD Graphics 630 on an i7-9700T, media engines
+# `vcs0`/`vecs0` live, iGPU enabled in BIOS on each. Measured again 2026-10-01, one layer up: all
+# three nodes advertise `gpu.intel.com/i915: 1` in `.status.allocatable`. This field renders no
+# nodeSelector; placement is a separate concern with its own justification.
+#
+# PLEX'S SCHEDULABILITY IS NOW COUPLED TO THE `intel-gpu-plugin` DaemonSet, and that is a real cost,
+# not a footnote. The current mechanism is a hard scheduling constraint: if the resource is
+# unavailable (plugin broken, removed, or not yet reconciled) this pod is UNSCHEDULABLE, and because
+# `apps-wave-0` is `wait: true` with `apps-wave-1` `dependsOn` it, that STALLS THE WHOLE APP TIER.
+# The previous hostPath mechanism degraded silently to software transcoding instead — but it never
+# actually granted access (device-cgroup EPERM, measured 2026-10-01), so there was nothing to keep.
+# THE ESCAPE HATCH IS THIS BLOCK: delete these two lines and re-render, and Plex goes back to
+# software transcoding and the tier unblocks. One field, no code change. `runbooks/plex-quicksync.md`
+# carries that procedure for whoever needs it at 2am.
+#
+# NO `podSecurityContext.supplementalGroups`, AND ADDING ONE WOULD BE INERT — do not "fix" this.
+# The device is `root:render` 0660 and this process is uid/gid 1000, so the obvious move is to push
+# the host `render` GID in as a supplemental group. It cannot work here, and this survived the
+# mechanism swap unchanged. MEASURED IN THE RUNNING CONTAINER, 2026-09-30 and again 2026-10-01: the
+# image's own `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` stats every char device
+# under `/dev/dri`, `groupadd`s a matching group INSIDE the container when none exists, and
+# `usermod -a -G`s the `plex` user into it — every start, nothing hardcoded, which also makes a host
+# GID renumbering self-healing (no GID is pinned anywhere in this repo) and which works the same way
+# on a PLUGIN-INJECTED device as on a bind-mounted one. Measured live, `Plex Media Server` ran with
+# `Groups: 44 100 992 1000` — 992 IS the host `render` GID, i.e. the hook had already done the job.
+# The service execs via `s6-setuidgid plex`, and `s6-setuidgid` REPLACES the process group list from
+# `/etc/group` rather than appending to it, so root's gid `0` — which PID 1 (`s6-svscan`) does hold —
+# is GONE, and anything Kubernetes supplies reaches s6-svscan and is discarded before Plex ever
+# starts. The group is the image's job and it does it correctly. `render.py --selftest` asserts the
+# ABSENCE of a pod securityContext for this field precisely so nobody adds an inert one back later.
+#
+# THE LAST MILE IS A MANUAL OPERATOR STEP AND CANNOT LIVE IN THIS REPO. Handing over the device does
+# not enable hardware transcoding — Plex's own `HardwareAcceleratedCodecs` preference does, and it
+# is ABSENT from `Preferences.xml` today (the toggle has never been on). NO environment variable can
+# set it: a grep of the running image's entire init surface (`/etc/cont-init.d`, `/etc/services.d`,
+# `/etc/fix-attrs.d`, `/plex-common.sh`, `/usr/local/bin`) found ZERO `PLEX_PREFERENCE*` support,
+# and the six prefs the image does write (`MachineIdentifier`, `ProcessedMachineIdentifier`,
+# `PlexOnlineToken`, `customConnections`, `allowedNetworks`, `TranscoderTempDirectory`) are all
+# unrelated. So there is no GitOps path to the toggle; an operator clicks it once in the web UI.
+# `runbooks/plex-quicksync.md` is that procedure, plus how to verify it took.
+hardware:
+  gpu: intel-quicksync
 exposure: public              # edge port-forward + public DNS; clients + Seerr reach it (docs/09; R-DNS/R-NET)
 egress: internet              # normal WAN egress (docs/03; R-NET-05)
 storage:

catalog/qbittorrent.yaml

@@ -448,6 +448,12 @@ auth:
     # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
     # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
     - platform-admins           # operators of these services must not be locked out of them
+  # NO `apiRoutes` HERE: UNMEASURED, not cleared. qBittorrent's WebUI drives everything through
+  # /api/v2/, so it is plausibly affected in the same way jackett is — but plausibility is what
+  # put a renderer constant on the table in the first place. MEASURED 2026-10-01 on the live pod
+  # (qbittorrent-5547bdb9bd-zq557, 0 restarts, 23h): 13,649 log lines, ZERO unauthenticated
+  # requests, nothing but health probes. No traffic means no evidence either way. Add `^/api/`
+  # when someone opens the WebUI and records the 302/401 split, as was done for jackett.
 update:
   channel: stable               # R-UPD-01
   range: ">=5.0.0 <6.0.0"       # stay on the qBittorrent 5.x stable line

catalog/radarr.yaml

@@ -112,6 +112,32 @@ auth:
     # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
     # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
     - platform-admins           # operators of these services must not be locked out of them
+  # Answer Radarr's machine endpoints with 401 instead of a 302 to Keycloak (schema 1.9.0;
+  # renders --api-route, singular and repeatable, onto the oauth2-proxy sidecar).
+  #
+  # `^/signalr/` — MEASURED ON THE LIVE POD, 2026-10-01, 23h after the cookie fix deployed
+  # (radarr-7d55cbffb9-42hmh, 0 restarts): 232 unauthenticated requests in the pod's whole
+  # lifetime and ALL 232 are the same call — `POST /signalr/messages/negotiate` from a browser
+  # UA, answered 302 by the proxy with upstream `-`, i.e. Radarr never saw one of them. That is
+  # a left-open tab whose session the cookie rename orphaned, polling SignalR forever, silently
+  # redirected every time and NEVER re-prompting the operator. It is the reported symptom
+  # ("reloading never fixes it and I am never re-prompted") still running in production on a
+  # service nobody was watching. With 401 the UI's connection breaks LOUDLY, the operator
+  # reloads, and `/` is outside both regexes so the reload signs them back in.
+  #
+  # `^/api/` — measured on radarr, but on the PRE-FIX pod and NOT by me: the 2026-09-30
+  # diagnosis (decisions/2026-09-30-jackett-oauth2-proxy-session.md) recorded radarr's 156
+  # unauthenticated requests as `/api/v3/command` (62), `/api/v3/queue/details` (46) and
+  # `/signalr/messages/negotiate` (43). The current pod has logged ZERO /api/ traffic, so I did
+  # not re-measure it; it is included because 108 of 156 measured rejections on this app were
+  # /api/v3 XHRs, which is this app's own evidence rather than an analogy to another app.
+  #
+  # SCOPE: both are JSON/RPC namespaces a browser never NAVIGATES to. Radarr's UI is served at
+  # `/` with assets under /Content/ and /MediaCover/, none of which match — so a plain page load
+  # is still 302'd into a silent SSO sign-in.
+  apiRoutes:
+    - ^/api/
+    - ^/signalr/
 update:
   channel: stable               # R-UPD-01
   range: ">=5.0.0 <6.0.0"       # stay on the Radarr v5 stable line

catalog/render.py

@@ -133,6 +133,87 @@ OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1"
 OAUTH2_PROXY_PORT = 4180
 KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
 
+# --- session lifetime -----------------------------------------------------------------
+# Keycloak's `homelab` realm issues access tokens with a 300-SECOND lifetime. MEASURED on a
+# live session, not read off a config page: `expires - created = 300.000s` exactly, with
+# `refresh_token:true` on the same line — so the proxy HOLDS a refresh token it was never
+# told to spend. `--cookie-refresh` defaults to 0, which the v7.8.1 binary's own help spells
+# "0 to disable", so every one of these sidecars dropped its session five minutes after
+# sign-in and sent the user back to Keycloak. Refresh at HALF the lifetime: two chances to
+# renew before expiry, so one failed refresh is not an immediate logout.
+#
+# NOTE for anyone tempted to add `--session-cookie-minimal` to shrink the cookie: it is
+# MUTUALLY EXCLUSIVE with this (provoked: "cookie_refresh > 0 requires oauth tokens in
+# sessions"). Dropping --pass-access-token is a least-privilege argument, not a size fix.
+OAUTH2_ACCESS_TOKEN_LIFETIME_S = 300
+OAUTH2_COOKIE_REFRESH = "2m30s"
+
+# --- metrics --------------------------------------------------------------------------
+# oauth2-proxy serves /metrics only when --metrics-address is given; with it absent the
+# listener never starts, so NO per-app sidecar was scrapable at all (measured: `ports` was
+# `['http']` on all six). 44180 is the platform's existing convention, not a guess — the
+# SHARED proxy already runs `--metrics-address=0.0.0.0:44180` with a `metrics` port and 2
+# ready endpoints. The flag is confirmed present in the PINNED v7.8.1 binary (`--help`),
+# which matters because the shared proxy is on v7.15.3 and could not have vouched for it.
+#
+# THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED. A container port is not a
+# Service port: these sidecars' Services expose the app port and :4180, and each namespace's
+# default-deny baseline has `ingress: []`, so Prometheus can neither resolve nor reach this
+# port until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor
+# are added. Do not read this flag as "metrics are working".
+OAUTH2_PROXY_METRICS_PORT = 44180
+
+# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------
+# The ServiceDefinition declares the CAPABILITY (schemas/servicedefinition.schema.json
+# `hardware.gpu`); everything below is the MECHANISM that currently satisfies it and is
+# deliberately confined to this file. This file is the ONLY place the mechanism appears,
+# which is what let it be swapped on 2026-10-01 -- from a `/dev/dri` hostPath to the Intel
+# GPU device plugin's extended resource -- with ZERO ServiceDefinition edits.
+#
+# The extended resource is advertised by the `intel-gpu-plugin` DaemonSet
+# (platform/intel-gpu-plugin/, pinned intel/intel-gpu-plugin:0.37.0). MEASURED on all three
+# prod nodes 2026-10-01: `.status.allocatable["gpu.intel.com/i915"] == 1`.
+QUICKSYNC_RESOURCE = "gpu.intel.com/i915"
+
+# --- API routes: 401 instead of 302, PER APP ------------------------------------------
+# oauth2-proxy answers an unauthenticated request with a 302 to Keycloak. Its built-in AJAX
+# guard would send a 401 instead, but it fires only on an EXACT `Accept: application/json`,
+# which these apps' JavaScript does not consistently send. So an unauthenticated XHR gets a
+# 302, the browser follows it INVISIBLY, the live Keycloak SSO session satisfies it silently,
+# and the call resolves 200 carrying the app's HTML where the JavaScript expected JSON — no
+# prompt, no console error, nothing logged as a failure. On a WRITE it is worse: a browser
+# following a 302 re-issues a POST as a GET and DROPS THE BODY, so the operator's change is
+# discarded and the UI reports success. MEASURED, twice:
+#   jackett  2026-09-30: 33 x 302 against 2 x 401 on /api/ paths, and 4 of 6 POSTs answered by
+#            the proxy and never reaching the app (access-log upstream field `-`, not `/`).
+#   radarr   2026-10-01, 23h after the cookie fix rolled: 232 x 302, EVERY ONE a
+#            `POST /signalr/messages/negotiate` from the operator's own browser, upstream `-`.
+#            A left-open tab whose cookie the rename orphaned, polling forever, silently
+#            redirected every time, NEVER re-prompting. That is the reported symptom
+#            ("reloading never fixes it and I am never re-prompted") still happening in
+#            production, on a service nobody was looking at.
+# `--api-route=<regex>` makes matching paths answer 401.
+#
+# WHY THIS IS NOT A RENDERER CONSTANT, AND MUST NEVER BECOME ONE. The path shape is a property
+# of the APP, not of the forward-auth pattern: radarr's broken path is `/signalr/`, jackett's is
+# `/api/`, and PIHOLE SERVES NO `/api/` AT ALL. One constant would apply one app's path shape to
+# five, and the failure mode of a wrong one is not "no improvement" but "this service is
+# permanently broken": a path a BROWSER NAVIGATES to, answered 401, shows a logged-out operator
+# a bare error page and never offers a sign-in. So the routes are declared per service in
+# `auth.apiRoutes` (schema 1.9.0) and rendered one `--api-route` per entry — the flag is
+# repeatable, and it is SINGULAR: `--api-routes` does not exist in v7.8.1 and the proxy refuses
+# to start with it (verified against the pinned binary's own --help, 2026-10-01:
+# `--api-route strings   return HTTP 401 instead of redirecting to authentication server...`).
+#
+# BREADTH RATCHET. Every entry must be anchored at `^/` and continue with literal path
+# characters. That rejects `^/`, `^/.*`, `.*`, an unanchored `/api/`, and any alternation — the
+# shapes that turn a page load into a 401. Alternation is refused rather than supported because
+# the flag is repeatable, so one regex per route costs nothing and stays readable. The same
+# pattern is in the schema (so ci/l0.sh's schema stage rejects it at the authoring surface) and
+# enforced HERE (so ci/render-golden.sh fails even if the schema stage is skipped) — a breadth
+# mistake must not need two independent gates to both be running.
+OAUTH2_API_ROUTE_PATTERN = re.compile(r"^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$")
+
 
 # =============================================================================
 # Defaults (renderer-contract.md "Default posture") — materialize BEFORE mapping.
@@ -158,6 +239,12 @@ def apply_defaults(sd: dict) -> dict:
     if "mode" not in auth:
         auth["mode"] = "forward-auth" if s["ingressRoutes"] else "none"
     auth.setdefault("groups", [])
+    # apiRoutes default posture is EMPTY, deliberately, and that is the whole point of the field
+    # (schema 1.9.0): an app gets --api-route only where someone MEASURED that it serves a machine
+    # API, so a newly-onboarded app renders no --api-route at all rather than inheriting another
+    # app's path shape. See OAUTH2_API_ROUTE_PATTERN above for why a default of `^/api/` would be
+    # wrong rather than merely generous.
+    auth.setdefault("apiRoutes", [])
     s["auth"] = auth
 
     update = dict(s.get("update") or {})
@@ -483,6 +570,64 @@ def r_deployment(s: dict) -> dict:
                 "persistentVolumeClaim": {"claimName": f"{s['name']}-{vol['name']}"},
             }
         )
+    # hardware capability (schema-optional): `hardware.gpu: intel-quicksync` means this
+    # workload needs the host iGPU's render node for hardware video transcode. The SD says
+    # only that; the wiring below is the platform's current mechanism for satisfying it.
+    #
+    # THE MECHANISM IS AN EXTENDED RESOURCE, AND THE hostPath IT REPLACED WAS MEASURED NOT
+    # TO WORK. Do NOT "simplify" this back into a `/dev/dri` hostPath volume. That was the
+    # original implementation (merged 2026-09-30) and it was proven broken in the live Plex
+    # pod on k3s-prod-2 on 2026-10-01: the device node was VISIBLE in the container
+    # (`crw-rw---- root video2 226,128 renderD128`) and could not be opened --
+    # `uid 0 O_RDWR -> EPERM` and `uid 1000 O_RDWR -> EPERM`. EPERM from the DEVICE CGROUP,
+    # not EACCES from file permissions, proven three ways: root on the HOST opens the same
+    # device rc=0; the node is unified `cgroup2fs`; and the container's runc `config.json`
+    # read `linux.resources.devices: [{allow: False, access: 'rwm'}]` with
+    # `linux.devices: None` -- one deny-all rule, no allow entry for major 226. A hostPath is
+    # a filesystem operation: it binds the device node in but adds NO cgroup allow rule,
+    # because Kubernetes cannot infer one from a volume.
+    #
+    # The device plugin is what produces the allow rule. Confirmed in the v0.37.0 source:
+    # `createDeviceSpecsFromDrmFiles` emits `DeviceSpec{..., Permissions: "rw"}`, which the
+    # kubelet turns into both the device injection and the matching cgroup allow.
+    #
+    # `limits`, NOT `requests`, AND THAT IS NOT A STYLE CHOICE: an extended resource must be
+    # expressed in `limits`. Kubernetes mirrors `requests` from `limits` automatically, and a
+    # requests-only extended resource is invalid. App container only, never a sidecar -- the
+    # oauth2-proxy sidecar does no transcoding.
+    #
+    # THE FAILURE MODE THIS TRADED INTO, STATED PLAINLY -- it is worse in blast radius, and
+    # unavoidable. BEFORE: a missing device meant `DirectoryOrCreate` mounted an empty dir,
+    # Plex ran and SILENTLY software-transcoded (invisible, but harmless to everything else).
+    # NOW: if the resource is unavailable (plugin broken, removed, or not yet reconciled) the
+    # pod is UNSCHEDULABLE, and because `apps-wave-0` is `wait: true` with `apps-wave-1`
+    # dependsOn it, that STALLS THE APP TIER. There is no graceful-degradation variant: an
+    # extended resource is a hard scheduling constraint, and it is also the only thing that
+    # produces the cgroup allow rule. Plex's schedulability is now coupled to the
+    # intel-gpu-plugin DaemonSet; that is the price of hardware transcoding working at all.
+    # ESCAPE HATCH (one field, no code change): delete the `hardware:` block from
+    # `catalog/plex.yaml` and re-render -- Plex returns to software transcoding and the tier
+    # unblocks. runbooks/plex-quicksync.md carries it.
+    #
+    # `podSecurityContext.supplementalGroups` is DELIBERATELY NOT SET here, and adding it
+    # would be a regression, not a hardening. MEASURED in the running container on prod-2
+    # (2026-09-30, re-measured 2026-10-01): the image's own cont-init hook
+    # `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` `find`s every char device
+    # under /dev/dri, `stat -c '%g'` each one's GID, `groupadd`s a matching group INSIDE the
+    # container when none exists, and `usermod -a -G` the app user into it -- every start,
+    # nothing hardcoded. It does exactly the same thing to the PLUGIN-INJECTED device, which
+    # is why this swap needs no group work either. cont-init.d completes before services.d,
+    # so the account database s6 reads at exec time already carries the group: measured live,
+    # `Plex Media Server` ran with `Groups: 44 100 992 1000`, where 992 is the host `render`
+    # GID. The service is started with `exec s6-setuidgid plex ...`, and `s6-setuidgid`
+    # REPLACES the process group list from /etc/group rather than appending to it, so
+    # anything Kubernetes puts in supplementalGroups reaches s6-svscan and is discarded
+    # before the app ever starts. A supplementalGroups entry here would be inert AND would
+    # mislead whoever debugs this next.
+    hw_gpu = (s.get("hardware") or {}).get("gpu")
+    if hw_gpu == "intel-quicksync":
+        container["resources"] = {"limits": {QUICKSYNC_RESOURCE: 1}}
+
     if volume_mounts:
         container["volumeMounts"] = volume_mounts
 
@@ -971,7 +1116,16 @@ def _oauth2_sidecar_container(s: dict) -> dict:
     auth.group (R-IDP-06: a valid session is not enough — the user must be in an authorized group,
     else /oauth2 returns 403). client-id/secret + cookie-secret come by name from the SOPS-encrypted
     `<app>-oauth2-proxy` Secret (R-IDP-04). Single provider => --skip-provider-button 302s straight
-    to Keycloak. The sidecar->app hop is pod-local (127.0.0.1) and not subject to network policy."""
+    to Keycloak. The sidecar->app hop is pod-local (127.0.0.1) and not subject to network policy.
+
+    --cookie-name and --cookie-domain are DERIVED per app (see the block comment on those two
+    args): every instance holds a different cookie-secret, so sharing one cookie name across the
+    zone makes the instances evict each other's sessions.
+
+    --api-route is DECLARED per app, from auth.apiRoutes, and is the one flag here that is
+    neither a constant nor derivable: it names the paths THIS app serves as a machine API, and
+    those differ per app (radarr `/signalr/`, jackett `/api/`, pihole none). An app that declares
+    none renders no --api-route at all."""
     route = s["ingressRoutes"][0]
     host = route["host"]
     upstream_port = route["port"]
@@ -991,11 +1145,85 @@ def _oauth2_sidecar_container(s: dict) -> dict:
     args += [
         "--skip-provider-button=true",
         f"--redirect-url=https://{host}/oauth2/callback",
-        f"--cookie-domain=.{ZONE}",
+    ]
+    # ── API ROUTES: 401 instead of 302, ON THIS APP'S OWN PATHS ONLY ───────────────────────
+    # One flag per declared route, in declared order. An app that declares NONE renders NO
+    # --api-route, which is the correct and common case — see OAUTH2_API_ROUTE_PATTERN above for
+    # the measurements, for why this cannot be a renderer constant, and for the breadth ratchet.
+    # Validated here as well as in the schema so a breadth mistake is caught by render-golden
+    # even on a run where the schema stage did not execute.
+    for route in s["auth"].get("apiRoutes", []):
+        if not OAUTH2_API_ROUTE_PATTERN.match(str(route)):
+            raise SystemExit(
+                f"render.py: {s['name']}: auth.apiRoutes entry {route!r} is not an anchored path "
+                f"regex. Each entry must start with '^/' and continue with literal path "
+                f"characters (e.g. '^/api/', '^/signalr/', '^/api/v3/.*'). Rejected shapes: '^/' "
+                f"and '^/.*' (match every page load), an unanchored '/api/' (matches mid-path), "
+                f"and alternation (the flag is repeatable — declare one entry per route). A "
+                f"too-broad --api-route answers a PAGE LOAD with 401, so a logged-out operator "
+                f"sees a bare error page and is never offered a sign-in: the service looks "
+                f"permanently broken rather than merely unauthenticated."
+            )
+        args.append(f"--api-route={route}")
+    args += [
+        # ── COOKIE ISOLATION (fleet fix 2026-09-30; crafty's 2026-09-29 fix, generalized) ──────
+        # EVERY oauth2-proxy on this platform — this sidecar, its four siblings, and the SHARED
+        # forward-auth proxy (platform/oauth2-proxy) — is an INDEPENDENT instance holding its OWN
+        # `cookie-secret`. Measured on prod 2026-09-30: seven `*-oauth2-proxy` Secrets, seven
+        # DISTINCT cookie-secret values, so no instance can decrypt any other's cookie.
+        #
+        # Left at oauth2-proxy's DEFAULTS (`_oauth2_proxy` / `_oauth2_proxy_csrf`) and scoped to
+        # `.${BASE_DOMAIN}`, the browser therefore keeps ONE cookie pair for ALL of them:
+        # whichever proxy authenticates last OVERWRITES every other's session. The loser logs
+        # "No valid authentication in request" and bounces to Keycloak, which — the SSO session
+        # still being live — redirects straight back and overwrites the cookie again. The
+        # user-visible symptom is "I signed into Keycloak but I'm still not signed in", and it
+        # sends you hunting in Keycloak, where nothing is wrong. Silent, intermittent, and
+        # misattributed: the exact shape this repo keeps getting burned by.
+        #
+        # WHY IT WAS *INTERMITTENT*, AND WHY RELOADING NEVER HELPED. The loser does not merely
+        # fail to read the cookie — it calls ClearSession, which EXPIRES the cookie on its own
+        # --cookie-domain, i.e. the whole zone. And the *arr UIs poll: measured on prod, radarr's
+        # rejections are `/api/v3/command`, `/api/v3/queue/details` and `/signalr/negotiate`,
+        # 155 of them in 12h against 1 successful auth. Every poll carrying a cookie radarr
+        # cannot decrypt calls ClearSession again — so A LEFT-OPEN RADARR TAB DELETES JACKETT'S
+        # COOKIE EVERY FEW SECONDS, from another namespace, with no user action at all. That is
+        # the intermittency, and it is why signing in again fixed nothing for longer than the
+        # gap between two polls. Host-scoping the cookie (below) is what contains the blast
+        # radius of a ClearSession to the app that issued it.
+        #
+        # DERIVED, NOT LISTED. Both flags come from the ServiceDefinition (`s['name']` and the
+        # route host), so a NEW forward-auth app is isolated the moment it is onboarded. A
+        # per-service table would leave the next app defaulting into the collision, which is how
+        # twelve ServiceMonitors acquired one identical wrong port name in a single edit.
+        # conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct asserts both properties
+        # over the WHOLE tree and asserts THIS function derives rather than lists.
+        #
+        #   --cookie-name   a unique name per instance, so two sessions cannot alias at all.
+        #                   This is the fix; the one below is defence in depth.
+        #   --cookie-domain HOST-scoped. A per-app reverse proxy has no reason to broadcast its
+        #                   session cookie to every other host in the zone — a zone-wide cookie
+        #                   is transmitted to every sibling app on `.${BASE_DOMAIN}`, so any
+        #                   FUTURE name collision would again be platform-wide instead of
+        #                   contained. The SHARED forward-auth proxy keeps the wide scope,
+        #                   because cross-host forward-auth is precisely its job.
+        #
+        # --whitelist-domain deliberately stays ZONE-WIDE: it governs permitted post-login
+        # REDIRECT targets, not cookie scope. Narrowing it here would break the sign-in bounce.
+        f"--cookie-name=_oauth2_proxy_{s['name']}",
+        f"--cookie-domain={host}",
         f"--whitelist-domain=.{ZONE}",
         "--cookie-secure=true",
+        # Spend the refresh token the proxy already holds, at half the 300s access-token
+        # lifetime. Without this the session dies 5 minutes after sign-in. See
+        # OAUTH2_ACCESS_TOKEN_LIFETIME_S above for the measurement and the
+        # --session-cookie-minimal incompatibility.
+        f"--cookie-refresh={OAUTH2_COOKIE_REFRESH}",
         "--pass-access-token=true",
         "--set-xauthrequest=true",
+        # Start the /metrics listener. Prerequisite only — not yet reachable by Prometheus;
+        # see OAUTH2_PROXY_METRICS_PORT above for exactly what is still missing.
+        f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}",
     ]
     secret = f"{s['name']}-oauth2-proxy"
     return {
@@ -1017,7 +1245,10 @@ def _oauth2_sidecar_container(s: dict) -> dict:
                 "valueFrom": {"secretKeyRef": {"name": secret, "key": "cookie-secret"}},
             },
         ],
-        "ports": [{"name": "http", "containerPort": OAUTH2_PROXY_PORT, "protocol": "TCP"}],
+        "ports": [
+            {"name": "http", "containerPort": OAUTH2_PROXY_PORT, "protocol": "TCP"},
+            {"name": "metrics", "containerPort": OAUTH2_PROXY_METRICS_PORT, "protocol": "TCP"},
+        ],
         "resources": {
             "requests": {"cpu": "25m", "memory": "64Mi"},
             "limits": {"memory": "128Mi"},
@@ -2440,6 +2671,128 @@ def _selftest() -> int:
         "an SD without podSecurityContext must not gain a pod securityContext",
     )
 
+    # 6b) hardware.gpu: intel-quicksync -> a `gpu.intel.com/i915` LIMIT on the app container
+    #     and NOTHING ELSE: no volume, no volumeMount, no securityContext, no nodeSelector.
+    #     The SD declares the CAPABILITY; these assertions pin the MECHANISM that currently
+    #     satisfies it -- including what it deliberately does NOT emit -- so a change to it
+    #     is a deliberate, visible edit. The previous mechanism (a /dev/dri hostPath) is
+    #     asserted ABSENT: it was measured not to work (EPERM from the device cgroup; see
+    #     the emission site), and the absence assertion is what stops it coming back.
+    hw_docs = render(
+        {
+            "name": "hw",
+            "source": {"image": "a:1"},
+            "hardware": {"gpu": "intel-quicksync"},
+        }
+    )
+    hw_pod = next(
+        d[1] for d in hw_docs if d[1]["kind"] == "Deployment"
+    )["spec"]["template"]["spec"]
+    hw_app = hw_pod["containers"][0]
+    # The whole mechanism, exactly: an extended resource in `limits`. `requests` is NOT set
+    # by the renderer -- Kubernetes mirrors it from `limits`, and a requests-only extended
+    # resource is invalid. Asserting the exact dict is deliberate: it fails if anyone adds a
+    # CPU/memory resources feature through this field (that would be scope creep, and this
+    # field is about the DEVICE only).
+    check(
+        hw_app.get("resources") == {"limits": {"gpu.intel.com/i915": 1}},
+        "hardware.gpu=intel-quicksync must render resources.limits "
+        "{gpu.intel.com/i915: 1} on the app container, and nothing else in resources",
+    )
+    # The retired mechanism must stay retired. A hostPath binds the device node in but adds
+    # no device-cgroup allow rule, so the open() was EPERM with the node plainly visible.
+    check(
+        not any(
+            "hostPath" in v or v.get("name") == "dri" for v in hw_pod.get("volumes", [])
+        )
+        and not any(
+            m.get("mountPath") == "/dev/dri" or m.get("name") == "dri"
+            for m in hw_app.get("volumeMounts", [])
+        ),
+        "hardware.gpu=intel-quicksync must render NO /dev/dri hostPath volume and no "
+        "/dev/dri mount (the hostPath mechanism was measured not to work: device-cgroup "
+        "EPERM; the device plugin's DeviceSpec Permissions: rw is what grants access)",
+    )
+    # The resource is the WHOLE feature: no pod securityContext is invented. The image's own
+    # cont-init hook adds the app user to the (plugin-injected) device's group inside the
+    # container, and s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway
+    # (see the emission site). Asserting the absence is what stops someone adding an inert
+    # one back.
+    check(
+        "securityContext" not in hw_pod,
+        "hardware.gpu must NOT invent a pod securityContext (supplementalGroups is inert "
+        "under s6-setuidgid; the image's cont-init hook handles the device group)",
+    )
+    # Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce)
+    # rejects `privileged: true` in an app namespace, and placement is a separate concern
+    # with its own justification -- this field is about the DEVICE only. (The extended
+    # resource now constrains placement to nodes the plugin advertises, which is the
+    # scheduler's own doing, not a nodeSelector this renderer writes.)
+    check(
+        "privileged" not in hw_pod.get("securityContext", {})
+        and "securityContext" not in hw_app
+        and "nodeSelector" not in hw_pod,
+        "hardware.gpu must emit no privileged securityContext and no nodeSelector",
+    )
+    # The regression that matters: absence renders NOTHING. Mirrors the podSecurityContext
+    # negative above -- an SD that does not opt in must be byte-identical to before.
+    nohw_pod = next(
+        d[1] for d in render({"name": "nohw", "source": {"image": "a:1"}})
+        if d[1]["kind"] == "Deployment"
+    )["spec"]["template"]["spec"]
+    check(
+        "volumes" not in nohw_pod
+        and "volumeMounts" not in nohw_pod["containers"][0]
+        and "resources" not in nohw_pod["containers"][0]
+        and "securityContext" not in nohw_pod,
+        "an SD without `hardware` must gain no resources block, no volume, no mount and no "
+        "securityContext",
+    )
+    # An SD that declares BOTH keeps its own podSecurityContext exactly as written -- the
+    # hardware field neither adds to it nor rewrites it.
+    hwm_pod = next(
+        d[1]
+        for d in render(
+            {
+                "name": "hwm",
+                "source": {"image": "a:1"},
+                "hardware": {"gpu": "intel-quicksync"},
+                "podSecurityContext": {"fsGroup": 1001, "supplementalGroups": [44]},
+            }
+        )
+        if d[1]["kind"] == "Deployment"
+    )["spec"]["template"]["spec"]
+    check(
+        hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]}
+        and hwm_pod["containers"][0].get("resources")
+        == {"limits": {"gpu.intel.com/i915": 1}},
+        "hardware.gpu must leave the SD's own podSecurityContext untouched",
+    )
+    # The resource goes on the APP container only. A forward-auth app renders an oauth2-proxy
+    # sidecar, which does no transcoding and must not consume one of the node's i915 slots.
+    hws_containers = next(
+        d[1]
+        for d in render(
+            {
+                "name": "hws",
+                "source": {"image": "a:1"},
+                "hardware": {"gpu": "intel-quicksync"},
+                "ingressRoutes": [{"host": "hws.home.keithjackson.dev", "port": 8080}],
+                "auth": {"mode": "forward-auth", "groups": ["media-users"]},
+            }
+        )
+        if d[1]["kind"] == "Deployment"
+    )["spec"]["template"]["spec"]["containers"]
+    check(
+        len(hws_containers) == 2
+        and hws_containers[0].get("resources")
+        == {"limits": {"gpu.intel.com/i915": 1}}
+        and "gpu.intel.com/i915"
+        not in hws_containers[1].get("resources", {}).get("limits", {}),
+        "hardware.gpu must put the i915 limit on the APP container only, never on the "
+        "oauth2-proxy sidecar",
+    )
+
     # 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that
     #     fronts the Gateway path (fail-closed), while oidc/native/none apps are unchanged.
     fa_docs = render(
@@ -2450,15 +2803,27 @@ def _selftest() -> int:
             "auth": {"mode": "forward-auth", "groups": ["media-users"]},
         }
     )
-    fa_dep = next(d[1] for d in fa_docs if d[1]["kind"] == "Deployment")
-    fa_containers = fa_dep["spec"]["template"]["spec"]["containers"]
+    fa_dep = next((d[1] for d in fa_docs if d[1]["kind"] == "Deployment"), None)
+    check(fa_dep is not None,
+          "a forward-auth app rendered NO Deployment, so there is no pod to gate at all")
+    fa_containers = (((fa_dep or {}).get("spec") or {}).get("template") or {}) \
+        .get("spec", {}).get("containers") or []
     check(
         len(fa_containers) == 2
         and fa_containers[0]["name"] == "fa"
         and fa_containers[1]["name"] == "oauth2-proxy",
-        "forward-auth app must render the app container FIRST + an oauth2-proxy sidecar SECOND",
+        f"forward-auth app must render the app container FIRST + an oauth2-proxy sidecar SECOND; "
+        f"rendered {[c.get('name') for c in fa_containers]!r}",
     )
-    side = fa_containers[1]
+    # EVERY assertion below reads `side`, so an absent sidecar must NOT be allowed to raise here.
+    # The whole point of the None-tolerant lookups in this block is that a missing sidecar (or a
+    # missing flag on it) reports as NAMED ASSERTIONS rather than as a traceback: an exception
+    # escaping _selftest() exits non-zero with the right verdict and ZERO diagnostics, and it
+    # hides every other assertion the same mutation broke. Measured by Desmond Oyelaran while
+    # wiring this selftest into the render-golden stage: deleting the --cookie-name emission gave
+    # rc 1 via an uncaught StopIteration and printed not one `SELFTEST FAIL:` line. Provoked
+    # again here with the sidecar removed entirely, which raised IndexError at this line.
+    side = fa_containers[1] if len(fa_containers) > 1 else {"args": [], "env": [], "ports": []}
     check(
         "--upstream=http://127.0.0.1:9117" in side["args"],
         "sidecar upstream must be the co-located app UI over http on 127.0.0.1:<uiport>",
@@ -2475,12 +2840,212 @@ def _selftest() -> int:
         not any(a.startswith("--ssl-upstream-insecure-skip-verify") for a in side["args"]),
         "http upstream must NOT skip TLS verify (that flag is only for crafty's self-signed https)",
     )
+    # COOKIE ISOLATION. Every oauth2-proxy instance holds its own cookie-secret, so two instances
+    # sharing a cookie NAME on a shared DOMAIN evict each other's sessions silently. Both flags
+    # must be DERIVED from this service, not defaulted and not listed.
     check(
-        side["env"][0]["valueFrom"]["secretKeyRef"]["name"] == "fa-oauth2-proxy",
+        "--cookie-name=_oauth2_proxy_fa" in side["args"],
+        "sidecar must declare a cookie name DERIVED from the service name — the oauth2-proxy "
+        "default (_oauth2_proxy) is shared with every other instance on the zone",
+    )
+    check(
+        "--cookie-domain=fa.home.keithjackson.dev" in side["args"],
+        "sidecar cookie-domain must be HOST-scoped (the app's own host), not zone-wide",
+    )
+    check(
+        f"--whitelist-domain=.{ZONE}" in side["args"],
+        "whitelist-domain must stay ZONE-wide — it governs post-login redirect targets, not "
+        "cookie scope; narrowing it breaks the sign-in bounce",
+    )
+    # SESSION LIFETIME. `--cookie-refresh` must be present AND strictly shorter than the
+    # measured 300s access-token lifetime, or the session dies mid-use. Asserted as a parsed
+    # duration rather than a string match, so "2m30s" -> "150s" -> "2.5m" stays green and a
+    # silent widening past the token lifetime does not.
+    refresh = next((a.split("=", 1)[1] for a in side["args"]
+                    if a.startswith("--cookie-refresh=")), None)
+    check(refresh is not None,
+          "sidecar must set --cookie-refresh; the v7.8.1 default is 0 = DISABLED, so the "
+          "session dies when the access token expires and the user is bounced to Keycloak")
+    if refresh is not None:
+        _m = re.fullmatch(r"(?:(\d+)m)?(?:(\d+(?:\.\d+)?)s)?", refresh)
+        _secs = (int(_m.group(1) or 0) * 60 + float(_m.group(2) or 0)) if _m else -1
+        check(0 < _secs < OAUTH2_ACCESS_TOKEN_LIFETIME_S,
+              f"--cookie-refresh={refresh} parses to {_secs}s, which is not inside "
+              f"(0, {OAUTH2_ACCESS_TOKEN_LIFETIME_S}s). A refresh at or past the access-token "
+              f"lifetime never renews in time, which is the same logout it was added to stop")
+    check(
+        not any(a.startswith("--session-cookie-minimal") for a in side["args"]),
+        "--session-cookie-minimal is MUTUALLY EXCLUSIVE with --cookie-refresh "
+        "('cookie_refresh > 0 requires oauth tokens in sessions') — the proxy refuses to start",
+    )
+    # METRICS. The listener does not exist unless the flag is given, and the port must be
+    # DECLARED or nothing can ever select it (neither a Service port nor a PodMonitor).
+    check(
+        f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}" in side["args"],
+        "sidecar must start the /metrics listener; without --metrics-address it is not merely "
+        "unscraped, it is not serving at all",
+    )
+    check(
+        [p["name"] for p in side["ports"]] == ["http", "metrics"]
+        and {p["containerPort"] for p in side["ports"]}
+        == {OAUTH2_PROXY_PORT, OAUTH2_PROXY_METRICS_PORT},
+        "sidecar must declare BOTH the http and metrics container ports — an undeclared port "
+        "cannot be named by a Service port or a PodMonitor, so the flag alone is inert",
+    )
+    # API ROUTES (schema 1.9.0). Three properties, and the FIRST is the one that keeps the field
+    # honest: an app that declares nothing must render NO --api-route. `fa` above declares none.
+    check(
+        not any(a.startswith("--api-route") for a in side["args"]),
+        "a forward-auth app that declares no auth.apiRoutes must render NO --api-route. A "
+        "default or a renderer constant would answer PiHole's page loads with 401 — PiHole "
+        "serves no /api/ at all — so the service would look permanently broken to a "
+        "logged-out operator instead of signing them in",
+    )
+    # Same `next(..., None)` discipline as _cookie_name() below, for the same reason: if the
+    # sidecar stops being rendered at all, this must report as a named assertion rather than as a
+    # StopIteration that swallows every other SELFTEST FAIL line in the run.
+    _faapi_dep = next(
+        (
+            d[1]
+            for d in render(
+                {
+                    "name": "faapi",
+                    "source": {"image": "a:1"},
+                    "ingressRoutes": [{"host": "faapi.home.keithjackson.dev", "port": 9119}],
+                    "auth": {
+                        "mode": "forward-auth",
+                        "groups": ["media-users"],
+                        "apiRoutes": ["^/api/", "^/signalr/"],
+                    },
+                }
+            )
+            if d[1]["kind"] == "Deployment"
+        ),
+        None,
+    )
+    fa_api_side = next(
+        (
+            c
+            for c in ((_faapi_dep or {}).get("spec", {}).get("template", {})
+                      .get("spec", {}).get("containers") or [])
+            if c.get("name") == "oauth2-proxy"
+        ),
+        None,
+    )
+    check(
+        fa_api_side is not None,
+        "a forward-auth app declaring auth.apiRoutes rendered NO oauth2-proxy sidecar, so the "
+        "routes have nowhere to land and the app is not gated at all (R-IDP-02)",
+    )
+    check(
+        fa_api_side is not None
+        and [a for a in fa_api_side["args"] if a.startswith("--api-route")]
+        == ["--api-route=^/api/", "--api-route=^/signalr/"],
+        "a declared auth.apiRoutes must render as one repeatable --api-route per entry, in "
+        "declared order, and NOTHING else. Note the flag is SINGULAR: --api-routes does not "
+        "exist in v7.8.1 and the proxy refuses to start with it",
+    )
+    # BREADTH. The renderer must REFUSE a regex that would match a page load, not merely
+    # discourage one, because the schema stage can be skipped and render-golden cannot.
+    for _bad in ("^/", "^/.*", ".*", "/api/", "^/api/|^/"):
+        try:
+            render(
+                {
+                    "name": "fabad",
+                    "source": {"image": "a:1"},
+                    "ingressRoutes": [{"host": "fabad.home.keithjackson.dev", "port": 9120}],
+                    "auth": {
+                        "mode": "forward-auth",
+                        "groups": ["media-users"],
+                        "apiRoutes": [_bad],
+                    },
+                }
+            )
+            _refused = False
+        except SystemExit:
+            _refused = True
+        check(
+            _refused,
+            f"the renderer ACCEPTED auth.apiRoutes entry {_bad!r}. A too-broad --api-route "
+            f"answers a plain page load with 401, so a logged-out operator sees a bare error "
+            f"page and is never offered a sign-in — the service looks permanently broken "
+            f"rather than merely unauthenticated, which is a worse failure than the 302 bug "
+            f"this flag exists to fix",
+        )
+    for _good in ("^/api/", "^/signalr/", "^/api/v3/.*", "^/api/v2.0/server/update$"):
+        check(
+            bool(OAUTH2_API_ROUTE_PATTERN.match(_good)),
+            f"the breadth ratchet rejected the legitimate route {_good!r}. A ratchet that "
+            f"blocks real API prefixes is a ratchet someone eventually deletes",
+        )
+
+    # Two forward-auth apps must not be able to collide. Renders a SECOND one and compares, so a
+    # future refactor to a constant (or to a per-service table with a copy-paste slip) fails here
+    # rather than silently in a browser six weeks later.
+    # Same None-tolerant shape as the `faapi` fixture above and for the same reason: a missing
+    # sidecar must surface as a named assertion, never as a traceback that suppresses the
+    # collected SELFTEST FAIL lines.
+    _fa2_dep = next(
+        (
+            d[1]
+            for d in render(
+                {
+                    "name": "fa2",
+                    "source": {"image": "a:1"},
+                    "ingressRoutes": [{"host": "fa2.home.keithjackson.dev", "port": 9118}],
+                    "auth": {"mode": "forward-auth", "groups": ["media-users"]},
+                }
+            )
+            if d[1]["kind"] == "Deployment"
+        ),
+        None,
+    )
+    fa2_side = next(
+        (
+            c
+            for c in ((_fa2_dep or {}).get("spec", {}).get("template", {})
+                      .get("spec", {}).get("containers") or [])
+            if c.get("name") == "oauth2-proxy"
+        ),
+        {"args": []},
+    )
+
+    # `next(..., None)` — the DEFAULT IS LOAD-BEARING, not defensive style. With a bare
+    # `next(genexpr)` this raises StopIteration the moment a sidecar renders no --cookie-name,
+    # which is precisely the mutation this assertion exists to catch. The traceback then escapes
+    # _selftest() before its collected `SELFTEST FAIL:` lines are printed, so the run exits
+    # non-zero with the RIGHT VERDICT AND NO DIAGNOSTICS — and it masks every other assertion the
+    # same mutation broke. Measured by Desmond Oyelaran while wiring the selftest into the
+    # render-golden stage: deleting the --cookie-name emission gave rc 1 via an uncaught
+    # StopIteration and suppressed all of the messages. A missing flag must report AS AN
+    # ASSERTION, with its name in the output, which is what the None + explicit check below do.
+    def _cookie_name(container):
+        return next((a for a in container["args"] if a.startswith("--cookie-name=")), None)
+
+    _fa_cn, _fa2_cn = _cookie_name(side), _cookie_name(fa2_side)
+    check(
+        _fa_cn is not None and _fa2_cn is not None,
+        f"a forward-auth sidecar rendered NO --cookie-name at all (fa={_fa_cn!r}, "
+        f"fa2={_fa2_cn!r}). Omitting the flag is not neutral: oauth2-proxy then claims its "
+        f"built-in default `_oauth2_proxy`, which every other instance on the platform also "
+        f"claims, so they overwrite and ClearSession-delete each other's sessions",
+    )
+    check(
+        _fa_cn != _fa2_cn,
+        "two forward-auth apps rendered IDENTICAL cookie names — they would overwrite each "
+        "other's session on every sign-in (the 2026-09-29 crafty defect, fleet-wide)",
+    )
+    # Index-safe for the same reason as `side` above: an absent sidecar has no env, and an
+    # IndexError here would exit non-zero with no diagnostics at all.
+    _fa_env = (side.get("env") or [{}])[0]
+    check(
+        (((_fa_env.get("valueFrom") or {}).get("secretKeyRef") or {}).get("name")
+         == "fa-oauth2-proxy"),
         "sidecar creds must come by name from the per-app <app>-oauth2-proxy Secret (R-IDP-04)",
     )
-    fa_svc = next(d[1] for d in fa_docs if d[1]["kind"] == "Service")
-    fa_svc_ports = {p["port"] for p in fa_svc["spec"]["ports"]}
+    fa_svc = next((d[1] for d in fa_docs if d[1]["kind"] == "Service"), None)
+    check(fa_svc is not None, "a forward-auth app must render a Service for the Gateway to reach")
+    fa_svc_ports = {p["port"] for p in (((fa_svc or {}).get("spec") or {}).get("ports") or [])}
     check(
         fa_svc_ports == {9117, 4180},
         "forward-auth Service must expose the app port (east-west) AND :4180 (gateway->sidecar)",

catalog/sonarr.yaml

@@ -112,6 +112,14 @@ auth:
     # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
     # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
     - platform-admins           # operators of these services must not be locked out of them
+  # NO `apiRoutes` HERE, AND NOT BECAUSE IT NEEDS NONE — because nothing measures that it does.
+  # Sonarr is nearly the same codebase as Radarr, which is exactly what makes `^/api/` +
+  # `^/signalr/` a tempting guess and still a guess. MEASURED 2026-10-01 on the live pod
+  # (sonarr-ddf485775-8ckwt, 0 restarts, 23h): its oauth2-proxy log contains 13,643 lines and
+  # ZERO unauthenticated requests — no /api/, no /signalr/, nothing but health probes. Nobody
+  # has opened this UI since the cookie fix rolled, so the measurement is empty rather than
+  # negative: it is evidence of no TRAFFIC, not evidence of no need. Add the routes when someone
+  # reproduces the 302-on-XHR here, with the numbers.
 update:
   channel: stable               # R-UPD-01
   range: ">=4.0.0 <5.0.0"       # stay on the Sonarr v4 stable line

ci/render-golden.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
-# ci/render-golden.sh — Stage: render==golden reproducibility gate.
+# ci/render-golden.sh — Stage: the renderer is trustworthy.
+#   (1) catalog/render.py passes its OWN output-contract self-test, and
+#   (2) every committed apps/ golden reproduces byte-for-byte from the canonical render.
+# Two independent assertions about one subject, catalog/render.py. (1) was added
+# 2026-10-01 because nothing in ci/ or .gitea/ had ever invoked `--selftest`; see the
+# block above the invocation for why it lives here and not in a conformance check.
 #
 # WHY: the committed apps/ tree is a set of GOLDENS produced by ONE canonical
 # invocation of catalog/render.py. If a golden is hand-edited, or a catalog
@@ -54,6 +59,54 @@ if [ ! -f "$RENDER_PY" ]; then
   summary "- render-golden: **SKIPPED** (renderer missing)"
   exit 3
 fi
+# --- The renderer's OWN self-test, before anything is compared ---------------
+# `catalog/render.py --selftest` is ~106 executed assertions on the renderer's output
+# contract (default-deny covers both directions, the Kyverno namespace hook label, a valid
+# auth-mode, the oauth2-proxy sidecar's cookie-name/-domain/-refresh flags, the
+# placeholder-substitution guard, the observability refusals...). Until now NOTHING in ci/
+# or .gitea/ invoked it — measured by grep on 2026-09-30 and recorded as the top follow-up
+# of decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md — so every one of those
+# assertions was dev-time only and a merge gate could not tell a sound renderer from a
+# gutted one.
+#
+# WHY HERE, AND NOT IN A CONFORMANCE CHECK. The lead was
+# conformance/tests/l0-static/auditability-proofs-producible/check.sh, which drives
+# `conformance/lib/{record_run,status}.py --selftest` "because nothing else drives them".
+# That loop is NOT the right hook: that check serves R-SEC-03 and its subject is the
+# EVIDENCE CHAIN (can a forged or stale run become a proof?). The renderer is not part of
+# that chain, and hanging it there would make the check's name and its requirement mapping
+# a lie. This stage's subject already IS catalog/render.py — it exists to assert that the
+# renderer's output is trustworthy — so the renderer's own contract test belongs in it.
+#
+# It runs BEFORE the catalog/empty-catalog exit-3 below on purpose: the self-test is
+# deterministic and catalog-independent (it renders synthetic ServiceDefinitions in
+# memory), so an empty catalog must not be able to skip it.
+#
+# render-golden and the self-test are INDEPENDENT assertions, which is why both are needed:
+# render-golden catches golden-vs-renderer drift but is satisfied by ANY renderer whose
+# output was re-rendered, including one that just lost a contract invariant and had all ten
+# goldens regenerated in the same commit. The self-test catches exactly that.
+#
+# FAILURE IS A HARD FAIL, NOT A SKIP: the renderer is source in this repo, so "could not
+# run its self-test" has no legitimate environmental cause once python3 and render.py are
+# both present (both asserted above). ci/gate.sh maps this stage's rc 1 to BLOCKED.
+log "Stage: catalog/render.py --selftest (the renderer's own output-contract assertions)"
+selftest_out="$(mktemp)"
+if "$PYTHON" "$RENDER_PY" --selftest >"$selftest_out" 2>&1; then
+  ok "render.py --selftest passed"
+  summary "- render-selftest: **passed** (\`catalog/render.py --selftest\`)"
+else
+  err "catalog/render.py --selftest FAILED — the renderer violates its own output contract."
+  err "  Every golden under apps/ is produced by this renderer, so the goldens below cannot"
+  err "  be trusted even if they reproduce byte-for-byte. Reproduce with:"
+  err "    ${PYTHON} catalog/render.py --selftest"
+  sed 's/^/    /' "$selftest_out" >&2 || true
+  summary "- render-selftest: **FAILED** (\`catalog/render.py --selftest\` — renderer contract violated)"
+  rm -f "$selftest_out"
+  exit 1
+fi
+rm -f "$selftest_out"
+
 if [ ! -d "$CATALOG_DIR" ] || ! compgen -G "${CATALOG_DIR}/*.yaml" >/dev/null 2>&1; then
   skip "no catalog ServiceDefinitions under '${CATALOG_DIR}' — nothing to render."
   summary "- render-golden: **SKIPPED** (empty catalog)"

clusters/base/intel-gpu-plugin.yaml

@@ -0,0 +1,104 @@
+---
+# Flux Kustomization: Intel GPU device plugin — reconciles ./platform/intel-gpu-plugin.
+# Makes the three prod nodes advertise `gpu.intel.com/i915`, which is the resource a later PR will
+# have `hardware.gpu: intel-quicksync` request in place of today's non-working /dev/dri hostPath.
+#
+# WHY ITS OWN KUSTOMIZATION, AND NOT ONE LINE IN platform/kustomization.yaml
+#
+# The cheap option was a line in the shared `platform` tier plus a new platform/ directory, and
+# neither path is sensitive in infra/gitea/approval-policy.json, so it would have needed no human
+# reviewer on the policy's terms. It is still the wrong place, for the reason the repo already
+# wrote down in platform/kustomization.yaml when Cilium was split out of that tier:
+#
+#   * `platform` is `wait: true` over eleven-plus components, so its Ready condition cannot tell
+#     you "the GPU plugin is wedged" from "longhorn is slow".
+#   * The only emergency valve would be `flux suspend kustomization platform` — freezing storage,
+#     TLS, DNS, ingress, monitoring, Keycloak and KubeVirt in order to stop touching a device
+#     plugin.
+#   * The tier's 10m interval / 5m timeout are explicitly called out there as WRONG for a 3-node
+#     DaemonSet rollout. This is a 3-node DaemonSet rollout.
+#
+# And the consequence is worse here than for an average component, because of what sits downstream:
+# `platform` gates platform-config, which gates apps-wave-0 (wait:true), which gates apps-wave-1.
+# A plugin DaemonSet that never went healthy inside that tier would stall every app on "dependency
+# not ready" — i.e. a component added to fix a SILENT NO-OP (software transcoding) would have
+# acquired the ability to cause a REAL OUTAGE. That trade is the whole argument.
+#
+# The usual objection to this placement is the approval cost, and it does not apply: Gitea enforces
+# `required_approvals: 1` with `block_admin_merge_override: true` on EVERY PR here regardless of
+# path, and authors cannot self-approve. So this change needed the operator's approval either way;
+# touching clusters/** costs nothing extra and buys a blast radius of exactly one component.
+#
+# dependsOn: [cilium], WHICH IS THE MINIMUM AND ALSO THE EARLIEST
+#
+# Not platform-config (what headlamp/crafty/homepage/oauth2-proxy use) — those are leaf UIs that
+# genuinely need Keycloak + wildcard TLS + split-horizon DNS + Gateway. This plugin needs none of
+# it. It never uses the network and never talks to the API server; it talks to the local kubelet
+# over a unix socket. Its only real prerequisite is a working CNI, so its pod can get a sandbox at
+# all, and `cilium` already depends on `flux-system` (the root Kustomization that applies the
+# cluster-vars ConfigMap this one substitutes from), so that edge is transitively satisfied too.
+#
+# Depending on `cilium` rather than `platform` also gets the resource advertised EARLIEST — this
+# reconciles in parallel with the `platform` tier instead of behind all eleven of its components.
+# That matters on a cold start once the consumer PR lands: a Plex pod requesting a resource no node
+# advertises is Unschedulable, and apps-wave-0 is wait:true. Reconciling early shrinks that window
+# instead of widening it. It is not a GUARANTEE — see the ordering note below, which is a real
+# constraint on the next PR and not a formality.
+#
+# Kyverno (in `platform`) is deliberately NOT a dependency: every policy that could match this
+# workload is scoped to namespaces labelled app-namespace=true, and this namespace is labelled
+# "false" (platform/intel-gpu-plugin/namespace.yaml). So there is no admission race to lose.
+#
+# wait: true — A HEALTH SIGNAL, NOT A GATE. This is the one place this component departs from the
+# headlamp/crafty leaf template, and the departure is deliberate. Those run wait:false because a
+# leaf UI can be legitimately not-Ready for a long time during a cold start while it waits on
+# Keycloak/DNS/TLS, so its health is not a useful signal. This DaemonSet has no such dependency: it
+# reads sysfs and opens a socket, so on healthy nodes it should be Ready in seconds, and
+# not-Ready genuinely means BROKEN. wait:true turns `flux get kustomization intel-gpu-plugin` into
+# a one-line answer to "did the plugin come up on all three nodes", which is exactly the question
+# the operator must answer before the consumer PR may merge.
+#
+# NOTHING IN THE GRAPH dependsOn THIS, so wait:true gates nothing — the blast radius of an unhealthy
+# plugin is this Kustomization's own Ready condition and nothing else. Anyone later adding
+# `dependsOn: [intel-gpu-plugin]` is choosing to make plugin health gate their thing; for the app
+# waves that would hand a device plugin the power to stall every app, which is the thing this file
+# exists to prevent. Think before adding that edge.
+#
+# ORDERING — DO NOT REORDER THE STACK. The renderer change (catalog/render.py emitting
+# `gpu.intel.com/i915: 1` for `hardware.gpu: intel-quicksync` instead of the hostPath) must NOT
+# land until this plugin is merged AND confirmed advertising the resource on all three nodes. If
+# the renderer requests a resource no node advertises, the Plex pod is Unschedulable; apps-wave-0 is
+# wait:true and apps-wave-1 dependsOn it, so that stalls the whole app tier — turning today's silent
+# no-op into a real outage. This PR's only job is to make the resource exist.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: intel-gpu-plugin
+  namespace: flux-system
+spec:
+  # Static vendored manifests with nothing to re-render; kustomize-controller is source-triggered,
+  # so this is only the drift-correction floor.
+  interval: 10m
+  retryInterval: 2m
+  # 10m, not the 5m the `platform` tier uses: a 3-node DaemonSet rollout under `wait: true` can
+  # exceed 5m on a cold start (image pull on three nodes), and the Cilium split-out comment calls
+  # out that exact mismatch as one of its reasons for leaving the shared tier.
+  timeout: 10m
+  prune: true
+  # See the header: a health SIGNAL for a component nothing depends on, not a gate.
+  wait: true
+  dependsOn:
+    - name: cilium   # the CNI — the only real prerequisite; transitively gives us flux-system too
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./platform/intel-gpu-plugin
+  # Retained for parity with every other tier / future overlay knobs. This component references no
+  # ${VARS} today, and its manifests contain no `$` literals for envsubst to eat (unlike
+  # clusters/base/runbooks.yaml, which omits substitution for exactly that reason).
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-vars
+  # No SOPS decryption patch: this component carries no Secret, so clusters/dev adds nothing for it
+  # (same as homepage/runbooks, unlike headlamp/crafty/oauth2-proxy/backup).

clusters/base/kustomization.yaml

@@ -12,6 +12,15 @@ resources:
   # interval/timeout, and `flux suspend kustomization cilium` as an emergency valve that does not
   # freeze the other eleven platform components. See clusters/base/cilium.yaml.
   - cilium.yaml          # Flux Kustomization -> ./platform/cilium (dependsOn flux-system, wait:true)
+  # Intel GPU device plugin — its OWN Kustomization (dependsOn cilium, wait:true), NOT a line in
+  # platform/kustomization.yaml. Same reasoning as the Cilium split-out, applied to a leaf: inside
+  # that wait:true tier a wedged 3-node DaemonSet would be indistinguishable from eleven unrelated
+  # components AND would stall platform-config + every apps wave on "dependency not ready" — so a
+  # component added to fix a silent no-op (Plex software transcoding) could cause a real outage.
+  # wait:true here is a HEALTH SIGNAL, not a gate: nothing in the graph dependsOn it.
+  # It advertises `gpu.intel.com/i915`; the renderer change that REQUESTS that resource must land
+  # strictly later. See clusters/base/intel-gpu-plugin.yaml.
+  - intel-gpu-plugin.yaml # Flux Kustomization -> ./platform/intel-gpu-plugin (dependsOn cilium)
   - platform.yaml        # Flux Kustomization -> ./platform (CONTROLLERS: HelmReleases / CRDs)
   - platform-config.yaml # Flux Kustomization -> ./platform-config (CONFIGS: CRs; dependsOn platform)
   # SECOND-ORDER CDI configs — CRs whose CRD is created by a controller AT RUNTIME, not by any

conformance/tests/l0-static/oauth2-proxy-api-routes-per-app/chainsaw-test.yaml

@@ -0,0 +1,134 @@
+# =============================================================================
+# L0 STATIC — `--api-route` is PER APP, ANCHORED, and DECLARED
+# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
+#
+# THE DEFECT THIS CLOSES. oauth2-proxy answers an unauthenticated request with a 302 to
+# Keycloak. Its built-in AJAX guard would send a 401 instead, but it fires only on an EXACT
+# `Accept: application/json`, which these apps' JavaScript does not consistently send. So a
+# browser XHR gets a 302, FOLLOWS IT INVISIBLY, has it satisfied silently by the live Keycloak
+# SSO session, and resolves 200 carrying the app's HTML where the JavaScript expected JSON — no
+# prompt, no console error, nothing logged as a failure. On a WRITE it is worse: a browser
+# following a 302 re-issues a POST as a GET and DROPS THE BODY, so the operator's change is
+# discarded and the UI reports success.
+#
+# MEASURED, twice, on two different apps and two different paths:
+#   jackett 2026-09-30 — on /api/ paths: 33 x 302 against 2 x 401, and 4 of 6 POSTs answered BY
+#           THE PROXY, never reaching Jackett (access-log upstream field `-`, not `/`). That is
+#           the operator's whole report: "GET works, most PUT/POST to add indexers fail,
+#           reloading never fixes it, and I am never re-prompted for login".
+#   radarr  2026-10-01, 23h after the cookie fix rolled (pod radarr-7d55cbffb9-42hmh, 0
+#           restarts) — 232 unauthenticated requests in the pod's whole lifetime and ALL 232 are
+#           the same call: `POST /signalr/messages/negotiate` from a browser UA, 302, upstream
+#           `-`. A left-open tab whose session the cookie rename orphaned, polling SignalR
+#           forever, silently redirected every time, NEVER re-prompting. The reported symptom,
+#           still running in production, on a service nobody was watching.
+#
+# WHY IT CANNOT BE A RENDERER CONSTANT — AND WHY THAT NEEDS A GATE. The path shape belongs to
+# the APP, not to the forward-auth pattern: radarr's broken path is `/signalr/`, jackett's is
+# `/api/`, and PIHOLE SERVES NO `/api/` AT ALL. One function renders all five sidecars
+# (catalog/render.py::_oauth2_sidecar_container), so the cheap fix is a constant — and a
+# constant applies one app's measurement to four apps nobody measured. The gate that already
+# exists cannot see that mistake: `ci/render-golden.sh` compares the render to the goldens, and
+# a constant plus regenerated goldens AGREE WITH EACH OTHER while both being wrong. Hence
+# assertion (2) below, which compares the rendered flag against each service's OWN
+# ServiceDefinition rather than against the renderer's output.
+#
+# WHY BREADTH IS THE DANGEROUS DIRECTION, AND IT IS NOT SYMMETRIC. A MISSING --api-route is a
+# silent 302 — bad, invisible, recoverable. A TOO BROAD one answers a plain PAGE LOAD with 401:
+# a logged-out operator gets a bare error page and is NEVER OFFERED A SIGN-IN, so the service
+# looks permanently broken and no reload recovers it. That is a worse outcome than the defect
+# the flag fixes, which is why the anchoring rule is asserted on three independent surfaces
+# (schema items.pattern, the renderer, and this check) and why assertion (5) fails if they drift
+# apart — if they do, the loosest one wins silently.
+#
+# GIVEN every oauth2-proxy sidecar this repo DEPLOYS (a container in any manifest at any depth:
+#       the rendered sidecars in apps/, crafty's hand-authored one in platform/), the
+#       ServiceDefinitions in catalog/ that generate five of them, and the renderer itself,
+# WHEN  each sidecar's `--api-route` values are collected (the flag is REPEATABLE, so all of
+#       them, in order) and each onboarded forward-auth service is paired with its golden,
+# THEN  (1) every rendered --api-route is anchored at `^/` and continues with literal path
+#           characters — `^/`, `^/.*`, `.*`, an unanchored `/api/` and any alternation FAIL;
+#       (2) every onboarded forward-auth service's rendered route list EQUALS its own
+#           ServiceDefinition's `auth.apiRoutes`, in order — which fails both a hand-edited
+#           golden and a renderer constant, neither of which render-golden can see;
+#       (3) BOTH cases are live in this tree: at least one service declares routes and at least
+#           one declares none and renders none (the pihole case), so neither path is an
+#           unexercised seam;
+#       (4) `_oauth2_sidecar_container()` reads `auth.apiRoutes` and contains NO literal path
+#           regex — a constant is the defect, back in the one place that reintroduces it
+#           fleet-wide in a single edit;
+#       (5) the anchoring pattern in the renderer and in the schema are the SAME pattern this
+#           check enforces;
+#       (6) the SHIPPED renderer, run as a subprocess on synthetic ServiceDefinitions, emits
+#           exactly the declared routes in order, emits NOTHING for a service declaring none or
+#           an empty list, and REFUSES all five breadth shapes.
+#
+# WHY ALSO A SCHEMA PATTERN, GIVEN THIS CHECK. The schema is the only surface that rejects a
+# too-broad regex WHERE A HUMAN TYPES IT — `ci/l0.sh` validates `catalog/*.yaml` against it on
+# every gate run, so the operator is told at the authoring surface rather than finding out from
+# a 401 on a page load. The renderer's own guard is there because `render-golden` runs even on a
+# gate invocation where the schema stage skipped. This check is there because neither of those
+# can see a renderer constant. Three surfaces, three different blind spots.
+#
+# WHY THIS IS A SIBLING AND NOT PART OF oauth2-proxy-cookie-names-distinct. That check's subject
+# is cookie IDENTITY: its inputs are names and domains, its failure mode is mutual session
+# eviction, its register is the shared proxy's claim on the default name. `--api-route` shares
+# none of those. Bundling them would make that check's name a lie and couple two unrelated
+# ratchets, so a future api-route decision could only be expressed by weakening a COOKIE check —
+# the same discipline that check's own header argues for, applied to this change.
+#
+# WHAT THIS DOES NOT ESTABLISH — read with the handoff entry, not instead of it:
+#   - That a 401 FIXES anything in a browser. No browser round-trip has been made, by anyone,
+#     for this flag. The 302/401 split, the dropped POST bodies and the orphaned SignalR poll
+#     are all read from access logs; the repair is reasoned from HTTP semantics.
+#   - That the regexes are RIGHT for these two apps beyond the measured paths. `^/api/` on
+#     radarr is justified by a measurement of radarr taken on its PRE-FIX pod by another
+#     engineer (decisions/2026-09-30-jackett-oauth2-proxy-session.md: 62 x /api/v3/command,
+#     46 x /api/v3/queue/details); the current pod has logged zero /api/ traffic.
+#   - Anything about sonarr, qbittorrent or pihole. Their logs were read (2026-10-01) and
+#     contain ZERO unauthenticated requests — no traffic, which is not the same as no need.
+#     Assertion (3) therefore proves the "declares none" PATH works; it does not claim those
+#     three apps are correctly configured.
+#   - That the LIVE pods carry what the tree says. git is not the cluster (CLAUDE.md §2b).
+#   - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06).
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: oauth2-proxy-api-routes-per-app
+  labels:
+    conformance.home.k3s/R-IDP-02: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-IDP-02"
+    conformance.home.k3s/layer: "L0"
+    # WHOLE-REPO SCOPE — the subject spans apps/wave-0, apps/wave-1, platform/crafty, every
+    # ServiceDefinition in catalog/, the renderer that generates five of the sidecars, and the
+    # ServiceDefinition schema. No single requirement path-maps to that set, and the renderer
+    # maps to no identity requirement at all, so a requirement-scoped selection would skip this
+    # check on precisely the change that can reintroduce the defect fleet-wide in one edit.
+    conformance.home.k3s/scope: "whole-repo"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      An unauthenticated XHR is answered 401, not 302, on exactly the paths each app was
+      MEASURED to serve as a machine API: every rendered --api-route is anchored at ^/ and
+      continues with literal path characters, every onboarded forward-auth service's rendered
+      routes equal its own auth.apiRoutes in order (so neither a hand-edited golden nor a
+      renderer constant can pass), an app declaring none renders none, the anchoring pattern is
+      the same in the schema and the renderer, and the shipped renderer is proven on synthetic
+      ServiceDefinitions in both directions — including refusing the five breadth shapes that
+      would answer a page load with 401.
+spec:
+  steps:
+    - name: assert-api-route-is-per-app-anchored-and-declared
+      description: >-
+        Inventory every oauth2-proxy sidecar's repeatable --api-route values, assert each is an
+        anchored path regex, cross-check each onboarded forward-auth service's rendered routes
+        against its own ServiceDefinition, require both the declares-routes and the
+        declares-none cases to be live in the tree, assert the renderer derives from
+        auth.apiRoutes and hardcodes no path regex, assert the schema and renderer share this
+        check's anchoring pattern, and prove the shipped renderer on synthetic
+        ServiceDefinitions in both directions.
+      try:
+        - script:
+            content: bash ./check.sh

conformance/tests/l0-static/oauth2-proxy-api-routes-per-app/check.sh

@@ -0,0 +1,433 @@
+#!/usr/bin/env bash
+# =============================================================================
+# L0 STATIC CHECK — `--api-route` is PER APP, ANCHORED, and DECLARED.
+# See chainsaw-test.yaml's header for WHY this exists and what it measured. In one paragraph:
+#
+# oauth2-proxy answers an unauthenticated request with a 302 to Keycloak. A browser XHR follows
+# that redirect invisibly, the live SSO session satisfies it silently, and the call resolves 200
+# carrying the app's HTML where its JavaScript expected JSON — and a POST following a 302 is
+# re-issued as a GET with its BODY DROPPED. `--api-route=<regex>` makes matching paths answer
+# 401 instead. The flag is right for jackett (`/api/`) and radarr (`/signalr/`, `/api/`) and
+# WRONG for pihole, which serves no `/api/` at all — so it cannot be a renderer constant, and
+# the point of this check is that it never becomes one again.
+#
+# The breadth direction is the dangerous one and it is NOT symmetric with the bug. A missing
+# `--api-route` is a silent 302; a TOO BROAD one answers a plain PAGE LOAD with 401, so a
+# logged-out operator gets a bare error page and is never offered a sign-in — the service looks
+# permanently broken. That is a worse outcome than the defect, which is why the anchoring rule
+# is asserted here as well as in the schema.
+#
+# Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, which the runner turns
+# into rc 3 = UNWIRED, never green, because this test declares the whole-repo scope) only if the
+# tree contains no forward-auth oauth2-proxy sidecar at all — genuinely nothing to audit.
+# =============================================================================
+set -euo pipefail
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
+PYTHON="${PYTHON:-python3}"
+
+# Honest SKIP (never green for a scope-declaring test) if the forward-auth pattern has left the
+# repo. Anchored on `image:` for the same reason as the sibling cookie check: a bare
+# `oauth2-proxy/oauth2-proxy:` also matches platform/oauth2-proxy/forward-auth-cec.yaml's Envoy
+# cluster reference (`serviceName: oauth2-proxy/oauth2-proxy:80`), which runs no proxy.
+if ! grep -rlE 'image:[[:space:]]*[^[:space:]]*oauth2-proxy' \
+       "${ROOT}/apps" "${ROOT}/platform" >/dev/null 2>&1; then
+  echo "check SKIPPED: no oauth2-proxy container image found under apps/ or platform/ — the forward-auth sidecar pattern is gone and there is nothing to audit" >&2
+  exit 3
+fi
+
+cd "${ROOT}"
+"${PYTHON}" - "${ROOT}" <<'PY'
+"""Static audit: --api-route is per-app, anchored, and declared in the ServiceDefinition.
+
+PyYAML (the same dependency as catalog/render.py and the sibling l0-static checks) rather than
+line-based parsing: a container's args live in a nested structure, and a line-based reader of
+one is precisely the approximate parser that silently stops matching.
+"""
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:                                  # pragma: no cover
+    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
+          file=sys.stderr)
+    raise SystemExit(1)
+
+ROOT = Path(sys.argv[1])
+sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree
+
+FAILURES: list[str] = []
+CHECKS = 0
+
+
+def check(cond, msg):
+    global CHECKS
+    CHECKS += 1
+    if not cond:
+        FAILURES.append(msg)
+
+
+# THE BREADTH RATCHET, stated once and asserted against three independent surfaces below (the
+# rendered tree, the authored catalog, the shipped renderer). Each entry must be anchored at
+# `^/` and continue with LITERAL path characters. This rejects:
+#   `^/`      and `^/.*`   — match every page load, so the UI answers 401 and never signs in
+#   `/api/`               — unanchored; oauth2-proxy would match it mid-path
+#   `.*`                  — everything
+#   `^/api/|^/`           — alternation smuggles breadth past a prefix-only reading
+# Alternation is REFUSED rather than supported because `--api-route` is repeatable: one entry
+# per route costs nothing and each entry stays readable. Kept byte-identical to
+# catalog/render.py's OAUTH2_API_ROUTE_PATTERN and to the schema's items.pattern; assertion (5)
+# below fails if they drift apart.
+ANCHORED = re.compile(r"^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$")
+SCHEMA_PATTERN = r"^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$"
+
+
+def read(p: Path) -> str:
+    try:
+        return p.read_text()
+    except (OSError, UnicodeDecodeError):
+        return ""
+
+
+def is_test_fixture(p: Path) -> bool:
+    """conformance/tests/**/{golden,fixtures}/** is assertion DATA, not deployed manifests."""
+    parts = p.parts
+    return "conformance" in parts and ("golden" in parts or "fixtures" in parts)
+
+
+def iter_containers(node):
+    """Yield every container mapping under any `containers`/`initContainers` list, at any depth.
+
+    Depth-agnostic on purpose: the same container shape appears under a Deployment, a DaemonSet,
+    a CronJob's jobTemplate or a bare Pod, and a reader that hardcoded one path would silently
+    stop seeing an instance the day one moved.
+    """
+    if isinstance(node, dict):
+        for key, val in node.items():
+            if key in ("containers", "initContainers") and isinstance(val, list):
+                for c in val:
+                    if isinstance(c, dict):
+                        yield c
+            else:
+                yield from iter_containers(val)
+    elif isinstance(node, list):
+        for item in node:
+            yield from iter_containers(item)
+
+
+def api_routes_of(args: list) -> list:
+    """Every `--api-route=<v>` / `--api-route <v>` value, in order. The flag is REPEATABLE, so
+    this must collect ALL of them — a single-value reader would audit the first and wave the
+    rest through, which is the shape of reader bug this file's header warns about."""
+    out = []
+    for i, a in enumerate(args):
+        if not isinstance(a, str):
+            continue
+        if a.startswith("--api-route="):
+            out.append(a.split("=", 1)[1])
+        elif a == "--api-route" and i + 1 < len(args) and isinstance(args[i + 1], str):
+            out.append(args[i + 1])
+    return out
+
+
+# ============================================================ (0) INVENTORY: THE RENDERED TREE
+# Every oauth2-proxy container this repo DEPLOYS, keyed by file + owner, with its api-routes.
+rendered: dict[str, list] = {}
+for p in sorted(ROOT.rglob("*.yaml")):
+    relparts = p.relative_to(ROOT).parent.parts
+    # NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such
+    # as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs,
+    # so a bare rglob audits up to 18 duplicates of every file — which both invents FAILs and,
+    # worse, can satisfy a "something declares X" check from a copy rather than from real source.
+    if any(part.startswith(".") for part in relparts) or is_test_fixture(p) or ".sops." in p.name:
+        continue
+    txt = read(p)
+    if "oauth2-proxy" not in txt:
+        continue
+    rel = str(p.relative_to(ROOT))
+    try:
+        docs = [d for d in yaml.safe_load_all(txt) if isinstance(d, dict)]
+    except yaml.YAMLError as exc:
+        check(False, f"{rel} is not parseable YAML ({exc}) — any oauth2-proxy instance in it "
+                     f"went unaudited, so a too-broad --api-route could be hiding there")
+        continue
+    for d in docs:
+        for c in iter_containers(d):
+            image = str(c.get("image") or "")
+            name = str(c.get("name") or "")
+            if "oauth2-proxy" not in image and name != "oauth2-proxy":
+                continue
+            owner = (d.get("metadata") or {}).get("name") or "?"
+            args = list(c.get("args") or []) + list(c.get("command") or [])
+            rendered[f"{rel}:{owner}/{name or 'oauth2-proxy'}"] = api_routes_of(args)
+
+check(len(rendered) >= 5,
+      f"only {len(rendered)} oauth2-proxy container(s) parsed out of the tree. This repo "
+      f"deploys more than that (five rendered sidecars plus crafty's hand-authored one), so the "
+      f"inventory has stopped seeing instances and every assertion below would pass vacuously "
+      f"while an unanchored --api-route went unaudited")
+
+# ============================================================ (1) EVERY ROUTE IS ANCHORED
+for key, routes in sorted(rendered.items()):
+    for r in routes:
+        check(bool(ANCHORED.match(r)),
+              f"{key} declares --api-route={r!r}, which is not an anchored path regex.\n"
+              f"       WHY THIS IS WORSE THAN THE BUG IT FIXES: --api-route makes matching paths "
+              f"answer 401 instead of 302ing to Keycloak. A regex that also matches a path a "
+              f"BROWSER NAVIGATES to — `^/`, `^/.*`, `.*`, or an alternation that reaches the "
+              f"UI root — answers a plain PAGE LOAD with 401. A logged-out operator then sees a "
+              f"bare error page and is NEVER OFFERED A SIGN-IN: the service looks permanently "
+              f"broken rather than merely unauthenticated, and no reload can recover it.\n"
+              f"       FIX: anchor it at '^/' followed by literal path characters "
+              f"(e.g. '^/api/', '^/signalr/', '^/api/v3/.*'), one entry per route — the flag is "
+              f"repeatable, so alternation is never needed. For a service rendered from catalog/, "
+              f"do NOT hand-edit the golden: declare auth.apiRoutes and re-render "
+              f"(ci/render-golden.sh's header has the canonical command)")
+
+# ============================================================ (2) DERIVED FROM THE CATALOG
+# The five rendered sidecars are OUTPUTS. Auditing only them would let someone "fix" a golden by
+# hand (reverted by the next render) or make the renderer emit a CONSTANT and regenerate every
+# golden to match — which render-golden cannot see, because render and golden would agree.
+# So: every onboarded forward-auth service's rendered route list must EQUAL its own
+# ServiceDefinition's auth.apiRoutes, in order.
+catalog_dir = ROOT / "catalog"
+declared: dict[str, list] = {}
+forward_auth: list[str] = []
+for sd_path in sorted(catalog_dir.glob("*.yaml")):
+    try:
+        sd = yaml.safe_load(read(sd_path))
+    except yaml.YAMLError as exc:
+        check(False, f"catalog/{sd_path.name} is not parseable YAML ({exc})")
+        continue
+    if not isinstance(sd, dict) or not sd.get("name"):
+        continue
+    auth = sd.get("auth") or {}
+    routes = auth.get("apiRoutes") or []
+    declared[sd["name"]] = list(routes)
+    if auth.get("mode") == "forward-auth" and (sd.get("ingressRoutes") or []):
+        forward_auth.append(sd["name"])
+    for r in routes:
+        check(bool(ANCHORED.match(str(r))),
+              f"catalog/{sd_path.name}: auth.apiRoutes entry {r!r} is not anchored at '^/' "
+              f"followed by literal path characters. See assertion (1) for why breadth here is "
+              f"the dangerous direction. The schema rejects this too (items.pattern); this "
+              f"assertion exists because the schema stage can be SKIPPED and this one cannot")
+    if routes:
+        check(auth.get("mode") == "forward-auth",
+              f"catalog/{sd_path.name} declares auth.apiRoutes with auth.mode="
+              f"{auth.get('mode')!r}. --api-route renders onto the per-app oauth2-proxy sidecar, "
+              f"and no sidecar is rendered for oidc/native/none — so this is a SILENT NO-OP: the "
+              f"operator believes the app's XHRs now answer 401 and nothing changed")
+
+matched = 0
+for name in sorted(forward_auth):
+    golden = next((p for p in sorted(ROOT.glob(f"apps/wave-*/{name}.yaml"))), None)
+    if golden is None:
+        continue                      # catalog service not onboarded to apps/ yet — no golden
+    key = next((k for k in rendered if k.startswith(str(golden.relative_to(ROOT)) + ":")), None)
+    check(key is not None,
+          f"apps/{golden.name} is the golden for forward-auth service {name!r} but no "
+          f"oauth2-proxy container was parsed out of it. Its --api-route configuration is "
+          f"UNAUDITED")
+    if key is None:
+        continue
+    matched += 1
+    check(rendered[key] == declared.get(name, []),
+          f"{name}: the rendered sidecar declares --api-route {rendered[key]!r} but "
+          f"catalog/{name}.yaml declares auth.apiRoutes {declared.get(name, [])!r}.\n"
+          f"       These must be IDENTICAL and in order. A mismatch means one of two things, "
+          f"both bad: a golden was hand-edited (the next re-render silently reverts it), or the "
+          f"renderer is emitting --api-route from somewhere other than this service's own "
+          f"ServiceDefinition — i.e. a constant or a per-service table. A constant is the defect "
+          f"this field exists to prevent: it applies ONE app's path shape to all five, and "
+          f"pihole serves no /api/ at all, so it would answer pihole's page loads with 401. "
+          f"render-golden cannot catch that, because render and golden would agree with each "
+          f"other while both being wrong")
+
+check(matched >= 4,
+      f"only {matched} onboarded forward-auth service(s) were cross-checked against their "
+      f"ServiceDefinition. This repo onboards five, so the catalog<->golden pairing has stopped "
+      f"resolving and assertion (2) is passing vacuously")
+
+# ============================================================ (3) BOTH CASES EXIST
+# Once the tree is consistent, (2) is satisfied by any pairing — including one where NOTHING
+# declares a route (vacuous) or one where EVERYTHING does (so the "renders nothing" path is
+# never exercised). Both halves are asserted to be live, because the per-app field only earns
+# its complexity if both outcomes actually occur in this tree.
+with_routes = [n for n in forward_auth if declared.get(n)]
+without_routes = [n for n in forward_auth if not declared.get(n)]
+check(bool(with_routes),
+      "no onboarded forward-auth service declares auth.apiRoutes, so assertion (2) cannot "
+      "distinguish a working renderer from one that emits the flag nowhere. If the flag is "
+      "genuinely not needed anywhere, delete the field rather than leaving an unexercised seam")
+check(bool(without_routes),
+      "EVERY onboarded forward-auth service declares auth.apiRoutes, so nothing in this tree "
+      "exercises the 'declares none => renders no flag' path. That path is the whole reason the "
+      "field is per-app: pihole serves no /api/, and a renderer-wide default would answer its "
+      "page loads with 401")
+for name in sorted(without_routes):
+    golden = next((p for p in sorted(ROOT.glob(f"apps/wave-*/{name}.yaml"))), None)
+    if golden is None:
+        continue
+    key = next((k for k in rendered if k.startswith(str(golden.relative_to(ROOT)) + ":")), None)
+    if key is None:
+        continue
+    check(rendered[key] == [],
+          f"{name} declares NO auth.apiRoutes but its rendered sidecar carries "
+          f"--api-route {rendered[key]!r}. An app that declares nothing must render NOTHING")
+
+# ============================================================ (4) THE RENDERER DERIVES
+render_py = ROOT / "catalog" / "render.py"
+check(render_py.is_file(),
+      "catalog/render.py is missing — the generator of five of these sidecars cannot be audited")
+if render_py.is_file():
+    src = read(render_py)
+    m = re.search(r"def _oauth2_sidecar_container\(.*?\n(?=\n*def |\n*# =)", src, re.S)
+    fn = m.group(0) if m else ""
+    check(bool(fn),
+          "could not locate _oauth2_sidecar_container() in catalog/render.py. It was renamed or "
+          "removed, so assertion (4) cannot run and the generator is unaudited")
+    if fn:
+        check("--api-route=" in fn,
+              "catalog/render.py's _oauth2_sidecar_container() emits no --api-route at all, so "
+              "every forward-auth app's XHRs are answered with a 302 the browser follows "
+              "invisibly — measured as 33x302-vs-2x401 on jackett and 232x302 on radarr, with "
+              "POST bodies dropped. auth.apiRoutes would then be a field with no consumer")
+        check(not re.search(r'--api-route=\^/[A-Za-z0-9]', fn),
+              "catalog/render.py hardcodes a literal path regex into --api-route. Whatever path "
+              "that is, EVERY forward-auth app now renders it: one app's path shape applied to "
+              "five. pihole serves no /api/, so a constant answers its page loads with 401 and "
+              "the service looks permanently broken to a logged-out operator. Render the routes "
+              "declared in each service's own auth.apiRoutes instead")
+        check("apiRoutes" in fn,
+              "catalog/render.py's _oauth2_sidecar_container() does not read auth.apiRoutes, so "
+              "the per-app declaration has no effect on what is rendered")
+
+# ============================================================ (5) ONE PATTERN, THREE SURFACES
+# The ratchet is written in three places (this check, the renderer, the schema) because each
+# guards a surface the others cannot. If they drift apart, the loosest one wins silently.
+if render_py.is_file():
+    # The renderer writes the pattern as a raw string literal, so the source text contains it
+    # verbatim. A substring test is deliberate: it also catches a rename of the constant.
+    check(SCHEMA_PATTERN in read(render_py),
+          f"catalog/render.py no longer contains the anchoring pattern {SCHEMA_PATTERN!r} that "
+          f"this check and the schema enforce. Three surfaces guard --api-route breadth (the "
+          f"authoring surface via the schema, the render via the renderer, the tree via this "
+          f"check); if they drift apart the loosest wins and the ratchet is decorative")
+
+# The schema's pattern is read from the PARSED JSON, not from the file text: in the file its
+# backslashes are JSON-escaped, so a substring test against the raw bytes would compare the
+# wrong string and pass or fail for the wrong reason.
+schema_file = ROOT / "schemas" / "servicedefinition.schema.json"
+schema_pat = None
+if schema_file.is_file():
+    try:
+        schema_pat = (((((json.loads(read(schema_file)).get("properties") or {})
+                         .get("auth") or {}).get("properties") or {})
+                       .get("apiRoutes") or {}).get("items") or {}).get("pattern")
+    except (ValueError, AttributeError) as exc:
+        check(False, f"schemas/servicedefinition.schema.json could not be read as JSON ({exc}) — "
+                     f"the schema stage that rejects a too-broad --api-route cannot be audited")
+check(schema_pat == SCHEMA_PATTERN,
+      f"schemas/servicedefinition.schema.json constrains auth.apiRoutes items with "
+      f"{schema_pat!r}, not {SCHEMA_PATTERN!r}. The schema is the only surface that rejects a "
+      f"too-broad regex WHERE A HUMAN TYPES IT (ci/l0.sh validates catalog/*.yaml on every gate "
+      f"run); if it is absent or looser than the renderer's, the operator finds out from a 401 "
+      f"on a page load")
+
+# ============================================================ (6) PROVE THE SHIPPED RENDERER
+# Once the tree is compliant, (1)-(3) have nothing left to catch and their passing is evidence
+# of nothing. This runs the SHIPPED renderer as a SUBPROCESS on synthetic ServiceDefinitions and
+# asserts the behaviour directly, in BOTH directions, so the guard stays falsifiable.
+def _sd(name: str, port: int, routes) -> str:
+    auth = {"mode": "forward-auth", "groups": ["platform-admins"]}
+    if routes is not None:
+        auth["apiRoutes"] = routes
+    return yaml.safe_dump({
+        "name": name,
+        "kind": "container",
+        "source": {"image": "example/app:1"},
+        "exposure": "internal",
+        "ingressRoutes": [{"host": f"{name}.home.keithjackson.dev", "port": port}],
+        "auth": auth,
+    }, sort_keys=False)
+
+
+def _render(name: str, port: int, routes):
+    with tempfile.TemporaryDirectory() as td:
+        f = Path(td) / f"{name}.yaml"
+        f.write_text(_sd(name, port, routes))
+        return subprocess.run([sys.executable, str(render_py), str(f)],
+                              capture_output=True, text=True)
+
+
+def _sidecar_args(rendered_yaml: str) -> list:
+    for doc in yaml.safe_load_all(rendered_yaml):
+        if not isinstance(doc, dict) or doc.get("kind") != "Deployment":
+            continue
+        for c in iter_containers(doc):
+            if c.get("name") == "oauth2-proxy":
+                return list(c.get("args") or [])
+    return []
+
+
+if render_py.is_file():
+    # (6a) DECLARED ROUTES RENDER, IN ORDER, AND NOTHING ELSE DOES.
+    proc = _render("apirtdecl", 8121, ["^/api/", "^/signalr/"])
+    check(proc.returncode == 0,
+          f"the shipped catalog/render.py failed on a synthetic forward-auth fixture declaring "
+          f"two apiRoutes: rc={proc.returncode} {proc.stderr.strip()[:300]}")
+    if proc.returncode == 0:
+        got = api_routes_of(_sidecar_args(proc.stdout))
+        check(got == ["^/api/", "^/signalr/"],
+              f"the shipped renderer turned auth.apiRoutes ['^/api/', '^/signalr/'] into "
+              f"--api-route {got!r}. It must emit one repeatable --api-route per entry, in "
+              f"declared order. Note the flag is SINGULAR: --api-routes does not exist in "
+              f"oauth2-proxy v7.8.1 and the proxy REFUSES TO START with it, so a plural would "
+              f"crashloop every sidecar it rendered")
+
+    # (6b) AN APP THAT DECLARES NOTHING RENDERS NOTHING. The pihole case, proven on the
+    #      generator rather than inferred from one committed golden.
+    for label, routes in (("absent", None), ("empty", [])):
+        proc = _render(f"apirtnone{label}", 8122, routes)
+        check(proc.returncode == 0,
+              f"the shipped renderer failed on a synthetic forward-auth fixture with apiRoutes "
+              f"{label}: rc={proc.returncode} {proc.stderr.strip()[:300]}")
+        if proc.returncode == 0:
+            got = api_routes_of(_sidecar_args(proc.stdout))
+            check(got == [],
+                  f"the shipped renderer emitted --api-route {got!r} for a synthetic service "
+                  f"whose auth.apiRoutes is {label}. A newly-onboarded forward-auth app must "
+                  f"render NO --api-route: pihole serves no /api/, so a default would answer "
+                  f"its page loads with 401 and the service would look permanently broken to a "
+                  f"logged-out operator — a worse failure than the 302 bug the flag fixes")
+
+    # (6c) THE RENDERER REFUSES BREADTH. Asserted on the generator because render-golden runs
+    #      on every gate invocation while the schema stage can be skipped.
+    for bad in ("^/", "^/.*", ".*", "/api/", "^/api/|^/"):
+        proc = _render("apirtbad", 8123, [bad])
+        check(proc.returncode != 0,
+              f"the shipped renderer ACCEPTED auth.apiRoutes {bad!r} and rendered a sidecar. "
+              f"That regex matches paths a browser NAVIGATES to, so a plain page load answers "
+              f"401 and the operator is never offered a sign-in. The renderer must refuse it, "
+              f"so a breadth mistake fails render-golden even on a run where the schema stage "
+              f"did not execute")
+
+# ============================================================ VERDICT
+inv = " | ".join(f"{k} -> {v or '(none)'}" for k, v in sorted(rendered.items()))
+print(f"oauth2-proxy sidecars audited ({len(rendered)}): {inv}")
+if FAILURES:
+    print(f"\nCHECK FAIL (oauth2-proxy-api-routes-per-app): {len(FAILURES)} failed, "
+          f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
+    for f in FAILURES:
+        print(f"  FAIL {f}", file=sys.stderr)
+    raise SystemExit(1)
+print(f"check OK: oauth2-proxy-api-routes-per-app — {CHECKS} assertions (R-IDP-02)")
+PY

conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/chainsaw-test.yaml

@@ -0,0 +1,152 @@
+# =============================================================================
+# L0 STATIC — no two oauth2-proxy instances can share a browser cookie
+# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
+#
+# THE DEFECT THIS CLOSES. Every oauth2-proxy on this platform is an INDEPENDENT instance with
+# its OWN `cookie-secret`: five per-app reverse-proxy sidecars (jackett, pihole, qbittorrent,
+# radarr, sonarr), crafty's, and the SHARED forward-auth proxy. Measured on prod 2026-09-30:
+# seven `*-oauth2-proxy` Secrets, SEVEN DISTINCT cookie-secret values — so no instance can
+# decrypt any other's cookie.
+#
+# They were all emitting oauth2-proxy's DEFAULT cookie names (`_oauth2_proxy` and
+# `_oauth2_proxy_csrf`), scoped to `.${BASE_DOMAIN}`. A cookie is keyed by (name, domain, path),
+# so the browser kept ONE pair for ALL of them. Whichever proxy authenticated last OVERWROTE
+# every other's session; the loser logged `No valid authentication in request`, bounced to
+# Keycloak, and — the SSO session still being live — was redirected straight back and overwrote
+# the cookie again. Six live instances, one cookie, mutual eviction.
+#
+# WHY IT IS THE WORST KIND OF BUG. The symptom is "I signed into Keycloak but I'm still not
+# signed in", which points the investigator at Keycloak, where nothing is wrong. It is
+# INTERMITTENT — it only appears once a browser has visited a SECOND gated app, so the obvious
+# single-app reproduction succeeds — and it is SILENT: every Deployment stays Ready, every
+# Kustomization stays Ready, the proxy's own log line names an authentication failure rather
+# than a collision. It went unnoticed across five services for the whole life of the pattern,
+# and was found only as a side effect of investigating something else (2026-09-29, crafty).
+#
+# WHY A COMMIT IS NOT ENOUGH, AND THIS IS A GATE STAGE. The collision is a property of the
+# PATTERN, not of any one app: the next forward-auth service onboarded inherits the defect by
+# doing nothing at all, because a missing flag IS the bug. Nothing about its absence is loud.
+# This is the same shape as the twelve inert ServiceMonitors in this tree — one omission,
+# copied, invisible for weeks — and the same remedy: make the omission fail the merge.
+#
+# GIVEN every oauth2-proxy instance this repo authors — a container in any manifest (the
+#       rendered sidecars in apps/, crafty's hand-authored one in platform/) and the shared
+#       proxy's HelmRelease values,
+# WHEN  each instance's EFFECTIVE cookie name is resolved (its declared `--cookie-name` /
+#       `OAUTH2_PROXY_COOKIE_NAME`, or oauth2-proxy's built-in default `_oauth2_proxy` when it
+#       declares none — an omission is a NAME, not an absence, and that is the whole bug),
+# THEN  (1) no two instances share an effective cookie name;
+#       (2) no instance's effective name is another's name + `_csrf` — oauth2-proxy derives the
+#           CSRF cookie that way, so `_oauth2_proxy` and `_oauth2_proxy_csrf` as two SESSION
+#           names would collide on the second cookie while looking distinct on the first;
+#       (3) the DEFAULT name has exactly ONE owner and it is the instance declared in
+#           SHARED_DEFAULT_COOKIE_OWNER. This is the deliberate ASYMMETRY. The shared
+#           forward-auth proxy MUST keep `_oauth2_proxy`: it is the single session every
+#           OTHER forward-auth-gated surface on the platform rides on, so renaming it signs
+#           every one of them out at once. The rule is therefore NOT "every instance must
+#           declare a name" — it is "the default is a NAME LIKE ANY OTHER, held by exactly one
+#           REGISTERED owner". That formulation needs no exemption from (1), and it closes the
+#           hole an exemption would leave: if the shared proxy is ever removed, an app silently
+#           inheriting the default would still FAIL here instead of quietly becoming its heir;
+#       (4) every instance that is NOT that owner is HOST-SCOPED: its `--cookie-domain` equals
+#           the host of its own `--redirect-url`. See the argument below;
+#       (5) catalog/render.py DERIVES both flags from the ServiceDefinition (the service name
+#           and the route host) and does not carry a per-service table. A list is how twelve
+#           ServiceMonitors acquired one identical wrong port name in a single edit, and a list
+#           leaves the NEXT app defaulting into the collision;
+#       (6) the SHIPPED renderer is proven END-TO-END on synthetic ServiceDefinitions, run as a
+#           subprocess: two forward-auth fixtures must yield two different, non-default cookie
+#           names, each scoped to its own host. Without this, once the tree is compliant (1)-(4)
+#           are satisfied by a tree with nothing left to catch, and a guard whose only evidence
+#           is "nothing failed" cannot be told apart from one that stopped working;
+#   AND (7) the check is LOAD-BEARING: this Test DECLARES `conformance.home.k3s/scope:
+#           whole-repo`, so ci/l0.sh Stage 3b's `--include-scope whole-repo` unions it into
+#           EVERY gate run, and ci/gate.sh blocks the merge when the l0 stage fails. It
+#           deliberately adds NO stage to ci/l0.sh — tests/l0-static/whole-repo-invariants-wired
+#           fails the gate if anyone adds one.
+#
+# WHY SCOPE, NOT REQUIREMENTS: ci/touched-requirements.sh maps `apps/*` to R-IDP-02/R-IDP-06
+# today, but the instances live in THREE unrelated subtrees (apps/, platform/crafty/,
+# platform/oauth2-proxy/) and the file most likely to reintroduce the defect is
+# catalog/render.py, which maps to R-GITOPS-02 and to no identity requirement at all. A
+# requirement-scoped selection would filter this check out of exactly the change that breaks it.
+#
+# ON ASSERTION (4), HOST-SCOPED COOKIE-DOMAIN — the argument, and the decision.
+#   AGAINST asserting it (overreach): distinct NAMES already close the eviction bug completely.
+#   Two cookies with different names do not overwrite each other however wide their domains are,
+#   so (4) fixes nothing that (1) has not already fixed. `--cookie-domain` is also a legitimate
+#   knob — a future proxy fronting two hostnames of one app would need a wider value — and a
+#   check that fails correct configurations is a check someone eventually disables.
+#   FOR asserting it (and this wins): a zone-wide cookie is TRANSMITTED BY THE BROWSER TO EVERY
+#   HOST IN THE ZONE. qBittorrent's session token is therefore sent to pihole, to radarr, to
+#   sonarr, and to every other name under `.${BASE_DOMAIN}` — including hosts behind no proxy at
+#   all. That is a session-credential disclosure across trust boundaries that no one needs and
+#   nobody asked for; the sidecar's session is only ever meaningful to the one app it fronts.
+#   And it is what makes a FUTURE name collision catastrophic rather than contained: with
+#   host-scoped cookies, two instances that accidentally share a name cannot even see each
+#   other's cookie, so the failure this whole test exists to prevent degrades from platform-wide
+#   to nothing. Defence in depth on the exact axis that already failed once.
+#   DECISION: assert it, for every instance except the registered shared proxy — whose whole
+#   job IS cross-host forward-auth, and which is exempted by name and with a reason, not by
+#   loosening the rule. `--whitelist-domain` is deliberately NOT touched: it governs permitted
+#   post-login REDIRECT targets, not cookie scope, and narrowing it breaks the sign-in bounce.
+#
+# STATUS: EXECUTABLE. `check.sh` is an offline assertion script (PyYAML, as catalog/render.py
+# and the sibling l0-static checks use); conformance/run.sh --layer l0 and ci/l0.sh run it
+# directly, and this Chainsaw Test wraps the SAME script.
+#
+# NOT proven (honest gaps; this is STATIC, with no cluster and no browser):
+#   - That anyone can actually sign in. No browser round-trip is performed, and none can be
+#     from L0. The 2026-09-29 crafty entry recorded the same gap and it is still open: the
+#     symptom has never been REPRODUCED in a browser, so this is a latent-bug fix reasoned from
+#     measured configuration, not a fix watched to work.
+#   - That the LIVE pods carry what the tree says. git is not the cluster (CLAUDE.md §2b). The
+#     live flags were measured separately, once, by hand; nothing here re-measures them, and a
+#     hand-applied drift would not be visible to this check.
+#   - That the cookie-secrets really differ. That is a Secret comparison and belongs in-cluster;
+#     it was measured out of band (seven distinct values, compared without printing any of
+#     them). If they were ever made IDENTICAL the collision would become harmless — but sharing
+#     one session-encryption key across trust zones is a worse idea than the bug it would mask,
+#     so this check does not offer it as an alternative and does not look for it.
+#   - Anything about the CONTENT of a session, group authorization, or the CSRF cookie's own
+#     behaviour. Names and domains only.
+#   - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06).
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: oauth2-proxy-cookie-names-distinct
+  labels:
+    conformance.home.k3s/R-IDP-02: "true"
+    conformance.home.k3s/R-IDP-06: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-IDP-02, R-IDP-06"
+    conformance.home.k3s/layer: "L0"
+    # WHOLE-REPO SCOPE — the instances span apps/wave-0, apps/wave-1, platform/crafty and
+    # platform/oauth2-proxy, and the generator that emits five of them is catalog/render.py.
+    # No single requirement path-maps to that set, and the renderer maps to no identity
+    # requirement at all, so a requirement-scoped selection would skip this check on precisely
+    # the change that can reintroduce the defect fleet-wide in one edit.
+    conformance.home.k3s/scope: "whole-repo"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      No two oauth2-proxy instances can share a browser cookie: every instance's effective
+      cookie name is unique, the built-in default belongs to exactly one registered owner (the
+      shared forward-auth proxy, which cannot be renamed without signing every gated app out),
+      every other instance is host-scoped, and catalog/render.py derives both flags from the
+      ServiceDefinition rather than from a per-service list — proven end-to-end on synthetic
+      fixtures so the guard stays falsifiable once the tree is clean.
+spec:
+  steps:
+    - name: assert-no-two-oauth2-proxy-instances-share-a-cookie
+      description: >-
+        Inventory every oauth2-proxy container and HelmRelease in the tree, resolve each one's
+        effective cookie name (declared, or oauth2-proxy's default when omitted), assert
+        pairwise distinctness including the session/CSRF derivation, ratchet the single
+        registered owner of the default name, assert host-scoped cookie-domain on every other
+        instance, assert catalog/render.py derives rather than lists, and prove the shipped
+        renderer on two synthetic forward-auth ServiceDefinitions.
+      try:
+        - script:
+            content: bash ./check.sh

conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/check.sh

@@ -0,0 +1,498 @@
+#!/usr/bin/env bash
+# =============================================================================
+# L0 STATIC CHECK — no two oauth2-proxy instances can share a browser cookie.
+# See chainsaw-test.yaml's header for WHY this exists, what it measured, and the argument
+# for assertion (4). In one paragraph:
+#
+# Every oauth2-proxy here holds its OWN cookie-secret (seven distinct values on prod,
+# 2026-09-30), so two instances sharing a cookie NAME on a shared DOMAIN cannot read each
+# other's session — they OVERWRITE it. The loser bounces to Keycloak, is redirected straight
+# back by the live SSO session, and overwrites the winner in turn. The symptom is "I signed
+# into Keycloak but I'm still not signed in", it only shows up after a second gated app is
+# visited, and nothing in the cluster reports an error. A missing --cookie-name IS the bug,
+# so the next forward-auth app inherits it by doing nothing.
+#
+# Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, which the runner
+# turns into rc 3 = UNWIRED, never green, because this test declares the whole-repo scope)
+# only if the tree contains no oauth2-proxy instance at all — genuinely nothing to audit.
+# =============================================================================
+set -euo pipefail
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
+PYTHON="${PYTHON:-python3}"
+
+# Honest SKIP (never green for a scope-declaring test) if there is nothing to audit at all.
+# Matches the IMAGE every instance runs and the CHART the shared one installs; if NEITHER
+# appears anywhere, the forward-auth pattern has left this repo and the check has no subject.
+#
+# ANCHORED ON `image:` DELIBERATELY. A bare `oauth2-proxy/oauth2-proxy:` also matches
+# platform/oauth2-proxy/forward-auth-cec.yaml's Envoy cluster reference
+# (`serviceName: oauth2-proxy/oauth2-proxy:80` — namespace/service:port, not a registry path).
+# That file configures the ext_authz listener that CALLS the proxy; it runs no proxy and holds
+# no cookie flags, so counting it made the grep/parse cross-check below report a phantom
+# unaudited instance.
+if ! grep -rlE 'image:[[:space:]]*[^[:space:]]*oauth2-proxy|chart:[[:space:]]*oauth2-proxy' \
+       "${ROOT}/apps" "${ROOT}/platform" "${ROOT}/catalog" >/dev/null 2>&1; then
+  echo "check SKIPPED: no oauth2-proxy image or chart found under apps/, platform/ or catalog/ — nothing to audit" >&2
+  exit 3
+fi
+
+cd "${ROOT}"
+"${PYTHON}" - "${ROOT}" <<'PY'
+"""Static audit: no two oauth2-proxy instances can share a browser cookie.
+
+PyYAML (the same dependency as catalog/render.py and the sibling l0-static checks) rather
+than line-based parsing: a container's args live in a nested structure and a HelmRelease's
+flags live in a values map, and a line-based reader of either is precisely the approximate
+parser that silently stops matching — the failure class this check exists to prevent.
+"""
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:                                  # pragma: no cover
+    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
+          file=sys.stderr)
+    raise SystemExit(1)
+
+ROOT = Path(sys.argv[1])
+sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree
+
+FAILURES: list[str] = []
+CHECKS = 0
+
+
+def check(cond, msg):
+    global CHECKS
+    CHECKS += 1
+    if not cond:
+        FAILURES.append(msg)
+
+
+def read(p: Path) -> str:
+    try:
+        return p.read_text()
+    except (OSError, UnicodeDecodeError):
+        return ""
+
+
+# oauth2-proxy's OWN built-in defaults (pkg/apis/options/cookie.go): the session cookie is
+# `_oauth2_proxy` and the CSRF cookie is that name with `_csrf` appended. An instance that
+# declares nothing is NOT unconfigured — it has claimed this exact name, which is why the
+# inventory below treats the default as a name like any other rather than as an absence.
+DEFAULT_COOKIE_NAME = "_oauth2_proxy"
+CSRF_SUFFIX = "_csrf"
+
+# =============================================================================================
+# REGISTER — SHARED_DEFAULT_COOKIE_OWNER. Exactly one entry, and it must match a real instance.
+#
+# The one instance permitted to hold DEFAULT_COOKIE_NAME, and the one permitted a zone-wide
+# cookie-domain. Both exemptions belong to the SAME object for the SAME reason, so they are one
+# register rather than two: the shared forward-auth proxy is not a per-app sidecar, it is the
+# single session that every forward-auth-gated surface on the platform rides on. Renaming its
+# cookie signs all of them out simultaneously, and host-scoping it would break the cross-host
+# forward-auth that is its entire job.
+#
+# Shrink-only, and exact in BOTH directions: an entry that matches no instance FAILS as stale
+# (the proxy was renamed or removed and the exemption is now a hole), and an instance holding
+# the default WITHOUT an entry FAILS as an unregistered heir. Adding an entry is a review event
+# on a sensitive path.
+#
+# Keyed by "<relpath>:<instance id>".
+# =============================================================================================
+SHARED_DEFAULT_COOKIE_OWNER: dict[str, str] = {
+    "platform/oauth2-proxy/helmrelease.yaml:oauth2-proxy":
+        "The SHARED forward-auth proxy (ADR-0014, platform/oauth2-proxy/README.md). It is the "
+        "single authenticator behind every forward-auth-gated surface that is NOT fronted by "
+        "its own sidecar, so its cookie is the platform session: renaming it signs every one "
+        "of them out at once, and that is why the 2026-09-29 crafty fix changed only the "
+        "sidecar side. Its cookie-domain is zone-wide for the same structural reason — "
+        "cross-host forward-auth is what it is for.",
+}
+
+
+def is_test_fixture(p: Path) -> bool:
+    """conformance/tests/**/{golden,fixtures}/** is assertion DATA, not deployed manifests.
+
+    Auditing a golden would be auditing a copy of something already audited through apps/.
+    Same exclusion the sibling l0-static checks make, and the same "reference, not
+    declaration" distinction ci/touched-requirements.sh draws for fixture paths.
+    """
+    parts = p.parts
+    return "conformance" in parts and ("golden" in parts or "fixtures" in parts)
+
+
+def iter_containers(node):
+    """Yield every container mapping under any `containers`/`initContainers` list, at any depth.
+
+    Depth-agnostic on purpose: a sidecar sits under spec.template.spec.containers in a
+    Deployment, but the same container shape appears under a CronJob's jobTemplate, a
+    DaemonSet, or a Pod, and a reader that hardcoded one path would silently stop seeing an
+    instance the day one moved.
+    """
+    if isinstance(node, dict):
+        for key, val in node.items():
+            if key in ("containers", "initContainers") and isinstance(val, list):
+                for c in val:
+                    if isinstance(c, dict):
+                        yield c
+            else:
+                yield from iter_containers(val)
+    elif isinstance(node, list):
+        for item in node:
+            yield from iter_containers(item)
+
+
+def flag_from_args(args: list, flag: str):
+    """Value of `--flag=value` or `--flag value` in an argv list, else None."""
+    pref = "--" + flag
+    for i, a in enumerate(args):
+        if not isinstance(a, str):
+            continue
+        if a.startswith(pref + "="):
+            return a.split("=", 1)[1]
+        if a == pref and i + 1 < len(args) and isinstance(args[i + 1], str):
+            return args[i + 1]
+    return None
+
+
+def host_of(url):
+    """Host part of an https://host/path URL, else None."""
+    if not isinstance(url, str):
+        return None
+    m = re.match(r"^https?://([^/]+)", url)
+    return m.group(1) if m else None
+
+
+# ============================================================ (0) INVENTORY
+# instances: list of dicts {rel, ident, kind, declared_name, cookie_domain, redirect_host}
+instances: list[dict] = []
+unparseable: list[str] = []
+# Files the grep gate says contain an instance, so the parse can be cross-checked against it.
+grep_files: set[str] = set()
+
+for p in sorted(ROOT.rglob("*.yaml")):
+    relparts = p.relative_to(ROOT).parent.parts
+    # NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES
+    # such as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent
+    # runs, so a bare rglob audited up to 18 duplicates of every file — which both invents
+    # FAILs and, worse, can satisfy a "something declares X" check from a copy rather than from
+    # real source. Same fix as egress-paths-selected-by-rules. .claude/ is gitignored.
+    if any(part.startswith(".") for part in relparts) or is_test_fixture(p) or ".sops." in p.name:
+        continue
+    txt = read(p)
+    if not txt:
+        continue
+    rel = str(p.relative_to(ROOT))
+    # Same anchoring as check.sh's skip gate: `image:` (or a chart name), never a bare path —
+    # see the comment there for the Envoy `serviceName: oauth2-proxy/oauth2-proxy:80` near-miss.
+    if re.search(r"image:\s*\S*oauth2-proxy|chart:\s*oauth2-proxy", txt):
+        grep_files.add(rel)
+    elif "oauth2-proxy" not in txt:
+        continue
+    try:
+        docs = [d for d in yaml.safe_load_all(txt) if isinstance(d, dict)]
+    except yaml.YAMLError as exc:
+        unparseable.append(f"{rel} ({exc})")
+        continue
+    for d in docs:
+        # --- shape 1: a container running the oauth2-proxy image -------------------------
+        for c in iter_containers(d):
+            image = str(c.get("image") or "")
+            name = str(c.get("name") or "")
+            if "oauth2-proxy" not in image and name != "oauth2-proxy":
+                continue
+            args = [a for a in (list(c.get("args") or []) + list(c.get("command") or []))]
+            env = {e.get("name"): e.get("value")
+                   for e in (c.get("env") or []) if isinstance(e, dict) and "value" in e}
+            owner = (d.get("metadata") or {}).get("name") or "?"
+            instances.append({
+                "rel": rel,
+                "ident": f"{owner}/{name or 'oauth2-proxy'}",
+                "kind": "container",
+                "declared_name": flag_from_args(args, "cookie-name")
+                                 or env.get("OAUTH2_PROXY_COOKIE_NAME"),
+                "cookie_domain": flag_from_args(args, "cookie-domain")
+                                 or env.get("OAUTH2_PROXY_COOKIE_DOMAIN"),
+                "redirect_host": host_of(flag_from_args(args, "redirect-url")
+                                         or env.get("OAUTH2_PROXY_REDIRECT_URL")),
+            })
+        # --- shape 2: the shared proxy's HelmRelease values ------------------------------
+        if d.get("kind") != "HelmRelease":
+            continue
+        spec = d.get("spec") or {}
+        chart = (((spec.get("chart") or {}).get("spec") or {}).get("chart") or "")
+        if "oauth2-proxy" not in str(chart):
+            continue
+        values = spec.get("values") or {}
+        extra = values.get("extraArgs") or {}
+        if isinstance(extra, list):          # the chart also accepts a bare argv list
+            extra = {a.lstrip("-").split("=", 1)[0]: a.split("=", 1)[1]
+                     for a in extra if isinstance(a, str) and "=" in a}
+        cfg = values.get("config") or {}
+        get = lambda k: extra.get(k, cfg.get(k))     # noqa: E731 - two-source lookup, used 3x
+        instances.append({
+            "rel": rel,
+            "ident": str((d.get("metadata") or {}).get("name") or "oauth2-proxy"),
+            "kind": "helmrelease",
+            "declared_name": get("cookie-name"),
+            "cookie_domain": get("cookie-domain"),
+            "redirect_host": host_of(get("redirect-url")),
+        })
+
+for u in unparseable:
+    check(False, f"{u} is not parseable YAML — every oauth2-proxy instance in it went unaudited")
+
+for ins in instances:
+    ins["key"] = f"{ins['rel']}:{ins['ident']}"
+    # THE CORE MODELLING DECISION: an omitted --cookie-name is not "unset", it is a CLAIM on
+    # oauth2-proxy's built-in default. Resolving it here is what lets assertion (1) below be a
+    # plain distinctness test with no special case, and it is exactly the step whose absence
+    # made the defect invisible to every reviewer who read these manifests.
+    ins["effective_name"] = ins["declared_name"] or DEFAULT_COOKIE_NAME
+
+# Anti-vacuous. The floor is deliberately low (a real removal should not turn this check red),
+# so the load-bearing guard is the grep/parse CROSS-CHECK beneath it: every file the cheap
+# text scan says holds an instance must yield at least one PARSED instance. A parser that
+# quietly stopped matching would keep the floor and fail this.
+check(len(instances) >= 5,
+      f"only {len(instances)} oauth2-proxy instance(s) parsed out of the tree. This repo "
+      f"authors more than that (five rendered sidecars, crafty's, and the shared proxy), so "
+      f"the inventory has stopped seeing instances and every assertion below would pass "
+      f"vacuously while a collision went unaudited")
+
+parsed_files = {ins["rel"] for ins in instances}
+for rel in sorted(grep_files - parsed_files):
+    check(False,
+          f"{rel} contains an oauth2-proxy image or chart reference but NO instance was parsed "
+          f"out of it. Either it uses a shape iter_containers()/the HelmRelease reader does not "
+          f"recognise, or the file stopped parsing. Its cookie configuration is UNAUDITED, which "
+          f"is the same state the whole fleet was in before this check existed")
+
+# ============================================================ (1) DISTINCT EFFECTIVE NAMES
+by_name: dict[str, list[dict]] = {}
+for ins in instances:
+    by_name.setdefault(ins["effective_name"], []).append(ins)
+
+for nm, group in sorted(by_name.items()):
+    if len(group) < 2:
+        continue
+    who = ", ".join(sorted(g["key"] for g in group))
+    default_note = (
+        " Every one of them OMITS --cookie-name, so they are all claiming oauth2-proxy's "
+        "built-in default. Omitting the flag is not neutral: it IS this collision."
+        if nm == DEFAULT_COOKIE_NAME else "")
+    check(False,
+          f"{len(group)} oauth2-proxy instances share the cookie name '{nm}': {who}.{default_note}\n"
+          f"       WHY THIS IS A BUG AND NOT A STYLE RULE: these are INDEPENDENT instances with "
+          f"DIFFERENT cookie-secrets (measured on prod 2026-09-30: seven distinct values), so "
+          f"neither can DECRYPT the other's cookie — it can only OVERWRITE it. A browser keys a "
+          f"cookie by (name, domain, path), so signing into one app destroys the session of every "
+          f"other app sharing that key. The loser logs 'No valid authentication in request', "
+          f"bounces to Keycloak, is redirected straight back because the SSO session is still "
+          f"live, and overwrites the winner in turn. The user sees 'I signed into Keycloak but "
+          f"I'm still not signed in' and goes looking in Keycloak, where nothing is wrong.\n"
+          f"       FIX: give each per-app instance its own --cookie-name. For a service rendered "
+          f"from catalog/, do NOT hand-edit the golden — catalog/render.py derives the flag from "
+          f"the service name; re-render instead (ci/render-golden.sh's header has the canonical "
+          f"command). Exactly one instance may hold '{DEFAULT_COOKIE_NAME}', and it must be the "
+          f"one registered in SHARED_DEFAULT_COOKIE_OWNER.")
+
+# ============================================================ (2) SESSION vs CSRF DERIVATION
+# oauth2-proxy names the CSRF cookie <cookie-name>_csrf. So a SESSION name that happens to equal
+# another instance's name + "_csrf" collides on the CSRF cookie while assertion (1) sees two
+# perfectly distinct strings. This is the near-miss the obvious fix invites: naming the shared
+# proxy's neighbour `_oauth2_proxy_csrf` reads as "clearly different" and is not.
+effective = {ins["effective_name"] for ins in instances}
+for ins in sorted(instances, key=lambda i: i["key"]):
+    base = ins["effective_name"]
+    if not base.endswith(CSRF_SUFFIX):
+        continue
+    victim = base[: -len(CSRF_SUFFIX)]
+    check(victim not in effective,
+          f"{ins['key']} uses the SESSION cookie name '{base}', which is exactly the CSRF cookie "
+          f"oauth2-proxy derives for the instance named '{victim}' (it appends '{CSRF_SUFFIX}'). "
+          f"The two session names look distinct and the CSRF cookies collide, so sign-in breaks "
+          f"with a state-mismatch instead of a session loss — the same bug wearing a different "
+          f"error message. Pick a name that is not another instance's name + '{CSRF_SUFFIX}'")
+
+# ============================================================ (3) THE DEFAULT HAS ONE OWNER
+default_holders = sorted(ins["key"] for ins in instances
+                         if ins["effective_name"] == DEFAULT_COOKIE_NAME)
+registered = set(SHARED_DEFAULT_COOKIE_OWNER)
+
+for key in sorted(registered - set(ins["key"] for ins in instances)):
+    check(False,
+          f"SHARED_DEFAULT_COOKIE_OWNER registers '{key}', but no such oauth2-proxy instance "
+          f"exists in the tree. The register is SHRINK-ONLY and exact: a stale entry is a "
+          f"standing exemption for an object that is gone, so the next instance to inherit the "
+          f"default name would be silently excused. Remove the entry, or restore the instance")
+
+for key in default_holders:
+    check(key in registered,
+          f"{key} holds oauth2-proxy's DEFAULT cookie name '{DEFAULT_COOKIE_NAME}' but is NOT "
+          f"registered in SHARED_DEFAULT_COOKIE_OWNER. Exactly ONE instance may hold the default, "
+          f"and it is the SHARED forward-auth proxy — whose cookie is the platform-wide session "
+          f"every forward-auth-gated surface rides on, so renaming IT signs them all out at once. "
+          f"Every other instance must declare its own --cookie-name. If this instance really is "
+          f"the new shared proxy, register it here with the reason; if it is a per-app sidecar, "
+          f"give it a unique name")
+
+check(len(default_holders) <= 1,
+      f"{len(default_holders)} instances hold the default cookie name: {', '.join(default_holders)}. "
+      f"The default is a NAME, not an absence — it can have exactly one owner")
+
+# ============================================================ (4) HOST-SCOPED COOKIE DOMAIN
+# See the chainsaw-test header for the argument (distinct names alone already fix the eviction;
+# host-scoping is asserted anyway because a zone-wide cookie is TRANSMITTED to every host in the
+# zone, and because it makes a future name collision contained instead of platform-wide).
+for ins in sorted(instances, key=lambda i: i["key"]):
+    if ins["key"] in registered:
+        continue                          # the shared proxy: zone-wide by design, registered
+    dom, rhost = ins["cookie_domain"], ins["redirect_host"]
+    check(dom is not None,
+          f"{ins['key']} declares no --cookie-domain. oauth2-proxy then scopes the cookie to the "
+          f"request host, which happens to be correct — but it is correct BY ACCIDENT and says "
+          f"nothing to the next reader. Declare it explicitly as this app's own host")
+    if dom is None:
+        continue
+    check(not dom.startswith("."),
+          f"{ins['key']} sets --cookie-domain={dom}. A leading dot makes the cookie ZONE-WIDE: "
+          f"the browser sends this app's session token to every other host under that domain, "
+          f"including hosts behind no proxy at all, and it is what turns any future cookie-name "
+          f"collision into a platform-wide outage instead of a contained one. Scope it to this "
+          f"app's own host. Only the registered shared forward-auth proxy may be zone-wide, "
+          f"because cross-host forward-auth is its job")
+    if rhost:
+        check(dom == rhost,
+              f"{ins['key']} sets --cookie-domain={dom} but its --redirect-url host is {rhost}. "
+              f"The cookie is then set for a domain the sign-in flow never returns to (or for a "
+              f"wider one than this app owns), so the session either does not stick or leaks "
+              f"beyond the app. Both flags describe the SAME app: make them agree")
+
+# ============================================================ (5) THE RENDERER DERIVES
+# Five of these instances are GENERATED. Auditing only the goldens would let someone "fix" the
+# tree by hand and leave the generator emitting the defect for the next service onboarded — and
+# the next re-render would then silently revert the hand fix.
+render_py = ROOT / "catalog" / "render.py"
+check(render_py.is_file(),
+      "catalog/render.py is missing — the generator of five of these instances cannot be audited")
+if render_py.is_file():
+    src = read(render_py)
+    m = re.search(r"def _oauth2_sidecar_container\(.*?\n(?=\n*def |\n*# =)", src, re.S)
+    fn = m.group(0) if m else ""
+    check(bool(fn),
+          "could not locate _oauth2_sidecar_container() in catalog/render.py. It was renamed or "
+          "removed; assertions (5) below cannot run, and the generator is unaudited")
+    if fn:
+        cookie_name_lines = [ln for ln in fn.splitlines() if "--cookie-name" in ln]
+        check(bool(cookie_name_lines),
+              "catalog/render.py's _oauth2_sidecar_container() emits NO --cookie-name at all, so "
+              "every service it renders claims oauth2-proxy's default and collides with every "
+              "other. This is the defect itself, back in the one place that reintroduces it "
+              "fleet-wide in a single edit")
+        check(any("s['name']" in ln or 's["name"]' in ln for ln in cookie_name_lines),
+              "catalog/render.py emits --cookie-name without DERIVING it from the "
+              "ServiceDefinition's own name. A constant collides with every sibling by "
+              "construction; a per-service table leaves the NEXT app onboarded defaulting into "
+              "the collision, which is how twelve ServiceMonitors acquired one identical wrong "
+              "port name in a single edit. Derive it from s['name']")
+        cookie_dom_lines = [ln for ln in fn.splitlines() if "--cookie-domain" in ln]
+        check(any("{host}" in ln for ln in cookie_dom_lines),
+              "catalog/render.py's sidecar --cookie-domain is not derived from the route host. A "
+              "zone-wide literal (`.{ZONE}`) broadcasts every app's session cookie across the "
+              "whole zone — see assertion (4)")
+        check(not any(re.search(r"--cookie-name=_oauth2_proxy_[a-z0-9]", ln) for ln in cookie_name_lines),
+              "catalog/render.py hardcodes a literal service name into --cookie-name. Whatever "
+              "service that is, every OTHER service now renders the same constant and they all "
+              "collide. Derive the name instead")
+
+# ============================================================ (6) PROVE THE SHIPPED RENDERER
+# Once the tree is compliant, (1)-(4) have nothing left to catch and their passing is evidence
+# of nothing. This runs the SHIPPED renderer as a SUBPROCESS on two synthetic ServiceDefinitions
+# and asserts the property directly, so the guard stays falsifiable.
+def _sd(name: str, port: int) -> str:
+    return yaml.safe_dump({
+        "name": name,
+        "kind": "container",
+        "source": {"image": "example/app:1"},
+        "exposure": "internal",
+        "ingressRoutes": [{"host": f"{name}.home.keithjackson.dev", "port": port}],
+        "auth": {"mode": "forward-auth", "groups": ["platform-admins"]},
+    }, sort_keys=False)
+
+
+def _sidecar_args(rendered: str) -> list:
+    for doc in yaml.safe_load_all(rendered):
+        if not isinstance(doc, dict) or doc.get("kind") != "Deployment":
+            continue
+        for c in iter_containers(doc):
+            if c.get("name") == "oauth2-proxy":
+                return list(c.get("args") or [])
+    return []
+
+
+if render_py.is_file():
+    fixtures = {"cookiefixa": 8111, "cookiefixb": 8112}
+    rendered_args: dict[str, list] = {}
+    with tempfile.TemporaryDirectory() as td:
+        for svc, port in fixtures.items():
+            f = Path(td) / f"{svc}.yaml"
+            f.write_text(_sd(svc, port))
+            proc = subprocess.run([sys.executable, str(render_py), str(f)],
+                                  capture_output=True, text=True)
+            check(proc.returncode == 0,
+                  f"the shipped catalog/render.py failed on a synthetic forward-auth fixture "
+                  f"({svc}): rc={proc.returncode} {proc.stderr.strip()[:200]}")
+            if proc.returncode == 0:
+                rendered_args[svc] = _sidecar_args(proc.stdout)
+
+    names, domains = {}, {}
+    for svc, args in rendered_args.items():
+        names[svc] = flag_from_args(args, "cookie-name")
+        domains[svc] = flag_from_args(args, "cookie-domain")
+        check(bool(args),
+              f"the shipped renderer produced no oauth2-proxy sidecar for synthetic forward-auth "
+              f"fixture {svc} — the sidecar is how a forward-auth app is gated at all (R-IDP-02)")
+        check(names[svc] not in (None, DEFAULT_COOKIE_NAME),
+              f"the shipped renderer gave synthetic fixture {svc} the cookie name "
+              f"{names[svc]!r}. A newly-onboarded forward-auth app would therefore default "
+              f"straight into the fleet collision — which is exactly how five services acquired "
+              f"this defect without anyone making a decision")
+        check(domains[svc] == f"{svc}.home.keithjackson.dev",
+              f"the shipped renderer scoped synthetic fixture {svc}'s cookie to "
+              f"{domains[svc]!r} rather than to its own host — a newly-onboarded app would "
+              f"broadcast its session cookie across the whole zone")
+        check(any(isinstance(a, str) and a.startswith("--whitelist-domain=.") for a in args),
+              f"synthetic fixture {svc} lost its ZONE-WIDE --whitelist-domain. That flag governs "
+              f"permitted post-login REDIRECT targets, not cookie scope; narrowing it with the "
+              f"cookie domain breaks the sign-in bounce")
+
+    if len(rendered_args) == 2:
+        a, b = sorted(rendered_args)
+        check(names[a] != names[b],
+              f"the shipped renderer gave two DIFFERENT synthetic services the SAME cookie name "
+              f"({names[a]!r}). Whatever the committed goldens currently look like, the generator "
+              f"will collide every forward-auth app it renders from now on")
+        check(domains[a] != domains[b],
+              f"the shipped renderer gave two different synthetic services the same cookie domain "
+              f"({domains[a]!r}) — the cookies would share a scope even with distinct names")
+
+# ============================================================ VERDICT
+inv = " | ".join(f"{i['key']} -> {i['effective_name']} @ {i['cookie_domain'] or '<host>'}"
+                 for i in sorted(instances, key=lambda x: x["key"]))
+print(f"oauth2-proxy instances audited ({len(instances)}): {inv}")
+if FAILURES:
+    print(f"\nCHECK FAIL (oauth2-proxy-cookie-names-distinct): {len(FAILURES)} failed, "
+          f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
+    for f in FAILURES:
+        print(f"  FAIL {f}", file=sys.stderr)
+    raise SystemExit(1)
+print(f"check OK: oauth2-proxy-cookie-names-distinct — {CHECKS} assertions (R-IDP-02, R-IDP-06)")
+PY

conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh

@@ -129,6 +129,22 @@ KNOWN_EXTERNAL_MONITORS: dict[str, str] = {
         "a PodMonitor, so it resolves against POD ports, not Service ports — the Flux controller "
         "Deployments (and their `http-prom` container port) are installed by the Flux bootstrap "
         "manifests, not authored here. Live on prod: job flux-system/flux-system, 6 targets up.",
+    "platform/oauth2-proxy/podmonitor.yaml:oauth2-proxy-sidecars":
+        "a PodMonitor, so it resolves against POD ports, not Service ports. Unlike the other "
+        "entries here the target is NOT chart-owned — the `metrics` (44180) container port is "
+        "authored in this repo, on the oauth2-proxy sidecar in apps/wave-0/{jackett,qbittorrent,"
+        "pihole}.yaml and apps/wave-1/{radarr,sonarr}.yaml. It is unauditable here because NO "
+        "Service exposes 44180 BY DESIGN: adding one would mean changing catalog/render.py and "
+        "every apps/wave-* render and golden, to publish an endpoint only Prometheus should dial. "
+        "A PodMonitor needs no Service, which is the whole reason it was chosen. What assertion "
+        "(2) would have checked was instead MEASURED on prod 2026-10-01, read-only: the pods "
+        "declare ports [('http', 4180), ('metrics', 44180)], the selector label "
+        "homelab.keithjackson.dev/auth-mode=forward-auth is on exactly those five gate pods, and "
+        "port-forwarding 44180 returns http=200 with 10208 bytes of oauth2_proxy_* metrics. "
+        "CAVEAT, deliberately recorded: those targets are DOWN on prod, because the app "
+        "namespaces' default-deny NetworkPolicy drops the scrape from ns monitoring (same bug and "
+        "fingerprint as platform/longhorn/networkpolicy-metrics.yaml documents). The port name "
+        "resolves; the network path does not yet.",
 }
 
 MONITOR_KINDS = ("ServiceMonitor", "PodMonitor")

conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml

@@ -76,11 +76,14 @@ spec:
         - --allowed-group=platform-admins
         - --skip-provider-button=true
         - --redirect-url=https://paas-demo.home.keithjackson.dev/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --cookie-name=_oauth2_proxy_paas-demo
+        - --cookie-domain=paas-demo.home.keithjackson.dev
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -101,6 +104,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml

@@ -99,11 +99,14 @@ spec:
         - --allowed-group=media-admins
         - --skip-provider-button=true
         - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --cookie-name=_oauth2_proxy_sonarr
+        - --cookie-domain=sonarr.home.keithjackson.dev
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -124,6 +127,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml

@@ -99,11 +99,14 @@ spec:
         - --allowed-group=media-admins
         - --skip-provider-button=true
         - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
-        - --cookie-domain=.${BASE_DOMAIN}
+        - --cookie-name=_oauth2_proxy_sonarr
+        - --cookie-domain=sonarr.home.keithjackson.dev
         - --whitelist-domain=.${BASE_DOMAIN}
         - --cookie-secure=true
+        - --cookie-refresh=2m30s
         - --pass-access-token=true
         - --set-xauthrequest=true
+        - --metrics-address=0.0.0.0:44180
         env:
         - name: OAUTH2_PROXY_CLIENT_ID
           valueFrom:
@@ -124,6 +127,9 @@ spec:
         - name: http
           containerPort: 4180
           protocol: TCP
+        - name: metrics
+          containerPort: 44180
+          protocol: TCP
         resources:
           requests:
             cpu: 25m

decisions/2026-09-30-flux-deploy-notification-inhibited.md

@@ -0,0 +1,95 @@
+---
+date: 2026-09-30
+branch: fix/flux-deploy-notification-inhibited
+title: The deploy notification routed correctly and was then inhibited before sending — `severity: info` is a suppressed class in kube-prometheus-stack
+---
+
+## What changed
+
+`FluxRolloutInProgress` (shipped hours earlier in PR #25) fired correctly on the very first real
+deploy and **its mail was suppressed before Alertmanager sent it**. One label changed:
+
+- `platform/monitoring/alerts.yaml` — `FluxRolloutInProgress` now carries `notification: deploy` and
+  **no `severity` label at all**, where it previously carried `severity: info`.
+- `platform/monitoring/kube-prometheus-stack.yaml` — the deploy route matches
+  `notification = "deploy"` instead of `severity = "info"`. Receiver, subject and grouping unchanged.
+- `platform/monitoring/tests/flux-rollout.test.yaml` — the three `FluxRolloutInProgress` `exp_labels`
+  blocks updated. `FluxRolloutStuck` keeps `severity: warning` and is unaffected.
+
+**The mechanism, measured rather than reasoned about.** kube-prometheus-stack's `general.rules`
+ships:
+
+    InfoInhibitor:
+      group by (namespace) (ALERTS{severity="info"} == 1)
+        unless on (namespace) group by (namespace)
+          (ALERTS{alertname!="InfoInhibitor", alertstate="firing", severity=~"warning|critical"} == 1)
+
+which fires for any namespace holding a firing `severity="info"` alert **and nothing else wrong in
+it**. The chart's third `inhibit_rule` then drops exactly that alert:
+
+    source_matchers: [alertname="InfoInhibitor"]
+    target_matchers: [severity="info"]
+    equal:           [namespace]
+
+For a deploy notification this is the worst possible interaction, because the suppressed case is the
+*only* case that matters: on a healthy rollout of a healthy cluster, `flux-system` holds no warning
+or critical alert, so `InfoInhibitor` fires and the deploy mail is inhibited. It would have been
+delivered only when something else in `flux-system` was already broken.
+
+Upstream's intent is explicit and reasonable — `severity: info` means "only worth showing next to a
+real problem". A deploy notice is not that, so it must not claim that severity. `notification` is a
+label **no** `inhibit_rule` in the chart targets (all three match on severity values), and dropping
+`severity` also stops `InfoInhibitor` firing for `flux-system` at all rather than merely working
+around it. Both files carry the full reasoning inline, and `alerts.yaml`'s design header now names
+the severity label as a third thing that must not regress — "do not tidy this by adding a severity"
+— because the next reader's instinct will be to add one back.
+
+## What I verified
+
+Against live prod, after PR #25 merged as `64c72fd` and Flux converged:
+
+- **The metrics work.** `kube_flux_kustomization_info` 16, `kube_flux_kustomization_suspended` 16,
+  `kube_flux_gitrepository_info` 1 = **33 series, exactly as designed**. The `revision` label IS
+  populated (`main@sha1:296d6d3…` / `…64c72fd…`), so the `status.lastAppliedRevision` /
+  `status.artifact.revision` paths are correct — previously UNVERIFIED.
+- **The `podAnnotations` roll mechanism works.** The KSM pod carries
+  `homelab.keithjackson.dev/crs-config-rollout: 1` and is a new pod
+  (`…-7b56d98f46-56n8j`, replacing the 2026-09-19 one), so the config-only-change trap was
+  genuinely closed rather than merely documented.
+- **The rule works, in both directions.** It fired for **12** lagging Kustomizations while 4 had
+  converged (`count by (revision)` → 12 old / 4 new), each alert carrying its own `name` and
+  `revision`; then the rollout converged (`count(count by (revision) (…))` → **1**) and the alert
+  **cleared**. Firing and self-clearing are both confirmed on real data, which is the completion
+  notification's whole mechanism.
+- **The routing works.** `amtool config routes` shows
+  `{notification="deploy"} → receiver: email-deploy` under the default route, and
+  `amtool config routes test notification=deploy` returns `email-deploy`.
+- **The inhibition was real.** `InfoInhibitor` went active `2026-09-30T04:58:48Z` — the same moment
+  the alert began firing — and `amtool alert query` showed **no** `FluxRolloutInProgress` in
+  Alertmanager, active or `--inhibited`. `alertmanager_notifications_failed_total{integration=email}`
+  is 0 for every reason, so nothing failed to send; it was never dispatched.
+- promtool: **SUCCESS**, 53 rules, 6 scenarios, promtool 3.14.0 in the pinned toolbox.
+
+## What I did NOT verify
+
+- **No deploy mail has been observed in the inbox — this fix is UNVERIFIED end to end.** The
+  suppression was proven (nothing in Alertmanager, zero send failures) and the routing was proven
+  (`amtool config routes test` → `email-deploy`), but the delivered message is not. It can only be
+  proven by merging this and watching the next commit produce `[FLUX] applying` and then
+  `[FLUX] applied`. Same structural gap as PR #25, one layer further along.
+- **Whether the single email during the window was ours.**
+  `alertmanager_notifications_total{integration="email"}` went 226 → 227 while other alerts
+  (`Watchdog`, `CNPGPrimariesCoLocated`) were also active, and that counter carries no receiver
+  label, so the increment cannot be attributed. I am NOT claiming our mail was or was not sent — the
+  attribution is genuinely unavailable from this metric.
+- **Whether `InfoInhibitor` interacts with `FluxRolloutStuck`.** Stuck is `severity: warning`;
+  inhibit_rule 1 (`critical` → `warning|info`) requires an equal `alertname`, so it should not
+  apply — reasoned, not measured. Worth confirming the first time a rollout genuinely hangs.
+- **The `[FLUX] applying|applied` subject template** is still untested; `.Status` is assumed to be
+  `firing`/`resolved` in the notification template data.
+- **`required_approvals` disagreement, unresolved and not mine to settle.** Live branch protection
+  reads `required_approvals: 1`, while the `ci/gate.sh` comment introduced by PR #17 describes the
+  model as `required_approvals: 0` ("the gate is the approver"). `ci/gitea-settings.sh` may report
+  this as drift against `infra/gitea/expected-branch-protection.json`. Flagged only.
+- **The drift loop is still unexplained.** Five Kustomizations re-applying the same object every
+  10 minutes (event counts ~2500) — untouched, cause unknown, still deserves its own branch.

decisions/2026-09-30-jackett-oauth2-proxy-session.md

@@ -0,0 +1,273 @@
+---
+date: 2026-09-30
+branch: fix/jackett-oauth2-proxy-session
+title: Jackett — radarr's oauth2-proxy deletes jackett's session cookie; the fix cannot be made per-app
+---
+
+## What changed
+
+Reported as *"GET requests work, but most PUT/POST requests to add indexers fail. Reloading the page
+never fixes it, and I am never re-prompted for login if it is a login issue."*
+
+**This branch lands the diagnosis and a runbook. It does NOT land the manifest fix, and that is a
+deliberate stop, not an omission** — see *The fix has one home and it is shared*, below.
+
+Three mechanisms, all measured. **Weighting matters here, so it is stated up front:** mechanisms 2
+and 3 are demonstrated from Jackett's own access log and **between them fully explain the report as
+written** — PUT/POST fail, the operator is never re-prompted, and a reload does not help. Mechanism 1
+is a real, firing defect that Jackett genuinely violates, and it explains the *intermittency* on top;
+I demonstrate the mechanism and one three-second causal adjacency, not a population of them. Fix all
+three; do not rest the case on the first.
+
+**1. Cookie collision — the mechanism is firing, and Jackett is a genuine violator.** Six oauth2-proxy
+instances (seven pods) emit oauth2-proxy's **default** cookie name `_oauth2_proxy` scoped to
+`.home.keithjackson.dev` while holding seven **different** cookie-secrets. The ping-pong is in two
+pods' own logs, interleaved to the second:
+
+    15:23:17  radarr   [AuthSuccess]                                   -> radarr writes _oauth2_proxy
+    15:23:44  jackett  "cookie signature not valid, removing session"  -> re-login #1
+    15:23:58  radarr   "cookie signature not valid, removing session"  -> radarr DELETES jackett's
+    15:27:17  jackett  [AuthSuccess]                                   -> re-login #2
+    15:27:59  radarr   "cookie signature not valid, removing session"
+    15:31:06  jackett  [AuthSuccess]                                   -> re-login #3
+    15:31:29  radarr   "cookie signature not valid, removing session"  <- deletes it again
+    15:31:32  jackett  "No valid authentication in request"            <- 3 SECONDS later, no cookie
+
+`removing session` is oauth2-proxy calling ClearSession, which expires the cookie on **its own**
+`--cookie-domain` — the whole zone. So jackett's session is not merely overwritten, it is **deleted
+by a process in another namespace**, 26 s after a clean login whose own `expires` was five minutes
+out. That is the answer to the open question in the brief ("26 s is well inside the 5-minute window,
+so plain expiry does not explain it"): it was never expiry, and it was never Jackett.
+
+It also explains the half of the report that looked like user error. **Reloading genuinely does not
+help**, because the operator's Radarr tab re-clears the cookie seconds after each new login — and
+Radarr is not a passive tab. Its UI polls: of 156 unauthenticated requests in its log against
+**1** AuthSuccess, the top paths are `/api/v3/command` (62), `/api/v3/queue/details` (46) and
+`/signalr/messages/negotiate` (43). Every one of those polls, arriving with a cookie Radarr cannot
+decrypt, calls ClearSession again. **A left-open Radarr tab is a cookie-deletion engine firing every
+few seconds**, which is why Jackett's sessions die at irregular, short, apparently random intervals.
+
+**A counter-hypothesis was put to me and it does not survive bounding.** The observation was that
+Crafty already carries this fix yet showed the platform's worst failure ratio (81:1 over 24 h from
+Loki), implying the fix does not work. But Crafty's pod started **2026-09-30T03:57:19Z with 0
+restarts**, so a 24 h window is ~10 h of the *pre-fix* pod. Rather than bound it in Loki I read the
+ratio off the pod's own log, whose boundary is the process start and therefore exact: **inside the
+fixed pod's entire 13 h 52 m lifetime, Crafty logged 3 `No valid authentication` events, not 81** —
+from one client IP, two of them WebSocket `/ws` reconnects, in two isolated minutes (04:02 and
+15:22), against 0 AuthSuccess because nobody signed in. Post-fix Crafty is idle, not broken. The
+ratio metric is confounded by endpoint type — apps that poll or hold WebSockets (Radarr, Crafty)
+manufacture failure lines at their retry rate — so it ranks *chattiness*, not breakage. Correctly
+bounded, Crafty is evidence **for** the fix.
+
+**2. Session lifetime pinned to a 5-minute access token.** Measured from the sidecar's own
+`[AuthSuccess]`: `created:15:31:06.391860254` / `expires:` + **300.000 s** exactly, with
+`refresh_token:true` on the same line. `--cookie-refresh` is unset and its default is `0`
+(disabled), so the proxy holds a refresh token it has no mandate to spend.
+
+**3. XHRs answered with 302 instead of 401 — why none of it was visible.** oauth2-proxy's built-in
+AJAX guard fires only on an **exact** `Accept: application/json`, which Jackett's jQuery does not
+consistently send. Measured on `/api/` paths in this pod's log: **33 × 302 against 2 × 401**, and the
+split runs straight through Jackett's own POSTs (`/api/v2.0/server/adminpassword` → 401,
+`/api/v2.0/server/update` → 302). A 302 on an unauthenticated XHR is followed invisibly by the
+browser, satisfied silently by the live Keycloak SSO session, and resolves with HTML where the
+JavaScript expected JSON — no prompt, no error. On a write it is worse: a browser following a 302
+re-issues a POST as a **GET and drops the body**, so the indexer configuration is discarded with
+nothing logged as a failure anywhere.
+
+### The fix has one home, and it is shared — this is the blocker
+
+The brief assigned me `apps/wave-0/jackett.yaml` as exclusively mine, and assigned the other four
+apps to a sibling branch. **That split is not implementable, and I found out by trying it.**
+
+`apps/wave-0/jackett.yaml` is a **generated golden**. The sidecar's argument list is built by one
+shared function, `_oauth2_sidecar_container()` in `catalog/render.py`. `catalog/jackett.yaml`
+declares only `auth.mode` and `auth.groups`, and the `auth` schema is `additionalProperties: false`
+— **there is no per-app override hook**. I wrote the fix into the rendered file first; the merge
+gate rejected it exactly as designed:
+
+    [fail] GOLDEN DRIFT: apps/wave-0/jackett.yaml does not match the canonical render
+    [fail] stage render-golden: FAIL          ->  MERGE GATE: BLOCKED   (rc 1)
+
+So I made the change at its real source instead and **measured the blast radius** before committing
+to it. A change confined to that one function rewrites the sidecar block in **all five** rendered
+apps, and `render-golden` requires all five be regenerated in the same commit:
+
+    CHANGED: apps/wave-0/jackett.yaml
+    CHANGED: apps/wave-0/pihole.yaml
+    CHANGED: apps/wave-0/qbittorrent.yaml
+    CHANGED: apps/wave-1/radarr.yaml
+    CHANGED: apps/wave-1/sonarr.yaml
+
+Four of those five are the files I was explicitly told not to touch, and all four are being edited
+right now on `fix/oauth2-proxy-cookie-collision-fleet`. **The gate and the ownership boundary are in
+direct contradiction**: I cannot fix Jackett's sidecar without regenerating four files another
+branch owns, and two branches rewriting the same generated block across five files is precisely the
+cross-branch conflict this repo's §7 exists to prevent.
+
+I stopped rather than force it. **The cookie fix is one change, not five**, and it should be re-cut
+as a single renderer commit on one branch. The intended content, validated below:
+
+    --cookie-name=_oauth2_proxy_{name}        # per-app, derived
+    --cookie-domain={host}                    # host-scoped, derived
+    --whitelist-domain=.{ZONE}                # unchanged: redirect targets, not cookie scope
+    --cookie-refresh=2m30s                    # half the measured 300 s token lifetime
+    (drop --pass-access-token, --set-xauthrequest)
+
+`--api-route=^/api/` is the one flag I would **not** apply renderer-wide on this evidence: I
+measured the 302/401 split on Jackett only, and PiHole's UI does not serve `/api/` at all. It needs
+either a per-app hook or four more measurements.
+
+**Metrics — asked for, and declined here for the same structural reason, not on merit.** It is true
+and I re-measured it: **no per-app sidecar exposes metrics**. Only the shared proxy carries
+`--metrics-address=0.0.0.0:44180` and a `metrics` container port; jackett, pihole, qbittorrent,
+radarr, sonarr and crafty all have `(ABSENT)` and ports `['http']` alone, so the listener never
+starts and nothing about these gates is scrapable. Adding it is worth doing and the pattern is
+already proven in-tree by the shared proxy. But it is **not** "inside my file": the container's
+`args` *and* its `ports` both come from `_oauth2_sidecar_container()`, so adding
+`--metrics-address` plus a `metrics` port hits the identical five-file blast radius as the cookie
+change. It belongs in the same single renderer commit — which is an argument for cutting that commit
+soon, not for me to hand-edit a golden a second time after the gate already refused the first.
+(Crafty is separately hand-written and could take the metrics flag on its own.)
+
+**Crafty is unaffected by all of this** — its sidecar is hand-written in
+`platform/crafty/deployment.yaml`, not rendered, which is why it could be and was fixed alone on
+2026-09-29.
+
+### What this branch does land
+
+`runbooks/oauth2-proxy-session.md` — the four sidecar log lines that tell Keycloak, the sidecar and
+the app's own login apart (including the upstream `-` field that means the proxy answered a request
+the app never saw), the cookie-collision cross-log diagnosis, the 5-minute-session check, the
+302-vs-401 trap, and the generated-golden warning with its measured blast radius. Registered in the
+runbooks `configMapGenerator` **and** listed in `runbooks/README.md`, so the URL resolves rather
+than the file merely existing.
+
+## What I verified
+
+- **The collision is the live failure.** `[stored_session.go:94] Error loading cookied session:
+  cookie signature not valid, removing session` — **1 occurrence in jackett's log (15:23:44), 8 in
+  radarr's** (15:22:49, 15:23:58, 15:27:59, **15:31:29**, 15:32:29 ×4), and **zero** in pihole,
+  qbittorrent, sonarr, crafty and both shared-proxy replicas. The 15:31:29 → 15:31:32 pair is the
+  operator's whole report.
+- **Six instances / seven pods are in the collision set** — the brief said five; the shared
+  forward-auth proxy in namespace `oauth2-proxy` is the sixth and runs 2 replicas. Read from live
+  pod specs: jackett, pihole, qbittorrent, radarr, sonarr and `oauth2-proxy` all have **no**
+  `--cookie-name` argument and `--cookie-domain=.home.keithjackson.dev`. Only crafty is fixed.
+- **All seven cookie-secrets differ.** Compared in-cluster by sha256, first 12 hex only, values
+  never printed: `d9c5…`/`0b3b…`/`b025…`/`bb6a…`/`25f4…`/`e679…`/`911a…`. So no two of these proxies
+  can read each other's cookie — the precondition for the eviction.
+- **The 5-minute lifetime**, from four `[AuthSuccess]` lines: `expires − created = 300.000 s`, with
+  `refresh_token:true` and `groups:[platform-admins]`. Live args confirm `--cookie-refresh` absent.
+- **The 302/401 split** on `/api/` paths in this pod's log: 33 × 302, 18 × 200, 2 × 401, 2 × 204.
+- **4 of the 6 POSTs never reached Jackett** — the access log's upstream field is `-`, not `/`:
+  `adminpassword` → 401, `adminpassword` → 401, `server/update` → 302, `server/update` → 302. The
+  two that did reach it (`adminpassword`, `indexers/1337x/config`) carry the authenticated email.
+- **The proposed argument list starts, against the pinned image and the real issuer.** Ran
+  `quay.io/oauth2-proxy/oauth2-proxy:v7.8.1` locally with the full proposed args; OIDC discovery
+  against `keycloak.home.keithjackson.dev` succeeded and it printed back its own reading:
+  `Cookie settings: name:_oauth2_proxy_jackett secure(https):true httponly:true expiry:168h0m0s
+  domains:jackett.home.keithjackson.dev path:/ samesite: refresh:after 2m30s` and
+  `API route - Path: ^/api/`.
+- **`--api-route` is singular.** The brief specified `--api-routes`, which does not exist in v7.8.1;
+  the proxy would have refused to start. Read off `--help`.
+- **`--session-cookie-minimal` is mutually exclusive with `--cookie-refresh`** — provoked, not
+  assumed: `invalid configuration: cookie_refresh > 0 requires oauth tokens in sessions.
+  session_cookie_minimal cannot be set`. This kills the brief's premise that dropping
+  `--pass-access-token` shrinks the cookie: only `--session-cookie-minimal` strips tokens from the
+  session, and it is unavailable while refresh is on. Dropping those two flags is justified on
+  **least privilege and inertness**, not size.
+- **`--set-xauthrequest` is inert here**: HTTPRoute `jackett-9117` backendRef is
+  `Service/jackett port 4180`, there is no ext-authz attachment on the `shared` Gateway, so
+  `/oauth2/auth` is never called.
+- **`apps/**` is generated and the fix cannot be scoped to one app** — both halves measured, not
+  inferred: the gate's `render-golden` stage rejected the hand-edit, and re-rendering after a
+  renderer-only change listed all five apps as changed (transcript above).
+- **Crafty's post-fix failure rate, bounded exactly.** Pod `crafty-647c7d8cc5-5spv9`,
+  `startTime=2026-09-30T03:57:19Z`, `restarts=0`, live args `--cookie-name=_oauth2_proxy_crafty` and
+  `--cookie-domain=crafty.home.keithjackson.dev`. Over that pod's whole lifetime: **3**
+  `No valid authentication`, 0 AuthSuccess, one client IP, 2 of the 3 on `/ws`. The 81:1 figure is a
+  24 h window over a 13.9 h pod and is dominated by the pre-fix process.
+- **The ratio metric ranks chattiness, not breakage.** Radarr 156:1 with the top paths being
+  `/api/v3/command`, `/api/v3/queue/details` and `/signalr/messages/negotiate` — UI polling, not 156
+  distinct user-visible failures.
+- **No per-app sidecar exposes metrics.** `--metrics-address` is `(ABSENT)` and `ports` is
+  `['http']` on all six per-app sidecars; only the shared proxy has
+  `--metrics-address=0.0.0.0:44180` and a `metrics` port.
+- **Gate:** `./ci/gate.sh` in the pinned toolbox, **rc 3 — UNDETERMINED, not green**. Green:
+  `spec-immutability`, `approval-policy`, `decisions-log`, `lint`, `l0`, `render-golden`, `l1`.
+  Skipped: `gitea-settings` (no `SETTINGS_API_TOKEN`) and L2/L3 conformance (`CLUSTER_AVAILABLE`
+  unset), leaving touched requirement `R-OBS-04` uncertified. Note `spec-immutability` is **green**,
+  not skipped — mounting the sibling spec repo at `/home-network-k3s` (the path
+  `ci/spec-immutability.sh` actually defaults to) in the ENGINE shim fixes that skip; the commonly
+  circulated shim mounts it at its host path, where `SPEC_REPO` never finds it.
+- **Gitea CI on this PR, read from the commit's `/statuses` rather than from a tick.** Two contexts
+  reported, and **only one of them ran**:
+
+      gate / gate (pull_request)         state=success   desc='Successful in 1m41s'
+      gate / conformance (pull_request)  state=success   desc='Has been skipped'
+
+  That second line is §2b's trap in the wild: a **skipped** Gitea job reports `state=success`. The
+  conformance job did not execute, so the green tick beside it certifies nothing. Read together with
+  the local rc 3 above, the two agree exactly — the offline L0/L1 subset passed and L2/L3 never ran.
+  Anyone reading the PR's check mark alone would conclude otherwise.
+- **`main` is protected**, re-measured this session rather than assumed:
+  `GET /repos/.../branches/main` → `protected=true`, `user_can_push=false`.
+
+## What I did NOT verify
+
+- **The operator's problem is NOT FIXED by this branch.** Only the diagnosis and the runbook land.
+  Jackett's sidecar is unchanged and will keep losing its session to Radarr until the renderer
+  change ships. **This is the single most important line in this entry.**
+- **No browser round-trip.** Every mechanism is measured from logs and specs and the argument list
+  is proven to start, but nobody has added an indexer through the proposed configuration. The human
+  test is below, for whoever lands the renderer change.
+- **Nothing was applied.** No `kubectl apply/edit/patch`, no `flux reconcile`. The running pod is
+  still `jackett-c5bfcbfb8-c75f9` (2/2 Running, 17 d, 0 restarts — unchanged by this session's
+  read-only probes). Note that when the fix does land, merging is what deploys, and it will restart
+  the pod (`strategy: Recreate` on an RWO Longhorn volume), briefly interrupting Radarr's and
+  Sonarr's direct `:9117` access.
+- **Cookie size and chunking: UNMEASURED, and I will not claim either way.** I could not read a
+  `Set-Cookie` header without a browser. I found no `_oauth2_proxy_0`/`_1` chunk evidence and no
+  4 kB warning in 36,287 log lines — but I did **not** establish that v7.8.1 emits such a warning at
+  this log level, so that absence is not evidence. If sessions still misbehave after the fix lands,
+  this is the next thing to measure, with `--session-store-type=redis` as the escape hatch, since
+  `--session-cookie-minimal` is ruled out (above).
+- **`--cookie-refresh=2m30s` has never executed a refresh here.** It depends on Keycloak accepting
+  the refresh token and on the realm's SSO Session Idle/Max, none of which I read (that needs admin
+  credentials). If Keycloak's *refresh* token is also short-lived, the five-minute logout persists
+  and the realm is where to look next.
+- **`--api-route` is measured for Jackett only.** PiHole and qBittorrent were not measured, and
+  PiHole's UI does not serve `/api/`. Do not apply it renderer-wide on my evidence.
+- **The four sibling apps still collide with each other** regardless of what happens to Jackett.
+  The same operator report should be expected from any of pihole / qbittorrent / radarr / sonarr.
+- **L2/L3 conformance did not run** (`CLUSTER_AVAILABLE` unset), so nothing about runtime behaviour
+  is certified by the gate. `gitea-settings` also skipped, so branch protection and runner state are
+  **unverified by me this session** — I did not re-measure them against the Gitea API.
+- **I did not query Loki.** The Crafty bounding above comes from the pod log, which I argue is the
+  better instrument (an exact process boundary rather than a wall-clock window against a 72 h
+  retention). But it means I cannot speak to the 24 h or 12 h *platform-wide* ratios, and a
+  per-app ratio over a window longer than the current pod's life is not something I measured at all.
+- **Collision causation is one adjacency, not a population.** The 15:31:29 → 15:31:32 pair is three
+  seconds and the mechanism is documented and present, but I did not enumerate every jackett failure
+  and match it to a radarr ClearSession. Treat mechanism 1 as **necessary and firing, not proven
+  sufficient** — the coordinator's framing, and I agree with it on this evidence.
+- **The metrics change is NOT in this branch** (reason above). Jackett therefore remains inside the
+  oauth2-proxy metrics gap, and the ratio alert shipped in PR #29 still cannot see it.
+- **The runbook's commands were not all executed verbatim.** The log-reading ones I ran; the
+  per-namespace `for` loops are edited from the scripted forms I actually used.
+
+### The human test — for whoever lands the renderer fix
+
+After it merges and the pod rolls (watch for a **new** pod name at 2/2):
+
+1. **Sign in** at `https://jackett.home.keithjackson.dev/UI/Dashboard`.
+2. **Open Radarr in a second tab** and sign in there too. **This is the step that breaks Jackett
+   today** — it is the 15:31:29 line. Return to the Jackett tab.
+3. **Add an indexer** — *+ Add indexer*, pick one, *Configure*, *Okay*. It must save.
+4. **Wait six minutes on the open dashboard, then save another.** That is the `--cookie-refresh`
+   test; the session currently dies at five.
+5. **If something does still fail, it must now fail LOUDLY** — DevTools → Network, a broken `/api/`
+   call shows **401**, not a 302 followed by HTML. A silent success-looking no-op means the change
+   did not take; check the running pod's args before assuming anything else.
+6. **A plain page reload must still sign you in silently** (`/UI/Dashboard` is outside `^/api/`).
+   A 401 page on reload means the regex is too broad — that is a regression.

decisions/2026-09-30-node-health-render-device.md

@@ -0,0 +1,283 @@
+---
+date: 2026-09-30
+branch: feat/node-health-render-device
+title: The Intel render device is now self-re-measuring, not a fact someone checked once
+---
+
+## What changed
+
+### Why this probe has value at all
+
+**Nothing else in the platform would notice the device going missing.** Plex exposes no
+ServiceMonitor, so no alert fires on a silent fall back to software transcoding. And the
+manifest side mounts the device with `hostPath type: DirectoryOrCreate`, which means a node
+*without* the device mounts an **empty directory** — Plex starts, runs, and degrades quietly
+rather than failing loudly. There is no crash loop, no failed mount, no alert: just a hot CPU.
+This check is the only thing in the system that would surface it, which is the actual
+justification for the role carrying it.
+
+### The change
+
+`roles/node_health` grew a graphics / render-device probe. Nothing was installed, no module
+was loaded, no group was changed, nothing was rebooted: the prod nodes already have a working
+Intel render device (measured by hand on all three on 2026-09-30) and the point of this change
+is that the measurement now **re-runs forever** instead of decaying silently in a handoff note.
+
+The failure this exists to catch is silent by construction. Plex hardware-transcodes through
+`/dev/dri/renderD128`; if that node stops existing — node rebuild, a BIOS change that disables
+the onboard IGD, a kernel that binds a different DRM driver, a driver regression — Plex falls
+back to software transcoding on its own and the only symptom is a hot CPU.
+
+Four files:
+
+- `files/node_health_collect.py` — new `collect_graphics()`, wired into the **unprivileged**
+  pass (`main()`). Everything it reads is world-readable: `lstat()` on `/dev/dri/*`,
+  `grp.getgrgid()` for the owning group name, the `/sys/class/drm/card*/device/driver` symlink
+  target for the bound driver, `/sys/bus/pci/devices/*/class` for display-class (PCI base class
+  `0x03`) devices whether or not a driver is bound, `/sys/class/drm/card*/engine/*` for the
+  `vcs*`/`vecs*` media engines, `/proc/modules`, `/proc/cmdline`, and
+  `/sys/module/{i915,xe}/parameters/enable_guc`. `lspci` and `vainfo` were added to the
+  existing tool probe; both are optional and neither is depended on.
+- `tasks/assess.yml` — one shared derive task plus three checks: `graphics.render_device`
+  (PASS/FAIL/SKIP), `graphics.drm_driver` (PASS/WARN/FAIL/SKIP), `graphics.media_stack`
+  (INFO/SKIP).
+- `defaults/main.yml` — `node_health_graphics_expected_driver: i915`, so the one comparison the
+  check makes is data rather than task logic.
+- `templates/summary.txt.j2` — a `GRAPHICS / RENDER DEVICE` section, and
+  `roles/node_health/README.md` gained a section explaining the design and two new "known
+  limits".
+
+Three design rulings worth stating, because each is the kind of thing a later edit would get
+wrong:
+
+1. **The render node's GID is reported, never asserted.** It read `root:render`, gid **992**,
+   mode `0660` on all three prod nodes. The check does *not* compare against `992`. `render` is
+   a **systemd dynamic** allocation (`systemd-sysusers`), not a Debian static gid from
+   `base-passwd`, so a rebuild can legitimately move it — and Plex's container entrypoint stats
+   the device and joins whichever group owns it, so drift genuinely is not a fault. The
+   observed group name and number appear in the detail line, which makes drift *visible*
+   without making it an error. A hard `!= 992` would be a false alarm waiting for a reinstall.
+2. **The PCI scan is deliberately separate from the DRM scan.** `/sys/class/drm/card*` exists
+   only once a driver has *bound*, so a GPU with nothing attached has no `card0` and would
+   otherwise be indistinguishable from a machine with no GPU. Scanning the PCI bus separately
+   is what lets "no display device at all" be an honest **SKIP** (a VM guest — the same
+   convention `thermal.max` already uses for "no sensor on this hardware") while "a GPU is on
+   the bus and there is still no render node" is a **FAIL**.
+3. **GuC/HuC and host VAAPI userspace are INFO and must stay INFO.** GuC/HuC off
+   (`i915.enable_guc=0`) is an accepted state here and costs low-power H.264 encode *quality*
+   only. No host `vainfo` is not a defect at all — Plex ships its own libva and media driver
+   inside its container. Grading either would put a permanent WARN on a chosen configuration,
+   or invite someone to `apt install vainfo` on a node to clear a health check, and this role
+   installs nothing.
+
+`xe` bound instead of `i915` is a **WARN**, not a FAIL: the device still works, but the two
+drivers do not share a userspace media stack. On these nodes `xe.ko.xz` exists on disk and is
+bound to nothing.
+
+## What I verified
+
+### The real prod run (read-only, all three nodes)
+
+`--list-hosts` first, every time. It named exactly three hosts and nothing else:
+
+    play #1 (k3s_prod): Inspect node hardware health (read-only)
+        pattern: ['k3s_prod']
+        hosts (3): k3s-prod-3  k3s-prod-2  k3s-prod-1
+
+Then `--syntax-check` (rc 0), then the run:
+
+    ansible-playbook playbooks/node-health.yml -e node_health_hosts=k3s_prod --limit k3s_prod
+
+    k3s-prod-1 : ok=51  changed=3  unreachable=0  failed=0  skipped=1
+    k3s-prod-2 : ok=51  changed=2  unreachable=0  failed=0  skipped=1
+    k3s-prod-3 : ok=51  changed=2  unreachable=0  failed=0  skipped=1
+    PLAY rc=0
+
+(The `changed=` counts are the *controller-side* artifact writes under
+`infra/ansible/_artifacts/node-health/`, delegated to localhost. Nothing on any node changed —
+the collector task is `changed_when: false` and the role touches nothing.)
+
+All three nodes graded **PASS / PASS / INFO** on the new checks, 0 FAIL overall:
+
+    k3s-prod-1  VERDICT  0 FAIL · 1 WARN · 18 PASS · 2 SKIP · 6 INFO
+    k3s-prod-2  VERDICT  0 FAIL · 3 WARN · 15 PASS · 3 SKIP · 6 INFO
+    k3s-prod-3  VERDICT  0 FAIL · 2 WARN · 17 PASS · 2 SKIP · 6 INFO
+
+    [PASS] graphics.render_device — DRM render node present for hardware transcoding (/dev/dri/renderD*)
+    [PASS] graphics.drm_driver   — DRM driver bound to the display device (expected i915)
+    [INFO] graphics.media_stack  — Media engines, GuC/HuC firmware and host VAAPI userspace (informational)
+
+(The pre-existing WARNs and SKIPs are other checks — SMART, memattr, resolvers — and are not
+mine; none of them moved.)
+
+The new report section, **byte-identical on all three nodes**:
+
+    --------------------------------------------------------------------------------
+     GRAPHICS / RENDER DEVICE (the Plex hardware-transcoding path)
+    --------------------------------------------------------------------------------
+     pci  0000:00:02.0 00:02.0 VGA compatible controller [0300]: Intel Corporation CoffeeLake-S GT2 [UHD Graphics 630] [8086:3e98] (rev 02)
+                  driver i915
+     drm  card0        driver i915     at 0000:00:02.0  engines bcs0 rcs0 vcs0 vecs0
+     dev  card0        character special  mode 0660  root:video gid 44
+     dev  renderD128   character special  mode 0660  root:render gid 992
+     GuC/HuC     NOT READ — /sys/module/i915/parameters/enable_guc is not
+                  readable without root (i915 module parameters are mode 0400)
+                 no i915/xe parameter on the kernel cmdline
+     modules     amdgpu=false, i915=true, nouveau=false, nvidia=false, radeon=false, xe=false (/proc/modules — corroboration only; the bound-driver symlink above decides)
+     vainfo      NOT installed on this host — not a defect: Plex ships its own libva/media driver inside its container
+
+That reproduces every item of the 2026-09-30 hand measurement independently: `renderD128`
+character special, `root:render`, **gid 992**, mode `0660`; `card0` `root:video` gid 44 mode
+`0660`; `i915` bound; `xe` **not** loaded and bound to nothing; `[8086:3e98] (rev 02)`
+CoffeeLake-S GT2 UHD Graphics 630 at `00:02.0`; `vcs0` and `vecs0` live; no host `vainfo`.
+
+**One measured correction to my own phase-1 assumption:** `lspci` **is** present on all three
+prod nodes (`/usr/bin/lspci` — Debian ships `pciutils` by default). I had written that it was
+absent. It is still *treated* as optional, because `pciutils` is genuinely not declared in
+`node_baseline_packages` and depending on an undeclared package would be wrong — but the
+comments and README that asserted it was missing were factually wrong and are now corrected.
+Its presence is why the `pci` line above carries a readable device name rather than bare IDs.
+
+### The gate
+
+    DEV_KUBECONFIG=/dev/null ENGINE=<shim> ./infra/devcontainer/dev.sh run bash -c './ci/gate.sh'
+    GATE rc=3
+
+**rc 3 — UNDETERMINED. NOT green.** Nothing failed, and nothing is certified either. Exactly
+three stages did not run, all three the known local ceiling from a worktree, and **no fourth
+skip**:
+
+    [skip] spec-immutability  spec repo not present at /work/../home-network-k3s (set SPEC_REPO)
+    [skip] gitea-settings     SETTINGS_API_TOKEN is not set — cannot read the live Gitea rule
+    [skip] conformance        no cluster configured (CLUSTER_AVAILABLE!=true) — L2/L3 NOT RUN
+
+    [ ok ] decisions-log: pass      [ ok ] lint: pass       [ ok ] l0: pass
+    [ ok ] render-golden: pass      [ ok ] l1: pass         [ ok ] approval-policy: pass
+
+Both CLAUDE.md §3 worktree workarounds were needed and both worked: `DEV_KUBECONFIG=/dev/null`
+(the credential in the main checkout was **not** copied) and an `ENGINE` shim that bind-mounts
+the main checkout's `.git` read-only. No `tracked_files()` traceback appeared, which is how I
+know the shim took effect rather than the git stages silently degrading.
+
+`ansible-lint` was **not** run, because **`ci/` does not invoke it anywhere** — I grepped
+`ci/` and `.gitea/` and the lint stage is `yamllint -c ci/yamllint.yaml` only. I am not
+claiming to have matched a lint convention that does not exist.
+
+### The CI gate on the PR — a green tick that is exit 3, read from the log
+
+The required context `gate / gate (pull_request)` reported **`success`, "Successful in 1m40s"**
+on commit `a78f010`. That description matters: a *skipped* Gitea job reports `success` with
+"Has been skipped", so this one genuinely ran, and 1m40s matches CLAUDE.md §2's ~1m35s.
+
+**But the green tick does NOT mean `gate.sh` returned 0.** `.gitea/workflows/gate.yml:138` is
+`run: ./ci/gate.sh || [ "$?" -eq 3 ]` — the job tolerates rc 3 by design. So I fetched the job
+log (`/actions/runs/173/jobs/0/logs`; the `/api/v1/.../tasks/{id}/logs` endpoints 404 on this
+Gitea, the web route serves it) and read the gate's own headline:
+
+    [fail] MERGE GATE: UNDETERMINED — greenness was NOT ESTABLISHED.
+    [fail]   Stages that did not run: spec-immutability
+
+**So CI is also rc 3, with exactly ONE stage skipped — `spec-immutability`.** The green Gitea
+check is the tolerance swallowing that 3, not a rc 0.
+
+**CI verified two things my local run could not, and both came back green:**
+
+    [ ok ] stage gitea-settings: pass
+    [ ok ]   enable_push = false          enable_status_check = true
+    [ ok ]   required_approvals = 1       block_admin_merge_override = true
+    [ ok ]   block_on_rejected_reviews = true   dismiss_stale_approvals = true
+    [ ok ]   runners online: 1 (>= 1)
+    [ ok ] Gitea settings match the declaration.
+    [ ok ] stage approval-policy: pass
+    [ ok ] no architecturally/security-relevant paths touched — the GATE is the approver.
+
+My local run **skipped** `gitea-settings` (no `SETTINGS_API_TOKEN`) and could not give
+`approval-policy` a PR number. CI has the secret and `REQUIRE_GITEA_SETTINGS: "true"`, so a skip
+there would have failed the job. That is measured confirmation that the live Gitea matches
+`infra/gitea/expected-branch-protection.json`, and that `infra/ansible/**` needs no human
+reviewer. `lint` passed over 405 files. `conformance` SKIPPED in CI too
+(`CLUSTER_AVAILABLE != true`), so L2/L3 remains unverified there as well.
+
+Corroborated independently: `GET /repos/.../branches/main` →
+`protected=true user_can_push=false enable_status_check=true required_approvals=1`,
+contexts `["gate / gate (pull_request)"]`; `GET /admin/actions/runners` → `incluster-dind
+status=online`. And note the commit's **combined** state is `pending`, not success, because
+`gate / conformance (pull_request)` sits at "Blocked by required conditions" — that context is
+not in the required list, so it does not block, but the PR is not "all green" either.
+
+The gate's `lint` stage passed with one `line too long` **warning** on
+`roles/node_health/tasks/assess.yml:189` — that is pre-existing, in the SMART block, well
+above my insertion point, and warnings do not fail the stage.
+
+### Offline work (before the lock was released)
+
+- `python3 -m py_compile` on the collector: clean.
+- `collect_graphics()` executed for real against this workstation's hardware (an unrelated
+  machine, Raptor Lake-S UHD Graphics + an NVIDIA discrete GPU) and returned the whole
+  structure with no exception, correctly reporting two cards, two render nodes, the bound
+  driver per card, the `vcs0/vcs1/vecs0` media engines, and `enable_guc` as *not readable
+  without root* rather than as a fabricated `0`.
+- That last point is a real finding and changed the design: `/sys/module/i915/parameters/*` is
+  **mode 0400**, so an unprivileged pass cannot read `enable_guc`. It now reports
+  `NOT READ — <reason>` and `/proc/cmdline` is read as the world-readable second source. An
+  earlier draft would have silently reported a null.
+- Two other bugs found and fixed by running it rather than reasoning about it:
+  `/dev/dri/by-path` is a **directory** and was being `lstat()`ed as a device; and
+  `/sys/class/drm` carries one entry per *connector* (`card0-HDMI-A-1`), which had to be
+  excluded from the card scan.
+- All three checks and the new template section were rendered through a standalone Jinja
+  harness (Ansible is not on the host — CLAUDE.md §5) against **seven** fact sets: the real
+  workstation; GPU present with the render node removed; no display device at all; `xe` bound
+  in place of `i915`; a display device with **no** driver bound; `renderD128` present but not a
+  character device; and a shape with neither `lspci` nor `vainfo`. Zero template errors, and
+  the statuses came out as designed — `FAIL/PASS/INFO`, `SKIP/SKIP/SKIP`, `PASS/WARN/INFO`,
+  `FAIL/FAIL/INFO`, `FAIL/PASS/INFO` respectively. The harness supplies stand-ins for the
+  Ansible-only filters (`bool`, `dict2items`, `flatten`, `zip`, `regex_replace`) and matches
+  Ansible's `trim_blocks=True`.
+  **Read that for exactly what it is: evidence that the LOGIC is right, and no evidence
+  whatsoever that the HARDWARE DETECTION is.** Six of those seven fact sets were hand-mutated
+  dictionaries, not machines. They are not hardware tests and must not be written up as if
+  they were.
+
+## What I did NOT verify
+
+- **The FAIL path has never fired on a real node.** Every prod node has a working render
+  device, and this role is read-only, so a device cannot be taken away to prove the check
+  catches its absence. The failure branches were exercised against **synthetic fact sets
+  only** (see above). I am not claiming the failure path is tested against hardware.
+- **`xe` detection is likewise untested against reality.** No node here has `xe` bound — the
+  module exists on disk as `xe.ko.xz` and is bound to nothing — so the WARN branch was only
+  ever driven by a mutated fact set. If `xe` ever does bind, the *shape* of
+  `/sys/class/drm/card*` under it is something I have assumed matches i915's (same
+  `device/driver` symlink, same `engine/*` directories). That assumption is unverified.
+- **The media-engine sysfs path is confirmed on these nodes but not proven portable.**
+  `vcs0`/`vecs0` were read from `/sys/class/drm/card0/engine/*` on all three prod nodes, so it
+  is measured here. I did **not** confirm against kernel source that no other kernel version or
+  driver publishes engines elsewhere. If one does, the check reports "none exposed" — an INFO
+  line, not a false verdict, which is why this failure mode is tolerable.
+- **`enable_guc` is still unread on prod.** The report says `NOT READ — mode 0400`, which is
+  honest but means I have **not** confirmed by measurement that GuC/HuC is off on these nodes;
+  I only know the kernel cmdline carries no `i915.enable_guc=` override. The brief's ground
+  truth says `i915.enable_guc=0`; my check does not corroborate that value, it declines to
+  guess it. Reading it would need root, and it is INFO either way, so I left it alone.
+- **Not a transcode test.** These checks prove the render *device* exists, is a character
+  device, and which driver owns it. They do **not** prove a transcode succeeds — that needs a
+  VAAPI round trip inside Plex's own container with Plex's own libva, and is out of scope for a
+  node-level check that installs nothing.
+- **Greenness was never established, locally or in CI.** Both runs returned **rc 3**. My local
+  run skipped three stages (`spec-immutability`, `gitea-settings`, `conformance`); CI skipped
+  one (`spec-immutability`) and its own headline says `MERGE GATE: UNDETERMINED`. **I am not
+  describing this PR as green**, and the green Gitea tick must not be read as one: the workflow
+  line is `./ci/gate.sh || [ "$?" -eq 3 ]`, so it is a tolerated 3.
+- **L2/L3 conformance did not run anywhere** — `CLUSTER_AVAILABLE != true` in both. Whatever
+  runtime requirements this touches remain UNVERIFIED; only the L0/L1 offline subset passed.
+- **`spec-immutability` ran in neither.** The sibling spec repo is checked out in neither my
+  worktree nor the CI workspace, so nothing confirms this change does not drift from the spec.
+  It is the expected, tolerated skip — but it is still a skip, not a pass.
+- **No claim about the live platform beyond what was printed.** Per CLAUDE.md §2b this repo is
+  not evidence about prod; the only prod facts I assert are the ones quoted verbatim above.
+  Branch protection and runner state I did query directly, and the CI `gitea-settings` stage
+  passed — so those two I assert on measurement, not on this file's prose.
+- **The stale dev inventory was left alone deliberately.** `hosts.ini` still defines the
+  `k3s-dev-*` groups as the default target of most playbooks and the dev cluster no longer
+  exists. That is real, it is out of this change's scope, and it is unclaimed work.

decisions/2026-09-30-oauth2-proxy-auth-failure-visible.md

@@ -0,0 +1,289 @@
+---
+date: 2026-09-30
+branch: feat/oauth2-proxy-auth-failure-visible
+title: The Jackett auth failure was invisible because the gates emit nothing — alert on the gap, not on a metric that does not exist
+---
+
+## What changed
+
+Jackett's UI broke in the way that costs the most: **HTTP 200 and a plausible page**. The
+operator's report named the property that matters — *"reloading the page never fixes the issue and
+the user is never re-prompted for login"*. Two sibling branches fix the auth defects. This one is
+about the fact that **no signal existed anywhere**, and it turns out that is not an oversight in
+the alert rules. It is an absence of data.
+
+**The measurement that decided the whole design** (prod, live Prometheus, 2026-09-30):
+
+    count by (__name__) ({__name__=~"oauth2_proxy.*"})
+      oauth2_proxy_requests_in_flight => 2
+
+Two series. Both from the **shared** forward-auth Deployment in the `oauth2-proxy` namespace.
+That is every oauth2-proxy metric this cluster holds. The six per-app gate sidecars — crafty,
+jackett, pihole, qbittorrent, radarr, sonarr — publish **nothing at all**: they run with
+`--http-address=0.0.0.0:4180` and no `--metrics-address`, so oauth2-proxy's metrics listener never
+starts; they declare only a `http:4180` container port; and no ServiceMonitor or PodMonitor in the
+cluster selects them.
+
+So the alert the brief asked for — *"a gated app is serving a sustained rate of auth redirects to
+a user who is authenticated"* — **is not writable**, and writing it anyway would have produced
+exactly the artefact this repo keeps getting burned by: a rule that is present, evaluating,
+permanently silent, and indistinguishable from health. It was not written.
+
+What was written instead alerts on the condition that made the incident invisible, from families
+that **are** scraped:
+
+- **`platform/oauth2-proxy/auth-visibility-alerts.yaml`** — `OAuth2ProxyGateUnobserved`:
+
+      count by (namespace) (kube_pod_container_info{container="oauth2-proxy"})
+        unless
+      count by (namespace) (up{container="oauth2-proxy"})
+
+  `for: 1h`, `severity: warning`. `kube_pod_container_info` is kube-state-metrics (a declared
+  owner in `alert-metrics-have-scrape-targets`) and sees the container whether or not it exports
+  anything; `up` is UNIVERSAL and carries a `container` label for any scrape target, so the rule
+  needs no advance knowledge of what job name a future per-app scrape will use. Nothing in it
+  touches `oauth2_proxy_*`, so it cannot be inert for the reason it is reporting. **It returns six
+  namespaces against live prod right now** and excludes `oauth2-proxy`, which is scraped. It
+  clears per-namespace as each gate gains a real scrape.
+- **`platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml`** — four states pinned:
+  fires for an unscraped namespace, never for the scraped one, honours `for: 1h` (pending at 59m),
+  and **stops** once a scrape appears under a *different* job name. Series shapes copied off live
+  prod, not invented.
+- **`runbooks/oauth2-proxy-auth-failures.md`** — how to tell "the gate refused me" from "the group
+  check refused me" from "the app refused me" from "the XHR silently got HTML", with the
+  2026-09-30 jackett trace as the worked example. Registered in `runbooks/kustomization.yaml`'s
+  `configMapGenerator` and listed in `runbooks/README.md`.
+- **`runbooks/README.md`** — a new row in *"Alerts you should not trust the silence of"* for the
+  per-app sidecars, plus the measured reason.
+- **`platform/oauth2-proxy/README.md`** — what this component can and cannot see, and the exact
+  three-part change that would close the gap.
+
+**Why the obvious alternative was rejected.** An alert on the *shared* proxy's
+`oauth2_proxy_requests_total{code="302"}` ratio is the one a reader expects to see here. It would
+need a new `SCRAPE_OWNERS` entry in `conformance/**` (a sensitive path needing human approval),
+and — decisively — **it would not have fired on this incident**: the shared instance is not in the
+data path for any sidecar-gated app, and its current pods export no `oauth2_proxy_requests_total`
+at all, having proxied nothing since they started (the counter exists in the TSDB only from an
+older ReplicaSet, `oauth2-proxy-78449b5657-*`, with `code` labels `302` and `401` and no `202`).
+Shipping it would have looked like coverage of the reported failure and been none.
+
+## What I verified
+
+Everything below was measured against live prod through the pinned container, read-only. No
+`apply`, no `edit`, no `patch`, no `flux reconcile`.
+
+- **The full oauth2-proxy metric inventory.** `count by (__name__) ({__name__=~"oauth2_proxy.*"})`
+  → `oauth2_proxy_requests_in_flight => 2`. `count by (__name__) ({job="oauth2-proxy"})` returns
+  go_*/process_*/promhttp_* plus that one family and nothing else. **Only the shared instance is
+  scraped**: `up{job="oauth2-proxy"}` → 2 targets, `10.0.1.189:44180` and `10.0.2.47:44180`, both
+  `1`, `namespace="oauth2-proxy"`.
+- **The per-app sidecars are unscraped, and unscrapable as configured.** Every pod carrying an
+  `oauth2-proxy` container, with its declared ports: the two shared replicas have
+  `[('http',4180),('metrics',44180)]`; crafty, jackett, pihole, qbittorrent, radarr and sonarr all
+  have `[('http',4180)]` only. The live jackett sidecar's args contain no `--metrics-address`.
+  `kubectl get servicemonitors,podmonitors -A` lists 16 ServiceMonitors and 5 PodMonitors; the
+  only oauth2-proxy entry is `oauth2-proxy/oauth2-proxy`.
+- **No existing alert is inert *for this reason*, because no alert references these metrics at
+  all.** `grep -rn oauth2_proxy --include=*.yaml` over the tree hits only cookie-name strings in
+  `platform/crafty/`. The 39 live PrometheusRules include `platform-health-identity`, whose four
+  rules are about Keycloak readiness and the client reconciler and are all keyed on `kube_*`
+  (scraped, fine). **The inertness here is the absence of a rule, not a broken one** — which is
+  why the runbook README's inert-alerts table gains a row phrased like the etcd one.
+- **Keycloak cannot express it either.** `count by (__name__) ({__name__=~"keycloak_.*"})` →
+  empty, and empty over `[30d]`. The `keycloak-service` scrape yields JVM/Agroal/JGroups metrics
+  only; there are no user-event or login counters, so "a burst of re-logins for one client" is not
+  observable from the IdP side.
+- **`oauth2_proxy_requests_total` has existed historically but not on the current pods.**
+  `count_over_time(oauth2_proxy_requests_total[30d])` returns three series, all from
+  `oauth2-proxy-78449b5657-*` (a prior ReplicaSet): `code="302"` on both replicas and `code="401"`
+  on one, ~634 samples each (~5h of a 30d window). **No `code="202"` ever.**
+- **The alert expression fires against live data.** Run verbatim on prod:
+  `count by (namespace) (kube_pod_container_info{container="oauth2-proxy"}) unless count by
+  (namespace) (up{container="oauth2-proxy"})` → `pihole, jackett, crafty, qbittorrent, sonarr,
+  radarr`, value 1 each; `oauth2-proxy` correctly absent.
+- **The log signal exists and is collected.** Loki (`loki-0`, SingleBinary) and a 3-node Alloy
+  DaemonSet are running, and Alloy labels pod logs with `namespace`/`container`. LogQL over 24h,
+  `{container="oauth2-proxy"}`: crafty 12036 lines, jackett 11669, oauth2-proxy 34559, pihole
+  11520, qbittorrent 11520, radarr 11763, sonarr 11520. Filtered to `|= "No valid authentication
+  in request"`: **crafty 81, jackett 39, radarr 67**; to `|= "AuthSuccess"`: **crafty 1, jackett
+  4, radarr 1**. Those ratios (81:1, 10:1, 67:1) are the incident's signature and are in the
+  runbook as the thing to compare against.
+- **Why `kubectl logs` hid it.** The jackett sidecar's stdout is ~11.5k lines/day and almost all
+  of it is `GET "/ping" ... 200 2` from the kubelet probe, twice every 15s. A `--tail=40` shows
+  forty `/ping` lines and looks perfectly healthy. The runbook's commands all filter.
+- **Runbook registration, both halves.** `- oauth2-proxy-auth-failures=oauth2-proxy-auth-failures.md`
+  is in `runbooks/kustomization.yaml`'s `configMapGenerator.files`, and the page is listed under
+  *Identity* in `runbooks/README.md`. The page clears the L0 quality floor (the four operator
+  sections, >1500 bytes, mentions `dev.sh` and `--context k3s-prod`), and the alert's
+  `runbook_url` points at exactly that name.
+- **Gate:** run in the pinned toolbox from this worktree with the `ENGINE` shim. Exit code and
+  per-stage outcomes in the next section — read them there, not here.
+
+## What I did NOT verify
+
+- **The alert has never fired in Alertmanager.** Nothing is applied; the PrometheusRule exists
+  only in git on this branch. What is proven is that its expression returns the expected vector
+  against live prod data and that promtool agrees about its behaviour over synthetic series.
+  **Merging is what deploys it.**
+- **`OAuth2ProxyGateUnobserved` would NOT have fired *during* the 2026-09-30 jackett incident in
+  any useful sense** — it describes a standing condition that has been true since the sidecars
+  were deployed, so it would have been firing before, during and after. That is deliberate and it
+  is the honest claim: it is a **gap tracker**, not an incident detector. It would have meant that
+  on 2026-09-30 an operator asking "why did nothing alert?" had the answer already on the board
+  instead of discovering it during the investigation. **No alert on this platform could have
+  detected the incident itself, and none can today.**
+- **The alert that the brief actually asked for does not exist, and cannot be written yet.** The
+  sustained-redirect-ratio rule needs `oauth2_proxy_requests_total` from the *per-app* sidecars.
+  **Enabling that requires editing files owned by the two sibling branches** — the sidecar args in
+  `apps/wave-0/jackett.yaml` (Mira Vance) and `apps/wave-0/{qbittorrent,pihole}.yaml`,
+  `apps/wave-1/{radarr,sonarr}.yaml` (Theo Bright) — so it was **not attempted here**. The change
+  is three things per app, together: `--metrics-address=0.0.0.0:44180`, a `metrics` containerPort
+  plus Service port, and a ServiceMonitor. Once any one app has them, the rule belongs beside
+  `OAuth2ProxyGateUnobserved`, and that app's namespace drops out of it automatically.
+- **The Loki alerting path is inert and I did not fix it.** Loki *has* a ruler stanza and the
+  chart's `loki-sc-rules` sidecar is running with `LABEL=loki_rule`, `FOLDER=/rules` — so a
+  ConfigMap-shipped LogQL rule would be delivered. But `GET /loki/api/v1/rules` answers **"unable
+  to read rule dir /var/loki/rules/fake: no such file or directory"** (the ruler reads a different
+  directory from the one the sidecar writes), and the rendered config contains **no
+  `alertmanager_url`** — `grep -i 'alertmanager\|rule_path'` over the `loki` ConfigMap returns
+  nothing. So a LogQL alert on `|= "No valid authentication in request"` — which *would* have
+  caught this incident, and is the single highest-value follow-up here — **would evaluate and
+  notify nobody**. Wiring it means `rulerConfig` (alertmanager_url + a matching rules directory)
+  in `platform/monitoring/loki.yaml`, which is not a file this brief gave me. **I did not ship an
+  inert rule ConfigMap to look productive.**
+- **Nothing was reproduced in a browser.** The 302-to-200-with-HTML mechanism is read off the
+  sidecar access log and the response sizes (`/UI/Dashboard 200 44379`), not observed in devtools.
+- **Loki retention is 72h** (`platform/monitoring/loki.yaml`), so the 24h ratios above are a
+  snapshot; anything older than three days is already gone. This is stated in the runbook because
+  it bounds what an operator can still investigate.
+- **The gate's L2/L3 conformance did not run** (`CLUSTER_AVAILABLE` unset), so no runtime
+  requirement touched by this branch is certified. See the exit code below — **rc 3 is
+  UNDETERMINED, not green.**
+
+### Follow-ups, in order of value
+
+1. **Wire the Loki ruler to Alertmanager** (`platform/monitoring/loki.yaml`). This is the only
+   route to an alert that would have caught the actual 2026-09-30 failure, and the data is
+   already being collected. The rule is one expression:
+   `sum by (namespace) (rate({container="oauth2-proxy"} |= "No valid authentication in request" [15m]))
+   > 5 * sum by (namespace) (rate({container="oauth2-proxy"} |= "AuthSuccess" [15m]))`.
+2. **Give one gate sidecar a metrics endpoint** and watch its namespace drop out of
+   `OAuth2ProxyGateUnobserved`. That is the cheapest proof the alert is a tracker and not a nag,
+   and it unlocks the ratio rule in Prometheus.
+3. **Consider whether `--set-xauthrequest` apps should get `--api-routes`** so API paths answer
+   401 instead of 302. That is the fix that would make this failure *loud* rather than merely
+   observable, and it belongs with whoever owns the sidecar args.
+
+---
+
+# Round 2 — 2026-10-01: the scrape got wired, and found a bug in this branch's own alert
+
+Added to the same PR (#29) rather than a fourth concurrent branch. PR #28 merged in the meantime
+and `main` moved three PRs ahead; `origin/main` is merged in here, and the `runbooks/kustomization.yaml`
+conflict was resolved by **keeping both** runbook entries (mine and `oauth2-proxy-session`).
+
+## What changed
+
+**`platform/oauth2-proxy/podmonitor.yaml` (new).** PR #28 gave the five fleet sidecars
+`--metrics-address=0.0.0.0:44180` and a named `metrics` port. No Service exposes 44180 on any of
+them, so a ServiceMonitor would have nothing to select — a **PodMonitor** targets pod ports directly
+and needs no renderer change. Selects on the pod label `homelab.keithjackson.dev/auth-mode:
+forward-auth`, which is on exactly the five fleet gate pods and absent from crafty's: the property,
+not an app list, so a sixth forward-auth app needs no edit.
+
+**`auth-visibility-alerts.yaml`: `up{container="oauth2-proxy"}` → `up{container="oauth2-proxy"} == 1`.
+This is a real bug fix, and it is the most important thing in this round.** A target that is
+discovered but failing still publishes `up`, with value **0**. The series exists, so
+`count by (namespace)` returned 1, `unless` removed the namespace, and `OAuth2ProxyGateUnobserved`
+would have **gone quiet while nothing was being scraped at all** — this branch's own disease,
+reproduced inside the alert written to cure it. Not hypothetical: the PodMonitor lands five targets
+whose scrape is dropped, so the bare form would have silently cleared on this very commit.
+
+**Two new unit-test scenarios** pinning that: a discovered-but-failing target must still fire, and a
+target that works and then breaks must re-fire rather than latch cleared.
+
+**Docs corrected rather than appended to.** The 2026-09-30 claim "the sidecars export no metrics at
+all" is now only half true and was rewritten in `runbooks/README.md`, `platform/oauth2-proxy/README.md`
+and the alert header, each showing the two-step history, because each step looked like the end of it.
+The runbook gains the policy-vs-no-target triage (step 1b) and the **radarr `/signalr` worked case**.
+
+**Deliberately NOT shipped: the ratio alert.** `oauth2_proxy_requests_total` now exists on the
+endpoint, but not in the TSDB, because the scrape is blocked. Writing the ratio rule today would make
+it inert for precisely the reason it was refused on 2026-09-30. It becomes writable the day a scrape
+succeeds, not the day a listener exists.
+
+## What I verified
+
+- **The endpoint serves — by payload, not by flag.** Port-forwarded to the jackett pod's 44180:
+  `http=200 bytes=10208`, exporting `oauth2_proxy_requests_in_flight`, `oauth2_proxy_requests_total`
+  (`code="200" 3`, `code="302" 2`) and `oauth2_proxy_response_duration_seconds`. `--metrics-address`
+  being in `args` and `/metrics` returning a body are different claims; both now hold.
+- **The scrape path is DROPPED, measured from the right place.** From grafana in ns `monitoring`
+  (beside Prometheus, same policy position) to the jackett pod IP: `:44180` → `wget: download timed
+  out`; `:4180` → **also** timed out; the shared proxy's `:44180` → 200 with `oauth2_proxy_*` present.
+  Cause, read from the objects: ns jackett's `default-deny` is `podSelector: {}`,
+  `policyTypes: [Ingress, Egress]`, with an `egress` rule and **no `ingress` key** (deny-all ingress);
+  the only allowance is CNP `jackett-allow-gateway-ingress`, `fromEntities: [ingress]` to port 4180.
+  The shared proxy is scrapable only because ns `oauth2-proxy` has **no NetworkPolicy at all**.
+- **No Service exposes 44180** on any of the five (enumerated).
+- **The pod label selector is exact:** `auth-mode=forward-auth` on all five fleet gate pods, absent
+  from crafty's. Crafty's sidecar still has no `--metrics-address` (args read in full).
+- **Flux ordering forbids shipping the fix from here:** `oauth2-proxy dependsOn=['platform-config']`
+  and `apps-wave-0 dependsOn=['platform-config']` — siblings, no ordering. A NetworkPolicy naming ns
+  `jackett` from this tier would apply today and fail on a cold rebuild.
+- **The `== 1` guard is non-vacuous — proven by mutation.** Reverting it to the bare `up` makes the
+  two new scenarios FAIL (`got: nil` at 1h5m and 2h5m) and leaves every other assertion passing;
+  restoring it returns the suite to SUCCESS under pinned promtool v3.14.0.
+- **Gate:** see the next section for the exit code and skips.
+
+## What I did NOT verify
+
+- **The PodMonitor has never been applied.** That it discovers the five pods is reasoned from the
+  measured pod labels and port name, not observed — no target list has been read with it in place,
+  because applying it is an operator action. **Expect `up == 0` on all five when it lands**, which is
+  the documented state, not a regression.
+- **I did not fix the NetworkPolicy, and this PR therefore does not make the gates observable.** It
+  makes them *discovered*, and makes the remaining gap loud instead of silent. The one narrow rule
+  per app (ns monitoring → `auth-mode=forward-auth` → TCP 44180, per
+  `platform/longhorn/networkpolicy-metrics.yaml`) has to be rendered with each app's own policy, in
+  the catalog/renderer that is in flight on another branch. **It needs an owner.**
+- **Whether Prometheus will actually select a PodMonitor in ns `oauth2-proxy` targeting other
+  namespaces.** `namespaceSelector.any: true` plus `podMonitorSelectorNilUsesHelmValues: false` and
+  the fact that cross-namespace ServiceMonitors/PrometheusRules already work here say yes; it is not
+  measured with this object.
+- **No session-lifecycle claim can be made from the current window.** `AuthSuccess` is **0** on both
+  jackett and radarr since the PR #28 restart, so nobody has signed in and `--cookie-refresh=2m30s`
+  has never executed a refresh. The cookie fix is unproven in the one dimension that matters most.
+- **The radarr `/signalr` finding is the coordinator's measurement, not mine** — I reproduced neither
+  the 206/209 sample nor the jackett 0-in-13,612. They are recorded as reported.
+- **L2/L3 conformance still did not run** (`CLUSTER_AVAILABLE` unset). Nothing is certified.
+
+### Round 2 addendum — CI is RED, by design, and the local gate cannot see why
+
+After pushing round 2, `gate / gate (pull_request)` reported **failure after 1m42s** while the local
+toolbox run was rc 3 with nothing failed. Measured cause, singular:
+
+    sensitive paths touched:
+      conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh   (matched conformance/**)
+    approving reviews: (no reviews at all)        author=gitea-admin
+
+`ci/approval-policy.sh` is fail-closed and exits 1 when a PR touches a `sensitive_paths` glob with no
+`APPROVED` review from a non-author — "a change to the gate cannot be gated by the gate". The single
+`KNOWN_EXTERNAL_MONITORS` entry is what trips it. **Nothing in L0/L1 failed.**
+
+**The local gate structurally cannot catch this, and said `approval-policy: green (N/A)`.** The stage
+short-circuits to N/A when `GITHUB_EVENT_NAME` is unset, which it always is off-CI. So for any branch
+touching a sensitive path, a clean local gate is a FALSE REASSURANCE — the PR run is the only honest
+signal. That is a general trap, not specific to this change, and it is worth knowing before someone
+concludes a red PR means broken code.
+
+To clear: a non-author approving review, then **re-run the gate** — approval does not re-trigger the
+status check, so the required context stays red until the job runs again. `required_approvals=1`
+already applied to every PR here, so the human was needed regardless; the conformance entry only
+turns the tick red while that is pending.
+
+The alternative is to drop the register entry, which means dropping `podmonitor.yaml` with it (the
+check fails without the declaration). I did not take it: the entry is the "declare the limit"
+mechanism the check explicitly asks for, it is shrink-only so it cannot rot quietly, and
+`flux-podmonitor.yaml` is the same case already in the register. But it is a real trade and the
+reviewer should make it knowingly.

decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md

@@ -0,0 +1,378 @@
+---
+date: 2026-09-30
+branch: fix/oauth2-proxy-cookie-collision-fleet
+title: The cookie collision was never Crafty's — it was the renderer's, and it shipped to five apps
+---
+
+## What changed
+
+Follow-up #3 of `decisions/2026-09-29-crafty-oauth2-proxy-signin.md`: *"Any further per-app
+reverse-proxy sidecar needs a unique `--cookie-name` too. The collision is a property of the
+pattern, not of Crafty."*
+
+It is a property of the pattern, and this branch found where the pattern actually lives.
+
+**Read the scope claim carefully before the detail.** This branch fixes THREE defects in the
+forward-auth sidecar contract, all in one function, all landing on all five apps at once:
+
+| Defect | Fix | Measured evidence |
+|---|---|---|
+| Six independent instances, seven different cookie-secrets, **one** cookie name on **one** zone-wide domain — so they overwrite and `ClearSession`-delete each other's sessions | `--cookie-name=_oauth2_proxy_<app>` + host-scoped `--cookie-domain` | 7/7 distinct secrets; 6 instances on the default name; radarr logs the eviction fingerprint 8× in 12h |
+| The session dies **5 minutes** after sign-in — `--cookie-refresh` defaults to 0 = *disabled*, so the refresh token the proxy already holds is never spent | `--cookie-refresh=2m30s` (half the lifetime) | access-token lifetime = **300.000s exactly**, `refresh_token:true`, flag absent |
+| **No** per-app sidecar serves `/metrics` at all — without `--metrics-address` the listener never starts | `--metrics-address=0.0.0.0:44180` + a `metrics` container port | `ports` was `['http']` on all six |
+
+Rejection *volume* is dominated by the second defect, not the first: ~95% of radarr's rejections
+carry no cookie at all. **Do not merge this expecting one number to drop** — three mechanisms were
+tangled together, and separating them is most of the value here.
+
+**Boundary note.** My brief originally reserved `apps/wave-0/jackett.yaml` for a concurrent branch.
+That split is not achievable and has been revoked: the file is generated, the fix is one renderer
+edit, and `render-golden` requires all five goldens regenerated in the *same commit*. I own all
+five. The proof I ran before touching jackett is under "What I verified".
+
+**The defect was not in the four manifests it was reported in.** `apps/wave-0/pihole.yaml`,
+`apps/wave-0/qbittorrent.yaml`, `apps/wave-1/radarr.yaml`, `apps/wave-1/sonarr.yaml` — and
+`apps/wave-0/jackett.yaml` — are **generated goldens**. One function emits all five sidecars:
+`catalog/render.py::_oauth2_sidecar_container()`, which hardcoded `--cookie-domain=.{ZONE}` and
+emitted no `--cookie-name` at all. Hand-editing a golden is not a fix twice over: `ci/render-golden.sh`
+FAILS on the drift (provoked, not assumed — see below), and the next re-render silently reverts it.
+
+So the fix is in the generator, and both flags are now **derived** from the ServiceDefinition:
+
+    --cookie-name=_oauth2_proxy_{s['name']}     unique per service, by construction
+    --cookie-domain={host}                      the app's own route host, not the zone
+
+`--whitelist-domain` deliberately stays zone-wide — it governs permitted post-login *redirect*
+targets, not cookie scope. `catalog/render.py --selftest` gained four assertions, including one
+that renders a **second** synthetic forward-auth service and requires the two cookie names to
+differ, so a future refactor to a constant fails in the renderer's own test.
+
+Re-rendering with the canonical command regenerated **five** goldens, not four. Jackett is the
+fifth, and this is the one thing in this branch the PM brief did not anticipate — see
+"the jackett collision" below. The three `conformance/tests/l1-rendering/*/golden/` fixtures are
+expected-output copies of the same renderer, so they are regenerated too; that is not collateral
+damage, it is the L1 stage doing its job (it FAILED on the first gate run and named the drift).
+
+**The new gate check** is `conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/`,
+declaring `conformance.home.k3s/scope: whole-repo` so `ci/l0.sh`'s `--include-scope` union runs it
+on **every** gate invocation. No bespoke `ci/l0.sh` stage was added — `whole-repo-invariants-wired`
+fails the gate if anyone adds one, and the scope annotation is the supported mechanism.
+
+It inventories every oauth2-proxy instance the repo authors (containers at any depth, plus the
+shared proxy's HelmRelease values) and resolves each one's **effective** cookie name — its declared
+name, **or oauth2-proxy's built-in `_oauth2_proxy` when it declares none**. That single modelling
+decision is the whole check: an omitted flag is not an absence, it is a *claim on a name*, and
+treating it as an absence is precisely why five reviewers read these manifests and saw nothing.
+Then: (1) no two effective names collide; (2) no name is another's name + `_csrf`, because that is
+how oauth2-proxy derives the CSRF cookie and two such session names look distinct while their CSRF
+cookies collide; (3) the default name has exactly one owner, and it is the one registered in
+`SHARED_DEFAULT_COOKIE_OWNER`; (4) every non-registered instance is host-scoped; (5)
+`catalog/render.py` derives rather than lists; (6) the **shipped** renderer is proven end-to-end on
+two synthetic ServiceDefinitions run as a subprocess.
+
+**On the shared proxy — "distinct" or "explicitly declared"?** The brief asked me to decide, and
+the answer is *neither, quite*. The shared forward-auth proxy at `platform/oauth2-proxy/` **must
+keep** `_oauth2_proxy`: it is the single session every forward-auth-gated surface without its own
+sidecar rides on, so renaming it signs all of them out at once. Requiring *every* instance to
+declare a name would therefore need an exemption, and an exemption is a hole: delete the shared
+proxy and whichever app next omitted the flag would silently inherit the default with the rule
+still satisfied. So the assertion is plain **distinctness over effective names**, with the default
+treated as a name like any other — held by **exactly one owner, named in a shrink-only register
+with its reason**. The shared proxy needs no exemption from the rule; it is simply the registered
+holder of that one name. A stale register entry FAILS (the exemption would outlive its object) and
+an unregistered holder FAILS (no silent heirs). Both were provoked and are quoted below.
+
+**On host-scoped `--cookie-domain` — overreach, or not?** Asserted, and here is the argument both
+ways. *Against:* distinct names alone close the eviction bug completely — two differently-named
+cookies do not overwrite each other however wide their domains are — so this assertion fixes
+nothing (1) has not already fixed, and a check that fails a legitimate configuration is a check
+someone eventually disables. *For, and this wins:* a zone-wide cookie is **transmitted by the
+browser to every host in the zone**. qBittorrent's session token was being sent to pihole, radarr,
+sonarr and every other name under the zone, including hosts behind no proxy at all — a
+session-credential disclosure across trust boundaries that nothing needs. And it is what makes a
+*future* name collision catastrophic rather than contained: with host-scoped cookies, two
+instances that accidentally share a name cannot see each other's cookie at all. Defence in depth
+on the exact axis that already failed once. The escape hatch is the same register, by name and
+with a reason, rather than a loosened rule.
+
+### The jackett boundary, and why it could not hold
+
+`apps/wave-0/jackett.yaml` carries the identical defect and was initially assigned to a concurrent
+branch, with an instruction to me not to touch it. **That separation is not achievable** — the
+ownership split was drawn across a generated file — and I did not discover this by reasoning about
+it, I provoked it. With the renderer fixed and jackett's golden left at its `main` content,
+`./ci/render-golden.sh` exits **1**:
+
+    [fail] GOLDEN DRIFT: apps/wave-0/jackett.yaml does not match the canonical render:
+    -        - --cookie-domain=.${BASE_DOMAIN}
+    +        - --cookie-name=_oauth2_proxy_jackett
+    +        - --cookie-domain=jackett.${BASE_DOMAIN}
+    [fail] render==golden check FAILED
+
+Jackett's golden is an *output* of the file that contains the defect. There are exactly three
+options and two of them are worse: hand-edit four goldens and leave the generator broken (fails
+`render-golden`, and reverts on the next render); add a per-service carve-out to the renderer so
+jackett alone keeps defaulting (ships the exact "list, don't derive" anti-pattern this check
+exists to forbid, and leaves jackett broken *by design*); or fix the generator and regenerate all
+five. I took the third. Jackett's golden here is **machine output of a one-line generator change**,
+not an independent edit of Mira's file, and it is byte-identical to what her branch must produce.
+
+**Consequence:** the jackett branch has no cookie work left in it — this commit fixes jackett
+identically and by the same mechanism, and both branches independently arrived at the *same two
+values* (`_oauth2_proxy_jackett`, `jackett.${BASE_DOMAIN}`), which is a good sign for the
+diagnosis. Its hand-edit of the rendered file will fail `render-golden` on its own terms, so it
+should be closed rather than rebased. **What must not be lost with it** is its evidence: the
+interleaved radarr/jackett log showing `removing session` deleting a clean session 26s after login,
+and the `expires − created = 300.000s` measurement that produced `--cookie-refresh` here. Both are
+carried into this entry and into the renderer's comments; the branch itself is now redundant, the
+work behind it is not.
+
+**The general lesson, which is worth more than this bug:** an ownership split across
+`apps/wave-*/*.yaml` cannot hold, because those files are *outputs*. Two agents assigned two
+rendered apps are assigned the same function and will collide in `render-golden` no matter how
+carefully they stay in their own files. Ownership of a rendered app is ownership of
+`catalog/render.py`, and briefs should say so.
+
+### What this branch deliberately does NOT do
+
+- **`--api-route=^/api/` is NOT added**, and not because of scope discipline alone: it would be
+  **wrong** renderer-wide. It was measured on jackett, and **PiHole does not serve `/api/`** — so
+  a renderer constant applies one app's path shape to five. Doing it properly means declaring API
+  path prefixes per service, and the `auth` block is `additionalProperties: false`, so that is a
+  *schema change*. Separate PR. (Note the flag is **`--api-route`, singular** — verified in the
+  pinned binary's own `--help`. The plural `--api-routes` does not exist in v7.8.1 and the proxy
+  refuses to start with it. That error was propagating through briefs; it stops here.)
+- **`--session-cookie-minimal` is NOT added** and must not be, as a cookie-size fix: it is
+  **mutually exclusive** with `--cookie-refresh` (*"cookie_refresh > 0 requires oauth tokens in
+  sessions"*). Dropping `--pass-access-token` is a least-privilege argument, never a size one.
+- **Metrics are NOT scraped, only served.** The flag and container port are the prerequisite;
+  a container port is not a Service port. These Services expose the app port and `:4180`, and each
+  namespace's default-deny baseline is `ingress: []`, so Prometheus can neither resolve nor reach
+  `:44180` until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor
+  exist. That wiring is someone else's follow-up and I have deliberately not faked its start.
+  **Nobody should read this commit as "the sidecars have metrics now."**
+- **The new check was NOT widened to cover cookie-refresh or metrics.** Its subject is cookie
+  *identity*; those two share neither its inputs nor its failure mode. Bundling them would make
+  the check's name a lie and couple unrelated ratchets, so a future metrics decision could only
+  be expressed by weakening a *cookie* check. The cost of that discipline is named honestly in
+  the follow-ups: those two flags are currently **ungated**.
+
+## What I verified
+
+- **The premise, in-cluster, rather than inherited from the handoff** (CLAUDE.md §2b). Read-only,
+  `kubectl --context k3s-prod`, through the pinned toolbox.
+  - **Seven** `*-oauth2-proxy` Secrets carry a `cookie-secret`: crafty, jackett, pihole,
+    qbittorrent, radarr, sonarr, and the shared `oauth2-proxy-credentials`. **All seven values are
+    DISTINCT.** Compared without printing anything: each value was hashed **in-cluster-side script,
+    never echoed**, and only the first 8 hex characters of each `sha256` were emitted — enough to
+    say same-or-different, recovering nothing. Secret *key names* were listed; no `.data` or
+    `.stringData` was printed (CLAUDE.md §6). **So no instance can decrypt another's cookie — it can
+    only overwrite it. The premise holds.**
+  - **The default cookie names really were in use.** Of 8 live oauth2-proxy containers, **6 declare
+    no `--cookie-name` at all** — jackett, pihole, qbittorrent, radarr, sonarr, and both replicas of
+    the shared proxy — so all six emit `_oauth2_proxy` / `_oauth2_proxy_csrf`. All six also carry
+    `--cookie-domain=.home.keithjackson.dev`. **One cookie pair, six independent instances.**
+  - Crafty's live pod (`crafty-647c7d8cc5-5spv9`) already carries
+    `--cookie-name=_oauth2_proxy_crafty` and `--cookie-domain=crafty.home.keithjackson.dev`, so
+    PR #22 is merged and reconciled. It is the only compliant per-app instance on the cluster.
+- **THE COLLISION IS REAL. IT IS NOT ESTABLISHED AS THE CAUSE OF THE REPORTED FAILURES.** These
+  are two different claims and only the first is supported. I was handed counter-evidence —
+  crafty, the app already fixed, showed the *worst* rejection ratio of the set over a 24h Loki
+  window (`crafty 81:1`, `radarr 67:1`, `jackett 39:4`) — and it is right to refuse the causal
+  claim. What I could do was bound the window. Read from **each sidecar's own log** rather than
+  Loki, because `kubectl logs` on a pod that has not restarted cannot reach a pod older than
+  itself, so a window inside the pod's age is inside the fixed spec's lifetime *by construction*:
+
+      namespace      age(h) rst  reject success    ratio  cleared  badsig
+      crafty           13.9   0       3       0      n/a        0       0   <- FIXED spec, whole life
+      radarr          422.5   0     155       1  155.0:1        8       8
+      jackett         422.6   0      39       4    9.8:1        1       1
+      pihole          395.7   0       0       0      0:0        0       0
+      qbittorrent     404.1   0       0       0      0:0        0       0
+      sonarr          422.5   0       0       0      0:0        0       0
+      oauth2-proxy    422.6   0       0       0      0:0        0       0   (both replicas)
+
+  `reject` = `No valid authentication in request`; `success` = `AuthSuccess`;
+  `cleared` = `removing session` (oauth2-proxy's `ClearSession`); `badsig` = `cookie signature
+  not valid`. Window `--since=24h`, which exceeds every pod's age only for crafty — whose pod is
+  13.9h old with **0 restarts**, so these are its complete post-fix figures.
+
+  Then I read crafty's four auth-relevant lines rather than counting them, because at n=3 the
+  content is the evidence:
+
+      03:57:20  Cookie settings: name:_oauth2_proxy_crafty secure(https):true httponly:true
+                expiry:168h0m0s domains:crafty.home.keithjackson.dev path:/
+      04:02:55  No valid authentication in request. Initiating login.
+      15:22:39  No valid authentication in request. Initiating login.
+      15:22:44  No valid authentication in request. Initiating login.
+
+  6743 log lines; those are all four. **Zero** `removing session`, **zero** `cookie signature
+  not valid`, zero anything else.
+
+  Four conclusions, kept separate:
+  1. **`crafty 81:1` is a window artefact. It is not evidence against the fix and must not be
+     cited as such.** The fixed pod emitted **3** rejections in 13h 52m; the 81 came from a 24h
+     window straddling it, i.e. from the pre-fix process.
+  2. **The eviction FINGERPRINT is gone where the fix is deployed.** `cookie signature not
+     valid` + `removing session` is what a clobbered instance logs — a cookie *is* present,
+     written with a key this instance does not hold, so it cannot decrypt it and then
+     `ClearSession` expires it on its own `--cookie-domain`. Fixed crafty: **0 and 0** across
+     its whole life. Unfixed radarr: **8 and 8** in 12h. Unfixed jackett: 1 and 1. That is a
+     real, directional signal and it is the strongest evidence available that the fix works.
+  3. **It is still not a demonstrated successful sign-in.** Crafty logged **0 `AuthSuccess`** —
+     so what is shown is *absence of the failure mode*, not a session observed working
+     end-to-end. Those three rejections are consistent with benign first-hits (one client, two
+     of them 5s apart, i.e. a page plus a `/ws` reconnect). I am comfortable saying the fix
+     works; I am not willing to say it has been watched working, and the distinction is the
+     whole reason this section exists.
+  4. **The collision is a MINORITY of the rejection volume.** radarr: 8 of 155 (5%) carry the
+     fingerprint; ~95% carry **no cookie at all**. Those are the 5-minute session expiry and
+     `302`-on-XHR — the first of which this branch now also fixes, the second of which it
+     deliberately does not.
+
+  Two cautions about the instrument. `reject` ranks **chattiness, not breakage**: radarr's are
+  `/api/v3/command`, `/api/v3/queue/details` and `/signalr/negotiate` polls, so a busy *arr
+  inflates the count with nobody inconvenienced. And that same polling is the *amplifier* that
+  made this intermittent — every poll carrying an undecryptable cookie calls `ClearSession`
+  again on the zone-wide domain, so **a left-open Radarr tab deleted Jackett's cookie every few
+  seconds, from another namespace, with no user action at all.** That is why signing in again
+  never helped for longer than the gap between two polls. It is now a comment in the renderer,
+  because it is the part of this bug that no one would reconstruct from the flags alone.
+- **The four target files are generated, not authored.** Rendering `catalog/` with the canonical
+  command into a temp dir and diffing against `apps/` reported all 10 onboarded goldens
+  byte-identical before any change. Hand-editing one is drift by definition.
+- **`render-golden` FAILS if jackett is excluded** — provoked (output quoted above), rc **1**.
+  Restoring the regenerated golden returns it to `all 10 onboarded service golden(s) reproduce`,
+  rc **0**.
+- **Every flag verified against the PINNED v7.8.1 binary**, not against docs, memory, or the
+  shared proxy (which runs **v7.15.3** and therefore could not vouch for v7.8.1):
+  `podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help`.
+  - `--cookie-name string ... (default "_oauth2_proxy")` — **this confirms the constant my whole
+    check is built on, from the binary itself.** The check's central claim, that an omitted flag
+    is a *claim on a name* rather than an absence, now rests on the implementation's own help
+    text rather than on my reading of its source.
+  - `--cookie-refresh duration  refresh the cookie after this duration; 0 to disable` — confirms
+    the default really is disabled.
+  - `--metrics-address string` — present in v7.8.1, so the new flag cannot crashloop the pods.
+  - `--api-route strings` — **singular**, confirming the brief's plural was wrong.
+  - `--session-cookie-minimal` — exists, and is the flag that conflicts with `--cookie-refresh`.
+- **44180 is the platform's existing convention, measured, not chosen.** The shared proxy already
+  runs `--metrics-address=0.0.0.0:44180` with a `metrics` container port, a `metrics:44180`
+  Service port, a ServiceMonitor whose endpoint is `metrics`, and **2 ready endpoint addresses** —
+  i.e. it demonstrably works on this platform.
+- **`catalog/render.py --selftest`: PASS**, now including the two-service distinctness assertion,
+  a `--cookie-refresh` assertion that PARSES the duration and requires `0 < refresh < 300s` (so
+  `2m30s` -> `150s` -> `2.5m` all stay green but a silent widening past the token lifetime does
+  not), a `--session-cookie-minimal` mutual-exclusion guard, and both metrics assertions.
+- **The check is selected by the whole-repo union.** `conformance/run.sh --layer l0 --requirements
+  r-nosuchreq-99 --include-scope whole-repo --print-selection` lists
+  `tests/l0-static/oauth2-proxy-cookie-names-distinct` among 20 tests — i.e. it runs even when the
+  touched-requirement filter matches nothing, which is the blind spot it was scoped for.
+- **The check was tested in BOTH directions**, against throwaway trees outside the repo, never by
+  mutating the branch:
+  - **Compliant tree (this branch): rc 0, 39 assertions, 7 instances audited.**
+  - **Pre-fix tree** (`git archive HEAD` + this test dropped in): **rc 1, 26 failures**, naming all
+    five violators, the 6-way `_oauth2_proxy` collision, the zone-wide domains, the renderer
+    emitting no `--cookie-name`, and both synthetic fixtures rendering `None`.
+  - **Targeted mutations on a compliant copy**, one assertion class each, all fired and nothing
+    else did: duplicate name (sonarr←radarr) → 1 failure; CSRF derivation
+    (`_oauth2_proxy_sonarr_csrf` as a session name) → fired; zone-wide `--cookie-domain` on pihole
+    → 2 failures; crafty's `--cookie-name` deleted → unregistered-default-heir + 2-way collision;
+    shared proxy renamed → **stale register entry** failure.
+  - A **false positive was found and fixed** by this testing: `platform/oauth2-proxy/forward-auth-cec.yaml`
+    contains `serviceName: oauth2-proxy/oauth2-proxy:80` — an Envoy `namespace/service:port`
+    reference, not an image — which the first inventory scan counted as an unaudited instance. The
+    scan is now anchored on `image:`, and the near-miss is documented in the check.
+- **Gate:** toolbox `./ci/gate.sh` — exit code and stage list in the next section, because it is
+  not a pass and that belongs there.
+
+## What I did NOT verify
+
+- **Local gate: rc 3 (UNDETERMINED), not 0.** Green: `spec-immutability`, `decisions-log`, `lint`,
+  `l0`, `render-golden`, `l1`, `approval-policy`. **Skipped:** `gitea-settings` (no
+  `SETTINGS_API_TOKEN` locally) and **L2/L3 conformance** (`CLUSTER_AVAILABLE` unset). Nothing
+  FAILED; nothing runtime was certified either. **rc 3 is UNDETERMINED, never green.** Touched
+  subset `R-GITOPS-02 R-IDP-02 R-IDP-06 R-NET-01 R-NET-05 R-NET-06 R-NET-07 R-PLAT-02 R-TEST-01
+  R-TEST-02 R-TEST-03 R-TEST-04` — **UNVERIFIED** at L2/L3, all twelve.
+- **`spec-immutability` was skipping because of a BUG IN MY TOOLING, not the environment.** The
+  worktree ENGINE shim in circulation mounts only the main checkout's `.git`, and `dev.sh`
+  computes the spec path from its own `REPO_ROOT` — which in a worktree is
+  `.claude/worktrees/agent-*/`, so `../home-network-k3s` does not resolve and it adds no spec
+  mount at all. The gate then reported the skip, and the skip read as "the one tolerated
+  environmental skip" while actually being a path bug being waved through. Adding
+  `-v /var/home/keith/source/home-network-k3s:/home-network-k3s:ro -e SPEC_REPO=/home-network-k3s`
+  makes **R-TEST-06 actually run and pass**. Worth putting in every worktree agent brief; the
+  two-mount shim in CLAUDE.md §3 should say so.
+- **CI: `gate / gate (pull_request)` reports FAILURE on `75a4029`, for exactly ONE reason, and it
+  is not this change.** Read from the job log rather than the tick:
+
+      [fail] approval-policy: a sensitive change lacks a non-author approval -> blocks merge
+      [fail] MERGE GATE: BLOCKED
+
+  Every substantive stage is **green in CI** — `gitea-settings: pass` (CI has the token my local
+  run does not), `decisions-log`, `lint`, `l0`, `render-golden`, `l1`. Skipped: `spec-immutability`
+  (the workflow does not check the spec repo out — the documented tolerated skip) and
+  `conformance` (declared). So CI is a **stricter, more complete** run than my local one on every
+  axis except the spec repo, and it agrees.
+  `infra/gitea/approval-policy.json` lists `conformance/**` as sensitive — *"the tests and the
+  harness; changing them changes what green MEANS"* — so this PR is **fail-closed pending a
+  non-author approving review**, which is CLAUDE.md §1 working, not a defect to route around.
+  **The red tick is the policy, and it clears when a human reviews it.** Note also that
+  `gate / conformance (pull_request)` reports `success` with description **"Has been skipped"** —
+  the precise trap CLAUDE.md §2b names; it certifies nothing.
+- **A SUCCESSFUL SIGN-IN, ANYWHERE.** The strongest post-fix evidence is the *absence* of the
+  eviction fingerprint on crafty (0 `removing session`, 0 `cookie signature not valid`, across
+  13h 52m) against radarr's 8-and-8. That is directional and I believe it. But crafty logged
+  **0 `AuthSuccess`** in the same window, so no session has been observed working end to end on a
+  fixed instance. Whoever closes this out should re-measure crafty **after real sign-in traffic**.
+- **No browser round-trip by me at all**, for any of the three defects. Reproducing the collision
+  needs two gated apps visited in one session. The concurrent jackett branch does carry an observed
+  log-level reproduction (interleaved radarr/jackett `removing session` deleting a clean session
+  26s after login) — real evidence for the MECHANISM, and it is theirs, not mine.
+- **`--cookie-refresh=2m30s` has never been exercised.** The value is derived from a measured
+  300.000s access-token lifetime and is asserted to sit inside `(0, 300s)`, but no session has been
+  held past the five-minute cliff to watch a refresh succeed. If Keycloak's refresh-token or SSO
+  idle timeouts are shorter than expected, the symptom moves rather than disappears. **This is the
+  fix most likely to need a second pass**, and the first thing to watch after merge.
+- **The `metrics` port has never been fetched.** The listener is asserted to start by flag
+  presence only; nobody has curled `:44180` on a rendered pod. And it is unreachable by design
+  until the Service/policy/ServiceMonitor wiring lands — see above.
+- **Nothing was applied.** No `kubectl apply/edit/patch`, no `flux reconcile`. Cluster access was
+  read-only throughout; the one container I ran was the oauth2-proxy image locally under podman to
+  read its `--help`. The running pods are still the pre-fix spec. **Merging is what deploys**, and
+  it will roll five Deployments; every user's existing session on those five apps is invalidated by
+  the cookie rename and they will re-authenticate once. Radarr/sonarr/qbittorrent use
+  `strategy: Recreate` on RWO volumes, so there is a brief gap per app. Pick the moment — and
+  note this now changes **three** flags plus a port on all five, so a bad `--cookie-refresh` value
+  would surface as five apps misbehaving, not one.
+- **The live flags are not re-measured by the check.** git is not the cluster. I measured the live
+  pods by hand, once; nothing in the gate re-reads them, so hand-applied drift would be invisible
+  to it. A live-state assertion is L2 and is not written.
+- **`--cookie-refresh` and `--metrics-address` are UNGATED, and I found out why.** Nothing in
+  `ci/` or `.gitea/` runs `catalog/render.py --selftest` — I grepped for it. So the twenty-odd
+  assertions in that selftest, including the six I just added, are dev-time only: a future edit
+  that drops either flag *and* re-renders the goldens would pass the entire merge gate.
+  `render-golden` catches accidental drift (golden vs renderer) but not a deliberate removal of
+  both together, and my L0 check deliberately does not cover these two flags. **The repo already
+  has the precedent for the fix and the one-line shape of it:**
+  `conformance/tests/l0-static/auditability-proofs-producible/check.sh:291` drives
+  `conformance/lib/<tool> --selftest` as a subprocess for exactly this reason — its own header
+  says it does so "because nothing [else] drives them". Adding `catalog/render.py` to that list,
+  or giving the renderer its own scope-declaring L0 test, closes it. I did not do it here because
+  it widens an already review-blocked PR into a third subject; it is the highest-value follow-up
+  in this entry.
+- **Log-derived counts are a weak instrument and I did not strengthen them.** `kubectl logs
+  --since=` returns what the kubelet still retains, so an old, busy container's window may be
+  truncated; and `No valid authentication in request` counts unauthenticated first hits and XHR
+  polls indiscriminately. The `cookie signature not valid` / `removing session` pair is the only
+  collision-specific signal in there, and it is the one I leaned on. None of this is gated —
+  there is no alert on it and no L2 check reads it.
+- **The cookie-secrets' distinctness is not gated either.** It was measured out of band and is
+  asserted nowhere. If they were ever made identical the collision would become harmless — but
+  sharing one session-encryption key across trust zones is a worse idea than the bug it would mask,
+  so the check does not offer it as an alternative and does not look for it.
+- **The shared proxy's own configuration is unchanged and untested by me.** I did not touch
+  `platform/oauth2-proxy/**` (owned by another agent this cycle); the check only reads it.
+- **Nothing about group authorization, CSRF behaviour, or session contents.** Names and domains only.
+- **`vars.CLUSTER_AVAILABLE` is still unset in CI**, so the PR's green tick — if it goes green —
+  proves the L0/L1 offline subset and nothing about runtime.

decisions/2026-09-30-plex-quicksync.md

@@ -0,0 +1,174 @@
+---
+date: 2026-09-30
+branch: feat/plex-quicksync
+title: Plex opts into `hardware.gpu: intel-quicksync` — the device mount is GitOps, the Plex-side toggle is a documented manual step, and hardware transcoding has NOT been observed working
+---
+
+## What changed
+
+Three things, and the third is the point of the branch.
+
+1. **`catalog/plex.yaml`** — added the four lines the whole feature reduces to:
+
+   ```yaml
+   hardware:
+     gpu: intel-quicksync
+   ```
+
+   placed between `env` and `exposure` to match the schema's own property order, with ~48 lines of
+   comment in the file's established style. The comment records: the field is **intent, not
+   mechanism** (a later move to the Intel device plugin is a `render.py` change and this block does
+   not move); that `/dev/dri/renderD128` is present on **all three** prod nodes with `i915` bound;
+   **why a `supplementalGroups` entry would be inert** (the image's cont-init hook does the group
+   work inside the container and `s6-setuidgid` replaces the process group list from `/etc/group`);
+   why `DirectoryOrCreate` is the load-bearing safe failure mode (`apps-wave-0` is `wait: true` with
+   `apps-wave-1` depending on it, so a failed mount would stall the whole app tier for a feature
+   that is not load-bearing); and that the Plex-side toggle is a **separate manual operator step**,
+   pointing at the new runbook.
+
+2. **`apps/wave-0/plex.yaml`** — re-rendered. Six added lines, nothing else, in the whole `apps/`
+   tree:
+
+   ```
+           - name: dri
+             mountPath: /dev/dri
+         - name: dri
+           hostPath:
+             path: /dev/dri
+             type: DirectoryOrCreate
+   ```
+
+3. **`runbooks/plex-quicksync.md`** (new), registered in `runbooks/kustomization.yaml`'s
+   `configMapGenerator` as `- plex-quicksync=plex-quicksync.md` and indexed in `runbooks/README.md`
+   under "Operator procedures". **The file alone would have been a 404** — the pod serves a
+   ConfigMap, not the git tree, and `conformance/tests/l0-static/alert-runbooks-resolvable` exists
+   precisely because "the file is in git" and "the URL resolves" are different claims. The page
+   covers: the GitOps/manual boundary as a table; the toggle (Settings → Transcoder → "Use hardware
+   acceleration when available", preference key `HardwareAcceleratedCodecs`); the real neighbouring
+   keys and the widely-cited one that **does not exist**; that the only proof is `Transcode (hw)` on
+   the Plex dashboard **and why no alert is possible**; the device-cgroup failure to expect first
+   with a concrete open() diagnostic and the Intel-device-plugin fallback; and the floating-`:latest`
+   caveat with a digest check.
+
+**Stacked on PR #27 (`feat/sd-hardware-capability`), which is open and unmerged.** This branch was
+cut from it rather than from `main`, so its one commit (`d9baa2a`, the schema + renderer + contract)
+is in this PR's range. That is deliberate: merging #27 is not an agent's action (CLAUDE.md §1), and
+duplicating its changes here would have been worse than showing them. **This must merge after #27.**
+
+Not done, deliberately: no `--emit-waves` (it writes into `clusters/base/`, a sensitive path that
+would force a human reviewer, and it is a no-op unless the *set* of files under `apps/` changes — it
+does not here). No `podSecurityContext`. No `nodeSelector`. No alert rule. No `kubectl apply`.
+
+### Two follow-ups left unclaimed, with the reasoning
+
+- **Transcode scratch is on the wrong medium, and it is invisible.** The image sets
+  `TranscoderTempDirectory=/transcode`, which is **not a volume** — it is the container's writable
+  overlay, i.e. the node's `/data`. That is NVMe on prod-2 but **SATA on prod-1 and prod-3**, so
+  transcode scratch performance changes silently on reschedule, with nothing reporting it. Both
+  candidate fixes need a **schema** change and so neither belongs in this PR: pin Plex to prod-2
+  (needs a `nodeSelector` field the schema does not have), or give `/transcode` a `medium: Memory`
+  emptyDir and transcode in RAM (~31 GiB per node; needs an emptyDir field the schema does not
+  have). **The RAM option is the better one** — it removes the node asymmetry outright instead of
+  working around it — which is exactly why it deserves its own PR and its own review rather than
+  being smuggled in behind a four-line catalog change. Documented in the runbook's "Known gaps".
+- **GuC/HuC firmware is not loaded** (`i915.enable_guc=0`). The cost is low-power H.264 **encode**
+  quality only; decode and this feature are unaffected. Changing it needs a kernel parameter plus a
+  **serialized rolling reboot** — an operator action under CLAUDE.md §5, on a cluster with a
+  documented history of that going badly (a prod node hung on a warm reboot on 2026-09-03). Kept
+  well clear of this PR.
+
+### Rebase note (2026-10-01, Ruth Okonkwo) — resolution only, no redesign
+
+Rebased onto `main` @ `08deb20` (was cut from `0fbf544`), restacked on the rebased
+`feat/sd-hardware-capability` (PR #27) so the stack stays coherent — this branch still carries
+exactly PR #27's commit plus its own, with no duplicate of anything already in `main`. The branch
+ref was force-updated with `--force-with-lease`, so PR #32 is the same PR with a new head.
+
+**The `catalog/render.py` conflict disappeared** once PR #27 was rebased; it was never this
+branch's own conflict. **One conflict remained, in `runbooks/README.md`, purely textual.** PR #30
+(`fix/jackett-oauth2-proxy-session`, merged as `08deb20`) appended an `oauth2-proxy-session` index
+entry under *Operator procedures* at the same point this branch appended its `plex-quicksync` entry.
+**Both entries were kept, in full.** Ordering follows this file's existing convention, which is
+grouped/narrative rather than alphabetical (the *Nodes* section runs down/io/memory/disk/cpu):
+`plex-quicksync` sits next to `plex-claim` as the other Plex page, leaving `oauth2-proxy-session`
+adjacent to `crafty-admin-credential`, which cross-references the same sidecar gate. The change to
+`runbooks/README.md` is an 8-line pure insertion, so PR #30's entry is untouched.
+`runbooks/kustomization.yaml` auto-merged and **both** `configMapGenerator` entries are present.
+
+Re-verified after the resolution, on the rebased stack: `--selftest` PASS (rc 0); a full re-render
+reproduces every committed golden byte-identically; and the **only** `apps/` diff against the new
+`main` is `apps/wave-0/plex.yaml`, confined to the expected six lines (the `dri` mount and its
+`DirectoryOrCreate` hostPath volume). No other service's golden moved — PR #28 added ~176 lines of
+oauth2-proxy sidecar rendering, and this confirms by measurement rather than assumption that Plex
+(`auth: native`, no sidecar) is unaffected by it.
+
+## What I verified
+
+- **The render is exactly the intended change, and nothing else moved.** After
+  `python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` (exit 0,
+  silent): `git diff --stat apps/` = `apps/wave-0/plex.yaml | 6 ++++++`, and
+  `git status --porcelain apps/` = a single ` M apps/wave-0/plex.yaml`. Read line by line, the diff
+  is the `dri` volumeMount and the `dri` hostPath volume, and nothing more. **Specifically checked:**
+  the same invocation also rewrites `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, which
+  are git-tracked but excluded from the golden comparison by `RENDER_GOLDEN_NOT_ONBOARDED` in
+  `ci/render-golden.sh` — a spurious diff there would have landed **ungated**. Neither file shows a
+  diff; whole-repo `git status --porcelain` lists only `apps/wave-0/plex.yaml` and `catalog/plex.yaml`
+  plus my new files.
+- **The rendered output matches what the contract promises** — `schemas/renderer-contract.md`
+  `### hardware` says exactly two things (a `/dev/dri` hostPath with `type: DirectoryOrCreate` and a
+  read-write mount on the app container) and exactly two things appeared. No `securityContext`, no
+  `nodeSelector`, no resources in the diff.
+- **The runbook is projected into the server, not merely committed.** `runbooks/kustomization.yaml`
+  carries `- plex-quicksync=plex-quicksync.md`, whose shape matches the generator-entry regex the
+  L0 check parses; the extensionless key is what makes `/runbooks/plex-quicksync` a plain static
+  lookup. Also indexed in `runbooks/README.md`.
+- **The runbook clears the L0 quality floor** that `alert-runbooks-resolvable` applies to
+  alert-linked pages, even though nothing links to this one (it is advisory for an unreferenced
+  page): all four required sections (`## What this means here`, `## First commands`,
+  `## Likely causes, most likely first`, `## When to escalate or stop`), well over the 1500-byte
+  floor, `dev.sh` used throughout, and `--context k3s-prod` on every command.
+- **The gate.** See the run recorded in the PR description for the per-stage table and the real rc.
+- **No secret was printed.** Every `Preferences.xml` read in the runbook greps one named key, and the
+  page states explicitly why: the file holds `PlexOnlineToken`. Nothing was `sops -d`'d.
+
+## What I did NOT verify
+
+**Hardware transcoding has NOT been observed working.** Nothing in this branch is evidence that it
+does. A green gate here proves the manifest renders and the golden matches — it proves nothing about
+runtime behaviour, and `vars.CLUSTER_AVAILABLE` is unset so L2/L3 conformance did not run at all.
+
+- **The device-cgroup question is OPEN, and it is the largest remaining technical risk.** Whether
+  containerd on this k3s build permits a **non-privileged** pod to `open()` a bind-mounted character
+  device is **UNVERIFIED**. It can only be settled by deploying and trying. The container inits as
+  uid 0, which settles *file* permissions, but uid 0 does **not** bypass the cgroup device gate — and
+  the two failures look identical from the outside. If it is refused, the fallback is the Intel
+  Device Plugin (`gpu.intel.com/i915`), which is a `catalog/render.py` change plus a re-render with
+  **no ServiceDefinition edit**; `privileged: true` is not available (Kyverno
+  `restrict-host-namespaces` is Enforce). The runbook has the diagnostic sequence.
+- **The Plex-side toggle was still unset at the time of writing.** `HardwareAcceleratedCodecs` was
+  **absent** from `Preferences.xml` — the toggle has never been enabled on this server. Until an
+  operator ticks it in the UI, this branch changes the pod spec and **nothing about transcoding**.
+  That is the whole reason the runbook exists.
+- **No metric, so no alert, so no automated detection — permanently.** `catalog/plex.yaml` declares
+  no `metricsEndpoint`, so Plex renders no ServiceMonitor; there is no transcode-mode signal for a
+  rule to evaluate. A silent degradation to software transcoding (a node losing its iGPU, the
+  `DirectoryOrCreate` path) will **not** be reported by anything. I deliberately did not add an
+  alert: with no metric behind it, it could never fire, and this repo has been burned at scale by
+  exactly that. Closing the gap honestly means an exporter first.
+- **I did not re-derive this session's node or in-container measurements.** The three-node
+  `renderD128`/`i915`/GID-992 facts, the cont-init hook behaviour, the `s6-setuidgid` group
+  replacement, the `PLEX_PREFERENCE*` absence, the six `setPref` keys and the
+  `HardwareAcceleratedCodecs` string were established earlier in this session and are cited, not
+  re-measured, by me. Per CLAUDE.md §2b, treat this file as a record of belief and re-measure against
+  the cluster before relying on any of it.
+- **The image is a floating tag.** Everything known about the hook came from digest
+  `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), while the Deployment runs
+  `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. A silent pull can invalidate the
+  "no `supplementalGroups` needed" conclusion **with no commit in this repo**. Not guarded against —
+  the runbook tells the reader to re-check the hook after an image bump, which is documentation, not
+  enforcement.
+- **I did not verify the live Gitea gate state** (branch protection, required contexts, runner
+  online) for this PR. Per §2b that must be queried, not read out of `CLAUDE.md`'s table; I did not
+  query it.
+- **`/runbooks/plex-quicksync` was not fetched.** The L0 check asserts every static precondition for
+  an HTTP 200; it does not make the request, and I did not either.

decisions/2026-09-30-sd-hardware-capability.md

@@ -0,0 +1,204 @@
+---
+date: 2026-09-30
+branch: feat/sd-hardware-capability
+title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` mounts /dev/dri and nothing else; the enum is what keeps the mechanism swappable
+---
+
+## What changed
+
+A new optional top-level ServiceDefinition property, `hardware`, with one optional member,
+`gpu`, whose enum today has exactly one value: `intel-quicksync`. **No catalog entry uses it.**
+This PR delivers the capability; consuming it (Plex) is separate work on top.
+
+- `schemas/servicedefinition.schema.json` — adds `hardware` (object, `additionalProperties: false`,
+  `minProperties: 1`) with `gpu` (enum `["intel-quicksync"]`). Schema version bumped **1.7.0 →
+  1.8.0**, changelog entry prepended to the schema `description` per that file's own convention.
+- `catalog/render.py` — two module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`) and the
+  emission in `r_deployment`.
+- `schemas/renderer-contract.md` — new `### hardware` field-mapping section, plus cross-field
+  invariant **8** in that file's numbered list.
+- `schemas/README.md` — its `**Schema version:**` line said `1.0.0`; it had been stale since 1.1.0.
+  Synced to 1.8.0 and annotated that the schema `description` is the changelog of record. Added one
+  bullet to "Cross-field rules" for the new enum. That was the whole of the staleness; nothing
+  ballooned.
+
+**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container only (never a
+sidecar), and it is exactly two things:
+
+1. a hostPath volume `dri` → `/dev/dri`, `type: DirectoryOrCreate`;
+2. a read-write `volumeMount` of `dri` at `/dev/dri` on the app container (deliberately not
+   `readOnly` — VA-API opens the render node read-write).
+
+**Mounting the device is sufficient.** It emits **no** `privileged: true` (Kyverno
+`restrict-host-namespaces` is Enforce on app namespaces and it is not needed), **no** `nodeSelector`
+(placement is a separate concern with its own justification), **no** resource requests (there is no
+app-container resource render path), and — the one that is worth spelling out — **no
+`podSecurityContext.supplementalGroups`**.
+
+**Why no `supplementalGroups`, when `/dev/dri/renderD128` is `root:render 0660`.** An earlier
+revision of this branch merged the host `render` GID into `supplementalGroups`. **It was measured to
+be a no-op and has been removed.** In the running container on prod-2, 2026-09-30:
+
+    PID 1   s6-svscan            Uid: 0     Gid: 0     Groups: 0
+    pid 246 Plex Media Server    Uid: 1000  Gid: 1000  Groups: 100 1000
+
+`/etc/services.d/plex/run` ends in `exec s6-setuidgid plex …`, and the child's group list is exactly
+the `/etc/group` account-database list — root's gid `0`, which PID 1 holds, has **vanished**. That is
+a `setgroups()` that *replaces* rather than appends, so anything Kubernetes puts in
+`supplementalGroups` reaches `s6-svscan` and is discarded before the app ever starts. What actually
+works is the image's own `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner`, which `find`s
+every char device under `/dev/dri`, `stat -c '%g'`s its GID, `groupadd`s a matching group inside the
+container when none exists, and `usermod -a -G`s the app user into it — every start, nothing
+hardcoded; `cont-init.d` completes before `services.d`, so the database is correct at exec time.
+
+Shipping an inert `supplementalGroups` would have been worse than shipping nothing: it would read as
+the mechanism to whoever debugs this next. The reasoning lives in a comment **at the emission site**
+and in `renderer-contract.md`, and `render.py --selftest` now asserts the *absence* of a
+`securityContext`, so it cannot be "helpfully" added back without a test failing. A pleasant
+corollary: because the hook re-`stat`s the device every start, host GID drift is **self-healing**,
+and no GID is pinned anywhere in this repo.
+
+**The design decision worth defending: the field is an enum of intent, not a device path.** The SD
+says *what class of host device the workload needs*; the hostPath and the mount exist only in
+`catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource
+requests, no hostPath at all) is then a renderer change with zero catalog edits. Both the schema
+`description` and the contract section say this explicitly, because the obvious "simplification" —
+letting an SD name `/dev/dri` — would push a platform implementation detail into the operator-facing
+contract and make that migration a catalog-wide edit.
+
+**`DirectoryOrCreate` is load-bearing.** A node that comes back without the render device (rebuild,
+BIOS change, driver regression, hardware swap) must degrade the workload to software transcoding,
+not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so a failed
+hostPath mount stalls the entire app tier for the sake of a feature that is not load-bearing.
+`/dev` is tmpfs, so the empty directory the kubelet creates on such a node leaves nothing durable
+behind. Do not "tighten" it to `Directory`. The reasoning is recorded at the emission site.
+
+### Rebase note (2026-10-01, Ruth Okonkwo) — resolution only, no redesign
+
+Rebased onto `main` @ `08deb20` (was cut from `0fbf544`). Re-landed as a single cherry-pick onto
+current `main`; the branch ref was force-updated with `--force-with-lease`, so PR #27 is the same
+PR with a new head.
+
+**One conflict, in `catalog/render.py`, purely textual.** PR #28
+(`fix/oauth2-proxy-cookie-collision-fleet`, merged as `6caa5f2`) inserted its own constants blocks
+(`OAUTH2_ACCESS_TOKEN_LIFETIME_S` / `OAUTH2_COOKIE_REFRESH`, and `OAUTH2_PROXY_METRICS_PORT`) at the
+same insertion point immediately after `KEYCLOAK_ISSUER` that this branch used for
+`DRI_VOLUME_NAME` / `DRI_DEVICE_DIR`. The two blocks are unrelated and do not interact, so **both
+sides were kept in full**, oauth2-proxy's first and the hardware block after it. Nothing was
+dropped and nothing was rewritten: the rebased commit adds the **same 126 `render.py` lines** as the
+pre-rebase commit (verified by comparing the added-line sets) and deletes none. In particular the
+`supplementalGroups`-is-deliberately-not-emitted and `DirectoryOrCreate`-is-load-bearing comments
+survive verbatim.
+
+Re-verified after the resolution: `--selftest` PASS (rc 0), and a full re-render still produces a
+**zero** diff in `apps/` — this PR remains inert, as it must be while no ServiceDefinition opts in.
+
+## What I verified
+
+Measured, in this worktree, on this branch:
+
+- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** New cases in the
+  existing `check(...)` style: the positive (volume + mount, and *no* `securityContext`, *no*
+  privileged, *no* nodeSelector), the negative that matters (an SD without `hardware` gains no
+  volume, no volumeMount and no `securityContext` — mirroring the existing `podSecurityContext`
+  negative), and a pass-through case (an SD declaring both `hardware` and its own
+  `podSecurityContext: {fsGroup: 1001, supplementalGroups: [44]}` keeps it **exactly** as written).
+- **The self-tests actually bite.** I mutated the guard (`if hw_gpu == "intel-quicksync":` → `if
+  True:`) in a scratch copy and re-ran: **rc 1**, failing on "an SD without `hardware` must gain no
+  dri volume, no mount and no securityContext". A test that cannot fail is not a test; this one can.
+- **Re-render is a zero diff.**
+  `python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` then
+  `git diff --stat apps/` → **empty**, and `git status --porcelain apps/` → **empty** (so the two
+  not-onboarded by-products, `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, were
+  rewritten byte-identically and needed no revert). The change is inert until a service opts in.
+- **Schema validity and catalog conformance.** `Draft202012Validator.check_schema` passes; all
+  **12** `catalog/*.yaml` validate with **0** errors. On the new field specifically: `gpu:
+  intel-quicksync` accepted; `gpu: /dev/dri` rejected (enum); `hardware: {gpu: …, path: …}` rejected
+  (`additionalProperties: false`); `hardware: {}` rejected (`minProperties: 1`).
+- **Hand-checked the emitted YAML** by rendering a throwaway SD carrying `hardware`, a Longhorn
+  volume and its own `podSecurityContext: {runAsUser: 1000, supplementalGroups: [44]}`. Result:
+  `volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `volumes` = the PVC then
+  `{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`; `securityContext` =
+  `{runAsUser: 1000, supplementalGroups: [44]}` — i.e. **unchanged**, nothing appended. That toy SD
+  was never written into `catalog/`.
+- **GATE: `rc 3` — UNDETERMINED, NOT green.** Run in the pinned toolbox from this worktree with
+  both CLAUDE.md §3 workarounds (`DEV_KUBECONFIG=/dev/null`, and an `ENGINE` shim bind-mounting the
+  main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f`; touched
+  requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Run **four** times on this branch with an identical
+  result each time, most recently against tree `c132893045b1` — the final commit differs from that
+  run only by this paragraph. Per stage:
+
+  | stage | result |
+  |---|---|
+  | spec-immutability | **SKIPPED** — sibling spec repo not at `/work/../home-network-k3s`. Expected from a worktree; tolerated by §3. |
+  | decisions-log | pass |
+  | lint | pass |
+  | l0 | pass — 19 offline cases, `0 failure(s), 0 skip(s), 0 unwired` |
+  | render-golden | pass — *"all 10 onboarded service golden(s) reproduce from the canonical render"* |
+  | l1 | pass — 3 offline checks, `0 failure(s), 0 skip(s), 0 unwired`; coverage 51/51 requirements scaffolded |
+  | approval-policy | pass (`event='none'`, not a pull_request event — N/A locally; it will evaluate for real on the PR) |
+  | gitea-settings | **SKIPPED** — `SETTINGS_API_TOKEN` unset. Expected; branch protection and runner registration are therefore UNVERIFIED by this run. |
+  | conformance (L2/L3) | **SKIPPED** — `CLUSTER_AVAILABLE!=true`. Phase-0 standing skip. |
+
+  The gate's own verdict: *"Stages that did not run: spec-immutability gitea-settings"*. **No stage
+  failed, and no stage skipped for a reason caused by this change.** rc 3 is reported as rc 3; I am
+  not calling this green.
+- **Blast radius of the schema change:** `hardware` is absent from every catalog entry, so
+  `apply_defaults` never materializes it and `s.get("hardware")` is `None` on all 12 — which is what
+  the zero diff above demonstrates rather than asserts.
+- **Touched no sensitive path.** Changes are confined to `schemas/`, `catalog/`, `decisions/`; per
+  `infra/gitea/approval-policy.json` none of those are sensitive. `conformance/**` and `clusters/**`
+  are untouched. I did not run `--emit-waves` (it writes `clusters/base/`, and the set of files
+  under `apps/` did not change).
+
+## What I did NOT verify
+
+- **Whether containerd's device cgroup permits a non-privileged pod to `open()` a bind-mounted char
+  device. Still unverified, and it is the single largest technical risk in this workstream.**
+  containerd installs a per-container **device cgroup** allow-list, and whether
+  `/dev/dri/renderD128` arriving as a bind-mounted device node is permitted by it on this
+  k3s/containerd version is **unmeasured**. Settling it requires applying a pod, which is an
+  operator action, not an agent one. The container's PID 1 running as uid 0 helps on the DAC side
+  but does **not** bypass the cgroup gate. If it blocks, the mount will succeed, the image's group
+  hook will do its job, and the `open()` will still fail — and the fallback is the Intel GPU device
+  plugin, which *sets* the cgroup entry. Note that that fallback is a **renderer-only change**: no
+  ServiceDefinition moves, because the authoring surface is an intent enum. That is the design
+  justifying itself, and it is the reason to keep the enum even if the hostPath mechanism loses.
+- **That a missing device degrades silently — and nothing will tell you.** This is the accepted cost
+  of `DirectoryOrCreate`. The workload will software-transcode and **no signal reports it**: media
+  apps in this catalog declare no `metricsEndpoint`, so no ServiceMonitor exists, and there is no
+  metric to alert on. I deliberately did **not** propose an alert, because an alert that cannot fire
+  is worse than none — see the 2026-09-13 correction in `renderer-contract.md` §`observability`,
+  where twelve monitors discovered zero targets. The only proof of hardware transcode is an operator
+  seeing it in the application's own dashboard.
+- **That "mounting the device is sufficient" generalizes beyond this one image.** It does not, and
+  I have not verified that it does. The claim was measured for an image that resolves the device
+  group itself in `cont-init.d`. A future consumer of `hardware.gpu` that runs non-root with no
+  equivalent hook would still need the group solved — and the fix for that case is a **renderer**
+  change (per-image handling, or the device plugin), never a new SD field. I scoped the claim in
+  `renderer-contract.md` rather than letting it read as general; the first such consumer must
+  re-measure.
+- **The feature now depends on a hook inside an image that floats.** The `cont-init.d` behaviour
+  above was read from digest `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), but the
+  deployment runs `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. **A silent pull can
+  change the behaviour this feature depends on, with no commit anywhere in this repo to show for
+  it** — and the failure would be the silent software-transcode described above. I have not fixed
+  that here (it is the Plex wiring's tag/digest decision, not the capability's), but it should not
+  be forgotten.
+- **GID drift is NOT on this list any more, deliberately.** An earlier revision pinned
+  `INTEL_RENDER_GID = 992` and listed "a node rebuild could move it" as an open risk. The hook
+  re-`stat`s the device on every start, so drift is self-healing and no GID is pinned anywhere. The
+  risk is retired, not overlooked.
+- **Anything on the live cluster.** I ran no `kubectl` and I opened no shell in any container. The
+  node facts (device present, `root:render` 0660, `i915` bound, no device plugin, no
+  `gpu.intel.com/*` advertised), the in-container measurements quoted above, and the Kyverno
+  posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups`
+  restriction; no PSA labels on app namespaces) are **prior recon handed to me, not re-measured by
+  me**. Per CLAUDE.md §2b, treat them as second-hand until re-queried.
+- **That CI exercises the self-tests. It does not.** `git grep -- --selftest ci/` returns nothing —
+  no gate stage invokes the renderer's self-test. They are the executable spec and house style, and
+  I ran them by hand, but a future regression in this code path **will not be caught by the gate**.
+  Wiring `--selftest` into a gate stage is a real gap and is not part of this PR (`ci/` is a
+  sensitive path and would drag in a human reviewer for a change unrelated to this one).
+- **Runtime behaviour of any kind.** The gate proves the L0/L1 offline subset only;
+  `vars.CLUSTER_AVAILABLE` is unset, so L2/L3 conformance does not run in CI.

decisions/2026-10-01-intel-gpu-device-plugin.md

@@ -0,0 +1,198 @@
+---
+date: 2026-10-01
+branch: feat/intel-gpu-device-plugin
+title: Intel GPU device plugin, in its own Kustomization — the resource exists; nothing has run yet
+---
+
+## What changed
+
+Deployed the **Intel Device Plugin for Kubernetes (GPU)**, standalone DaemonSet, pinned at
+**v0.37.0**, so the prod nodes advertise `gpu.intel.com/i915`. This is the measured fix for the
+Plex hardware-transcoding blocker: the `/dev/dri` hostPath merged on `main` @ `cdf3e14` makes the
+device node *visible* and not *openable* (`EPERM` from the device cgroup at both uid 0 and uid
+1000; the container's runc `config.json` carries one deny-all rule and `linux.devices: None`). A
+hostPath is a filesystem operation and adds no cgroup allow rule. A device plugin's `Allocate`
+response is what makes kubelet populate `linux.devices` *and* the matching cgroup allow rule, and
+it is the only non-privileged way to get there — `privileged: true` is blocked by Kyverno
+`restrict-host-namespaces` under Enforce in app namespaces, and `plex` is labelled
+`app-namespace=true`.
+
+New files:
+
+- `platform/intel-gpu-plugin/intel-gpu-plugin.yaml` — the DaemonSet, **vendored** as
+  `kubectl kustomize` output of upstream `deployments/gpu_plugin/base` at `v0.37.0`, under a
+  provenance header carrying the exact re-vendor command and both sha256s (generated body
+  `80ebd070…`, raw upstream source `c8773918…`). Generated rather than byte-copied because
+  upstream mixes YAML sequence-indentation styles within the one document and `ci/yamllint.yaml`
+  sets `indent-sequences: consistent` — a verbatim copy is a hard lint error that blocks the gate.
+  That is not a guess: the first gate run of this branch returned **rc 1 BLOCKED** on exactly that
+  error, and regenerating through kustomize is the fix (and is also the re-vendor idiom
+  `platform/external-snapshotter` documents). Vendored at all, rather than referenced as a remote
+  kustomize base, for the other reason `external-snapshotter` records: a remote base is
+  `git fetch`ed on every reconcile and those fetches have flaked a whole tier.
+- `platform/intel-gpu-plugin/namespace.yaml` — a PLATFORM namespace, explicitly
+  `homelab.keithjackson.dev/app-namespace: "false"` (following `vpn-egress`/`kyverno`).
+- `platform/intel-gpu-plugin/kustomization.yaml` — namespace scoping plus the image pin.
+- `platform/intel-gpu-plugin/README.md` — the measurement, the placement argument, the
+  non-choices, and the post-merge verification sequence.
+- `clusters/base/intel-gpu-plugin.yaml` — **its own Flux Kustomization**: `dependsOn: [cilium]`,
+  `wait: true`, `interval: 10m`, `timeout: 10m`, `prune: true`.
+- One wired line (plus its rationale comment) in `clusters/base/kustomization.yaml`.
+
+**Placement: its own Kustomization, not a line in the shared `platform` tier.** The shared tier was
+cheaper and needed no human reviewer on `approval-policy.json`'s terms, but `platform` is
+`wait: true` over eleven-plus components and the repo already argued this case when Cilium was
+split out: its Ready condition cannot distinguish "the plugin is wedged" from "longhorn is slow",
+the only emergency valve is freezing the entire tier, and the tier's 10m/5m interval/timeout are
+called out there as *wrong for a 3-node DaemonSet rollout* — which is exactly what this is. The
+consequence is worse than average here, because `platform` gates `platform-config` gates
+`apps-wave-0` (`wait: true`) gates `apps-wave-1`: a plugin that never went healthy inside that tier
+would stall every app, so a component added to fix a **silent no-op** would have gained the ability
+to cause a **real outage**. The approval cost of touching `clusters/**` is nil, because
+`required_approvals: 1` with `block_admin_merge_override: true` applies to every PR here regardless
+of path and authors cannot self-approve.
+
+`dependsOn: [cilium]` is the minimum *and* the earliest: the plugin never uses the network and
+never talks to the API server (it talks to the local kubelet over a unix socket), so a working CNI
+is its only real prerequisite, and `cilium` already depends on `flux-system`, which applies the
+`cluster-vars` ConfigMap this Kustomization substitutes from. Depending on `cilium` rather than
+`platform` also means it reconciles *in parallel with* the platform tier rather than behind all
+eleven of its components, which shrinks the cold-start window in which a node has not yet
+advertised the resource.
+
+`wait: true` is a deliberate departure from the `headlamp`/`crafty` leaf template. Those use
+`wait: false` because a leaf UI can be legitimately not-Ready for a long time during a cold start,
+making its health a useless signal. This DaemonSet reads sysfs and opens a socket, so not-Ready
+genuinely means broken — and since **nothing in the graph `dependsOn` it** (verified against the
+rendered graph), `wait: true` gates nothing while turning
+`flux get kustomization intel-gpu-plugin` into a one-line answer to the question that must be
+answered before the consumer PR may merge.
+
+Deliberate non-choices, each argued in the README: standalone DaemonSet rather than the operator
+(no CRD, no controller-manager, no cert-manager webhook, no second namespace to keep healthy);
+`--shared-dev-num` left at the upstream default of 1 (a higher count is a lie the scheduler
+believes — there is no GPU-side partitioning behind it, so it converts a loud "Unschedulable" into
+a quiet "transcodes are slow"); `--health-management` left OFF, because it is the one flag that
+*would* open a device and would reintroduce the exact `EPERM` this fixes; no NFD overlay (all three
+nodes are identical); no `supplementalGroups` anywhere, because the `pms-docker` cont-init hook
+already adds the `plex` user to each device's group and that works identically on a
+plugin-injected device.
+
+**Scope held deliberately:** `catalog/render.py`, `catalog/plex.yaml` and `apps/wave-0/plex.yaml`
+are untouched. Switching Plex from the hostPath to a `gpu.intel.com/i915: 1` request is a strictly
+later PR, and **it must not land until this plugin is merged AND confirmed advertising the resource
+on all three nodes** — a request for a resource no node advertises makes the Plex pod
+Unschedulable, and `apps-wave-0` is `wait: true` with `apps-wave-1` depending on it, so that would
+stall the whole app tier and turn today's silent no-op into a real outage.
+
+## What I verified
+
+Mechanism, read from the v0.37.0 source rather than assumed:
+
+- **The plugin only lists devices; it never opens one.** The scan path uses `os.ReadDir`,
+  `os.ReadFile`, `os.Stat` and `os.Readlink` over `/sys/class/drm` and `/dev/dri`. `grep` for
+  `os.Open|OpenFile|ioctl|syscall|unix\.` across `cmd/gpu_plugin/gpu_plugin.go` (908 lines) returns
+  **nothing**. Listing and `stat()` are what demonstrably work today, so the plugin needs no cgroup
+  grant and no extra privilege — it does not need more than the upstream manifest gives it.
+- **The fix mechanism is real:** `createDeviceSpecsFromDrmFiles` emits
+  `pluginapi.DeviceSpec{HostPath, ContainerPath, Permissions: "rw"}` per DRM node, which is the
+  thing kubelet turns into `linux.devices` plus a cgroup allow rule.
+- **The resource name is `gpu.intel.com/i915` by derivation, not by hope:** `scan()` calls
+  `devTree.AddDevice(devProps.driver(), ...)`, so the suffix is the kernel driver name, and these
+  nodes have `i915` bound.
+- `--shared-dev-num` default is `1` (`flag.IntVar(..., 1, ...)`); `--health-management` default is
+  `false` and `setupLevelZeroService` returns early unless it is set.
+- Missing `/dev/dri/by-path` is non-fatal (`newDevicePlugin` warns, sets `bypathFound = false`).
+
+Image pin, verified against the registry:
+
+- `docker.io/intel/intel-gpu-plugin:0.37.0` exists; manifest digest
+  `sha256:7001200071db10587fc31184eedda4f5cf291efba737adf85a8c80539f7fa5e1` (single amd64
+  manifest, not an index), fetched from `registry-1.docker.io` on 2026-10-01. `v0.37.0` is the
+  latest upstream release (2026-09-16) and is not a prerelease.
+- The Docker Hub tag list carries `0.35.0 / 0.36.0 / 0.37.0` — **image tags have no `v` prefix**
+  while GitHub release tags do.
+
+Policy scope, read from the policies rather than assumed:
+
+- All four app-scoped Kyverno policies (`restrict-host-namespaces`, `require-egress-class-label`,
+  `require-auth-mode-label`, `require-default-deny-baseline`) are gated on
+  `namespaceSelector`/`selector` `matchLabels: homelab.keithjackson.dev/app-namespace: "true"`.
+  The new namespace is labelled `"false"`, so none of them match it.
+- Measured on prod: `plex` **is** `app-namespace=true`, confirming `privileged: true` is genuinely
+  unavailable there; `kyverno` and `vpn-egress` are the two namespaces carrying the label as
+  `"false"`; the other platform namespaces leave it unset. No `intel*` namespace exists, so no
+  collision.
+
+Cluster before-state (read-only, `kubectl` in the pinned toolbox with the prod kubeconfig
+bind-mounted read-only — never copied into the worktree):
+
+- Three nodes `k3s-prod-{1,2,3}`, each `v1.36.4+k3s1`, Debian 13 trixie, `containerd://2.3.4-k3s1.36`.
+- Every extended resource currently advertised is `devices.kubevirt.io/{kvm,sev,tun,vhost-net}` on
+  all three nodes. **No node advertises `gpu.intel.com/*`** — queried explicitly, returned
+  "NONE". That is the before-state this change is meant to move.
+- Those `devices.kubevirt.io/*` resources are themselves advertised by a device plugin, which is
+  measured evidence that the device-plugin API and the default kubelet socket path
+  (`/var/lib/kubelet/device-plugins`) work on these nodes.
+
+Offline rendering (the main proof available to a PR):
+
+- `kubectl kustomize platform/intel-gpu-plugin` renders two objects: `Namespace/intel-gpu-plugin`
+  carrying `app-namespace=false`, and `DaemonSet/intel-gpu-plugin` in namespace
+  `intel-gpu-plugin`, with the image resolved to `docker.io/intel/intel-gpu-plugin:0.37.0`.
+- `kubectl kustomize clusters/base` renders, and the new Flux Kustomization appears as
+  `intel-gpu-plugin  ./platform/intel-gpu-plugin  wait=true  dependsOn=cilium`. Both overlays
+  (`clusters/prod`, `clusters/dev`) also render and carry it.
+- Across the whole rendered graph, **no Kustomization lists `intel-gpu-plugin` in `dependsOn`** —
+  so `wait: true` gates nothing.
+- The vendored DaemonSet is a **formatting-only** transform of upstream, proven rather than
+  asserted: `yq -P 'sort_keys(..)'` of the generated file and of the raw upstream file `diff`
+  clean, so passing it through kustomize changed no content. The committed body's sha256 matches
+  the one recorded in its own provenance header (`80ebd070…`), so the file has not drifted from
+  what was generated.
+
+Gate (pinned toolbox, from this worktree, with the `ENGINE` shim bind-mounting the main checkout's
+gitdir and `DEV_KUBECONFIG=/dev/null`): see the PR description for the per-stage table. **`rc 3`
+(UNDETERMINED), with exactly the three expected skips** — `spec-immutability` (sibling spec repo is
+not beside a worktree), `gitea-settings` (no CI token), `conformance` (no cluster in CI). Every
+other stage passed. **rc 3 is not green** and is not reported as such.
+
+## What I did NOT verify
+
+**Nothing about this component's runtime behaviour. It has never run.**
+
+- **The plugin has never executed.** No pod has been scheduled, no image has been pulled, no scan
+  has happened. It cannot run until the operator merges this and Flux reconciles — `kubectl apply`
+  and `flux reconcile` are operator actions (CLAUDE.md §5), so this is expected, not an oversight.
+- **The resource has never been advertised.** `gpu.intel.com/i915` does not exist on any node and
+  I have not seen it exist. The claim that it *will* be `i915` rather than `xe` is derived from
+  source plus the established fact that `i915` is bound — it is a derivation, not an observation.
+- **The DaemonSet has never been admitted.** That the Kyverno policies do not match this namespace
+  is read off the policy manifests; no admission request has actually been evaluated. Nor has the
+  `kubectl kustomize` output ever been server-side dry-run against the API — rendering is not
+  admission.
+- **The `open()` test has NOT been re-run.** The `EPERM` is exactly as it was. Nothing in this PR
+  touches Plex, so Plex is still software-transcoding and will continue to until the later renderer
+  PR lands. The decisive evidence is an `open()` as **uid 1000 through `s6-setuidgid plex`**;
+  passing as uid 0 is weaker and must not be accepted as proof.
+- **A green gate proves the manifests render, nothing more.** And this gate was not green: it was
+  `rc 3`. Note also that the Gitea `gate` job runs `./ci/gate.sh || [ "$?" -eq 3 ]`
+  (`.gitea/workflows/gate.yml:138`), so a green tick on this PR cannot distinguish rc 0 from rc 3,
+  and `vars.CLUSTER_AVAILABLE` is unset so no L2/L3 conformance runs in CI.
+- **`seLinuxOptions: type: container_device_plugin_t`** is carried verbatim from upstream. These
+  nodes are Debian 13 and I did not measure whether SELinux is enabled or whether that type exists
+  there. Expected to be inert (runc skips labelling when SELinux is disabled), but unverified — if
+  the pods fail to start with an SELinux error, that line is the first suspect.
+- **Renovate has not been run** against this pin. The inline-marker idiom is copied from
+  `platform/kubevirt` / `platform/external-snapshotter`, but whether Renovate normalises the
+  `v`-prefix mismatch (release `v0.37.0` vs image `0.37.0`) for this dep is unknown. If it proposes
+  `newTag: v0.38.0`, the fix is an `extractVersion` packageRule in `renovate.json`.
+- **Resource limits unexamined under load.** The upstream 115Mi/100m limits are taken as-is; I have
+  no measurement of this plugin's footprint on these nodes. Worth a glance given
+  `keycloak-operator` is OOMing at 91% of its limit elsewhere in this cluster.
+- **`clusters/dev` renders it, and there is no dev cluster** (the operator confirmed the
+  `k3s-dev-*` nodes are gone). That is inert rather than wrong — the unified graph (R-ENV-02) has
+  both overlays render the same base — but it is not a thing I could test.
+- **Branch protection / required-checks state not re-measured** in this session. CLAUDE.md §2b
+  says re-measure before relying on it; I did not, so treat the approval/gating claims in §2 as
+  unconfirmed-by-me.

decisions/2026-10-01-oauth2-proxy-api-route-per-app.md

@@ -0,0 +1,482 @@
+---
+date: 2026-10-01
+branch: feat/oauth2-proxy-api-route-per-app
+title: The config model had no way to say "this app's API lives here" — so --api-route could not ship. Now it can, per app.
+---
+
+## What changed
+
+Second half of PR #28 (`decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md`), which
+fixed the cookie collision, the 5-minute session and the missing metrics listener in one renderer
+function and **deliberately declined `--api-route`** for a reason I have preserved rather than
+worked around: that flag was measured on **one** app, and `PiHole serves no /api/ at all`, so a
+renderer-wide constant would apply one app's path shape to five. The `auth` block is
+`additionalProperties: false`, so there was no per-app override hook. **Creating the hook was the
+work; setting the flag is the easy part.**
+
+**Three surfaces, in order of what they protect:**
+
+| Surface | Change | What it guards that the others cannot |
+|---|---|---|
+| `schemas/servicedefinition.schema.json` (**1.8.0 → 1.9.0**) | new optional `auth.apiRoutes`: array of anchored path regexes, `uniqueItems`, default `[]` | rejects a too-broad regex **where a human types it** — `ci/l0.sh` validates `catalog/*.yaml` against this schema on every gate run |
+| `catalog/render.py` | emits one `--api-route=<r>` per declared entry, in order; **raises** on an unanchored entry | `ci/render-golden.sh` runs on every gate invocation even when the schema stage skips |
+| `conformance/tests/l0-static/oauth2-proxy-api-routes-per-app/` (new) | cross-checks each rendered sidecar against its own ServiceDefinition | catches a **renderer constant**, which neither of the above can see |
+
+**The `auth` block stays `additionalProperties: false`.** One named, documented key was added; the
+door is not opened. That property is the reason the gap existed, and it is worth keeping — it is
+what made "there is no override hook" a *fact* rather than a *guess* for the two engineers who hit
+it before me.
+
+**Rebased mid-flight.** `origin/main` advanced from `08deb20` to `cdf3e14` while I was working
+(PR #27 and PR #32, the `hardware.gpu: intel-quicksync` capability). **That change also took schema
+`1.8.0`**, so mine is renumbered to **1.9.0** and chained behind it; both version entries are
+present in the description, and `catalog/render.py` had a two-sided conflict in the constants block
+which is resolved by keeping both (the `DRI_*` constants and `OAUTH2_API_ROUTE_PATTERN` are
+unrelated). Re-rendering after the rebase still changes **only** jackett and radarr — plex's new
+golden reproduces byte-identically, so the two changes do not interact.
+
+**Shape:** a **list** of regexes, not a single string, because Radarr needs two
+(`^/api/` and `^/signalr/`) and `--api-route` is natively repeatable. A single string would have
+forced alternation into the value, and alternation is exactly how breadth gets smuggled past a
+prefix-only reading of a regex.
+
+**The anchoring ratchet.** Each entry must match `^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$` — anchored at
+`^/`, then literal path characters, with an optional trailing `.*` or `$`. The same pattern is in
+the schema, in the renderer, and in the new check, and assertion (5) of the check **fails if they
+drift apart**, because if they do the loosest one wins silently. Rejected shapes, all five
+provoked: `^/`, `^/.*`, `.*`, `/api/` (unanchored), `^/api/|^/` (alternation). Alternation is
+**refused rather than supported**: the flag is repeatable, so one entry per route costs nothing.
+
+**Why breadth is the dangerous direction, and it is not symmetric.** A *missing* `--api-route` is a
+silent 302 — bad, invisible, recoverable. A *too broad* one answers a plain **page load** with 401:
+a logged-out operator gets a bare error page and is **never offered a sign-in**, so the service
+looks permanently broken and no reload recovers it. That is a worse outcome than the defect the
+flag fixes, which is why it is gated on three surfaces rather than documented in one.
+
+### Also folded in: the selftest could exit 1 with ZERO diagnostics
+
+Not my change originally — Desmond Oyelaran found it while wiring `catalog/render.py --selftest`
+into the `render-golden` stage (his PR #33, which touches only `ci/render-golden.sh` and is **not**
+in my base; `origin/main` at `cdf3e14` does not yet drive the selftest from any gate stage). The
+coordinator asked me to fold the fix into this PR because the defect is in the file I own.
+
+`_selftest()` reached `_cookie_name()` through a bare `next(genexpr)` with **no default**. Delete
+the `--cookie-name` emission — exactly the mutation that assertion exists to catch — and it raises
+`StopIteration`, which escapes `_selftest()` **before its collected `SELFTEST FAIL:` lines are
+printed**. Right verdict, no diagnostics, and it masks every other assertion the same mutation
+broke. Fixed with `next(..., None)` plus an explicit `check()` so a missing flag reports **as a
+named assertion**.
+
+Provoking it turned up **three more instances of the same failure class** in the same block, which
+is why this is not literally a one-line change: `side = fa_containers[1]`,
+`side["env"][0][...]`, and the `fa_svc` lookup all raise rather than assert when the sidecar is
+absent entirely. All four are now None-tolerant, and the two fixtures I added for `--api-route`
+were written the same way from the start. Measured, on throwaway copies:
+
+| Mutation | Before | After |
+|---|---|---|
+| `--cookie-name` emission deleted (origin/main's renderer) | rc 1, **0** `SELFTEST FAIL` lines, 1 `StopIteration` | rc 1, **3** `SELFTEST FAIL` lines, 0 tracebacks |
+| the oauth2-proxy sidecar not appended at all | rc 1, **0** lines, `IndexError` | rc 1, **20** `SELFTEST FAIL` lines, 0 tracebacks |
+
+Note the consequence of his stage change for anyone reading this later: once PR #33 lands,
+`.gitea/workflows/gate.yml:138` runs `./ci/gate.sh || [ "$?" -eq 3 ]`, so **rc 1 is not tolerated
+in CI the way rc 3 is** — a selftest failure will block. Which is the right outcome, and it is why
+a selftest that prints nothing was worth fixing before that lands rather than after.
+
+### Declined, with reasons: the metrics-ingress NetworkPolicy
+
+The coordinator asked for my judgement on bundling Ines Dubois's finding — that #28's metrics
+listener genuinely serves (she measured `http=200`, 10,208 bytes,
+`oauth2_proxy_requests_total{code="302"} 2` via port-forward) but the scrape will be **dropped**,
+because namespace `jackett`'s default-deny has **no `ingress` key at all** and
+`jackett-allow-gateway-ingress` admits `fromEntities: [ingress]` only. The fix belongs in each
+app's own rendered policy, i.e. in my file and my goldens. **I declined to bundle it**, and asked
+for it as the immediate next PR. Five reasons, in the order that decided it:
+
+1. **The real reason is that it is a MODEL question, not a policy to bolt on.** Schema 1.6.0
+   already owns this: `observability.metricsEndpoint` renders the Service port **and** the
+   monitoring-namespace ingress allow. What the model cannot currently express is a **second**
+   metrics endpoint on the same pod belonging to a *sidecar the renderer injected*, rather than to
+   the workload the operator declared. Keying a bespoke policy off the pod label
+   `homelab.keithjackson.dev/auth-mode: forward-auth` works today, but it hardcodes "forward-auth
+   implies a metrics port on 44180" into a policy instead of **deriving** it from the sidecar
+   declaration that is already the single source of that port — the exact "list, don't derive"
+   anti-pattern `oauth2-proxy-cookie-names-distinct` assertion (5) exists to forbid, and which I
+   would be introducing one directory away from the check that forbids it. That deserves its own
+   review, and it is squarely the kind of decision I was brought in to make deliberately.
+2. **Different subject, and it is a trust-boundary change.** Mine is the auth *response code* on
+   API paths (R-IDP-02). That is an ingress allow (R-NET-01/02, R-OBS-01) admitting a new
+   namespace to a new port. Bundling a security-relevant ingress rule into a PR whose headline is
+   `--api-route` is how such a rule gets waved through on the strength of the other half — and this
+   PR already needs a human for a *different* sensitive-path reason, so the second subject would be
+   reviewed in the shadow of the first.
+3. **The conflict argument does not actually apply to my diff.** It assumes I am regenerating five
+   goldens; **I am regenerating two** (jackett, radarr), because the flag renders only where
+   declared. A metrics-ingress policy renders a *different document* — a CiliumNetworkPolicy — in
+   all five. Different hunks, in files git merges independently; and goldens regenerate
+   deterministically from the renderer, so the follow-up rebases onto `main` cleanly. The N-1
+   rebase problem `decisions/README.md` records was two branches appending to **the same hunk**,
+   which this is not. I checked my own diff before answering rather than accepting the premise.
+4. **Every fact in the finding is hers and I have measured none of it** — not the 200/10,208 bytes,
+   not the `wget: download timed out` on `:44180` *and* `:4180`, not the Flux sibling ordering that
+   makes `oauth2-proxy` and `apps-wave-0` unordered. Shipping a security-adjacent rule on wholly
+   inherited evidence, as a footnote in someone else's PR, is precisely what CLAUDE.md §2b exists
+   to stop. In its own PR I would measure it first.
+5. **Sequencing: the PodMonitor is in HER PR #29, not mine.** If I ship the policy and her
+   PodMonitor is reworked or delayed, I have opened an ingress allow for a scraper that selects
+   nothing — a hole with no consumer. The policy is only useful paired with the PodMonitor, which
+   argues for it landing adjacent to that, not here. It also drags in a
+   `KNOWN_EXTERNAL_MONITORS` entry in `conformance/tests/l0-static/servicemonitor-ports-resolve/`,
+   a second conformance interaction on top of the new check this PR already adds.
+
+Diff size is the tiebreaker rather than the argument: **1,245 insertions across 13 files**,
+including a ~400-line new conformance check. I would rather hand a reviewer one subject they can
+actually falsify. **Queued, not dropped** — and if the next person takes it, the model note in (1)
+is the part worth arguing about, not the YAML.
+
+### Which apps got which routes, and the measurement behind each
+
+| App | `auth.apiRoutes` | Evidence |
+|---|---|---|
+| **jackett** | `^/api/` | 2026-09-30, its own access log: **33 × 302 against 2 × 401** on `/api/` paths, and **4 of 6 POSTs answered by the proxy and never reaching Jackett** (upstream field `-`, not `/`), including both `/api/v2.0/server/update` calls |
+| **radarr** | `^/api/`, `^/signalr/` | `^/signalr/`: measured **by me, on the live post-fix pod** — see below. `^/api/`: measured on radarr by the PR #30 author, on the **pre-fix** pod |
+| **pihole** | *(none — deliberate)* | serves no `/api/`; **0** unauthenticated requests in its current pod's 23 h |
+| **sonarr** | *(none — not measured)* | 13,643 log lines, **0** unauthenticated requests. No traffic, not a negative result |
+| **qbittorrent** | *(none — not measured)* | 13,649 log lines, **0** unauthenticated requests. Same |
+
+**`^/signalr/` on radarr is the live proof this change was needed**, and it is the strongest
+evidence in this entry. 23 h after PR #28 deployed, on `radarr-7d55cbffb9-42hmh` (0 restarts), the
+sidecar logged **232 unauthenticated requests across the pod's entire lifetime, and all 232 are the
+same call**: `POST /signalr/messages/negotiate`, browser UA, **302**, upstream `-` (Radarr never saw
+one of them). That is a **left-open browser tab** whose session the cookie rename orphaned, polling
+SignalR forever, silently redirected every time, and **never re-prompting the operator**. It is the
+reported symptom — "reloading never fixes it and I am never re-prompted" — still happening in
+production, on a service nobody was looking at. With 401 the UI's connection breaks **loudly**; the
+operator reloads; `/` matches neither regex, so the reload signs them back in.
+
+**`^/api/` on radarr: included, and here is the honest basis.** The current pod has logged **zero**
+`/api/` traffic, so I did **not** re-measure it. It rests on a measurement of **radarr** recorded in
+the merged PR #30 handoff — its 156 unauthenticated requests were `/api/v3/command` (62),
+`/api/v3/queue/details` (46) and `/signalr/messages/negotiate` (43), i.e. **108 of 156 were
+`/api/v3` XHRs**. That is this app's own evidence, by another engineer, not an analogy to another
+app. I include it because `/api/v3/` is a pure JSON namespace a browser never navigates to (Radarr
+serves its UI at `/`, assets under `/Content/` and `/MediaCover/`), so the breadth risk is nil and
+the measured benefit is 70% of that app's rejection volume.
+
+**Sonarr and qBittorrent were measured and deliberately left alone.** Sonarr is nearly the same
+codebase as Radarr, which is exactly what makes `^/api/` + `^/signalr/` a tempting guess and still a
+guess. I read both pods' logs: **zero** unauthenticated requests, nothing but health probes, because
+nobody has opened either UI since the cookie fix rolled. **That is evidence of no traffic, not
+evidence of no need.** Each catalog file now carries a comment saying so, with the numbers and what
+to measure before adding the routes.
+
+### Goldens and fixtures regenerated
+
+Rendered with the canonical command
+(`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation`).
+**Two goldens changed, not five** — and that is the per-app property proving itself on the real
+tree rather than in a fixture:
+
+    apps/wave-0/jackett.yaml   + --api-route=^/api/
+    apps/wave-1/radarr.yaml    + --api-route=^/api/
+                               + --api-route=^/signalr/
+
+`apps/wave-0/pihole.yaml`, `apps/wave-0/qbittorrent.yaml` and `apps/wave-1/sonarr.yaml` are
+**byte-identical to `main`**. **No L1 fixture needed regenerating**: all three
+`conformance/tests/l1-rendering/*/fixtures/svcdef*.yaml` declare `auth.mode: forward-auth` with no
+`apiRoutes`, so they render no flag and their goldens are unchanged. PR #28 touched 5 goldens and 3
+L1 fixtures because it changed flags on *every* sidecar; this one changes flags only where a
+measurement says so, which is the whole point.
+
+### The L0 decision (and it is a decision, not a default)
+
+**Not** in `oauth2-proxy-cookie-names-distinct`. That check's subject is cookie **identity** — its
+inputs are names and domains, its failure mode is mutual session eviction, its register is the
+shared proxy's claim on the default name. `--api-route` shares none of those. Bundling them would
+make that check's name a lie and couple two unrelated ratchets, so a future api-route decision could
+only be expressed by weakening a *cookie* check. That is PR #28's own stated discipline, applied to
+my change rather than quietly dropped.
+
+**A sibling check, `oauth2-proxy-api-routes-per-app`** (whole-repo scope, so it runs on every gate
+invocation via `ci/l0.sh`'s `--include-scope` union — no bespoke stage, which
+`whole-repo-invariants-wired` would fail). **Nowhere was the wrong answer for exactly one reason:**
+the existing gates cannot see a renderer constant. `render-golden` compares the render to the
+goldens, so a constant **plus regenerated goldens agree with each other while both being wrong** —
+I provoked that and it regenerates with **rc 0**. The new check's assertion (2) compares each
+rendered sidecar against its **own ServiceDefinition**, which is the only reading that fails it.
+
+## What I verified
+
+- **`--api-route` is SINGULAR, read off the PINNED binary rather than taken from the brief.**
+  `podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help`:
+  `--api-route strings   return HTTP 401 instead of redirecting to authentication server if token
+  is not valid. Format: path_regex`. There is **no `--api-routes`** in the help output. The PM asked
+  me to verify rather than trust, and that is the result — the singular form is confirmed and the
+  help text also confirms the *semantics* I am relying on (401 instead of a redirect).
+- **The premise, in-cluster, read-only, through the pinned toolbox, `--context k3s-prod` named
+  explicitly** (CLAUDE.md §2b and §5). PR #28 **is** deployed: all five sidecars carry
+  `--cookie-name=_oauth2_proxy_<app>`, `--cookie-domain=<app>.home.keithjackson.dev`,
+  `--cookie-refresh=2m30s` and `--metrics-address=0.0.0.0:44180`. Pods, all **0 restarts**:
+  `jackett-6877fcd9f5-ht9d9`, `pihole-…`, `qbittorrent-5547bdb9bd-zq557`,
+  `radarr-7d55cbffb9-42hmh`, `sonarr-ddf485775-8ckwt` (started 2026-09-30T21:41–21:42Z).
+- **The full post-fix log census, parsed rather than grepped** (oauth2-proxy's access-log format,
+  with `/ping` health probes excluded), over each pod's whole life:
+
+      app            lines   unauth  AuthSuccess  removing session  cookie sig invalid
+      jackett       13,649        0            0                 0                   0
+      pihole        13,649        0            0                 0                   0
+      qbittorrent   13,649        0            0                 0                   0
+      radarr        14,099      232            0                 0                   0
+      sonarr        13,643        0            0                 0                   0
+
+  Three findings, kept separate:
+  1. **The collision fix works.** Jackett was 39 unauthenticated before; it is **0** in 13,612
+     non-probe lines. The eviction fingerprint (`removing session` / `cookie signature not valid`)
+     is **0 on all five**, where radarr previously had 8 and 8.
+  2. **Radarr's 232 are one request.** 100% `POST /signalr/messages/negotiate`, 100% **302**, 100%
+     upstream `-`, browser UA (`Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537…`). **Zero `/api/`
+     traffic of any kind** on this pod.
+  3. **0 `AuthSuccess` anywhere**, so nobody has signed in since the restart and `--cookie-refresh`
+     is **still** unexercised — unchanged from PR #28's own caveat.
+- **Only two goldens change, and three are byte-identical to `main`.** `git status` after the
+  canonical render lists exactly `apps/wave-0/jackett.yaml` and `apps/wave-1/radarr.yaml`.
+  `render-golden` is green.
+- **`catalog/render.py --selftest`: PASS**, with new assertions that (a) the `fa` fixture, which
+  declares nothing, renders **no** `--api-route`; (b) a fixture declaring two routes renders exactly
+  those two, in order; (c) the renderer **refuses** all five breadth shapes; (d) four legitimate
+  routes (`^/api/`, `^/signalr/`, `^/api/v3/.*`, `^/api/v2.0/server/update$`) are **accepted** — a
+  ratchet that blocks real API prefixes is a ratchet someone eventually deletes.
+- **The new check: rc 0, 43 assertions, 6 sidecars audited**, with its inventory printed so the
+  reviewer can see what it read:
+  `jackett -> ['^/api/'] | pihole -> (none) | qbittorrent -> (none) | radarr -> ['^/api/',
+  '^/signalr/'] | sonarr -> (none) | crafty -> (none)`.
+- **The check was tested in BOTH directions, on throwaway copies outside the repo**, never by
+  mutating the branch. Each mutation fired its own assertion class and nothing else:
+  - **Pre-change tree** (`git archive origin/main` + the test dropped in): **rc 1, 11 failures** —
+    no service declares routes, the renderer emits none, the renderer does not read `auth.apiRoutes`,
+    the pattern is absent from both renderer and schema, and the shipped renderer accepts all five
+    breadth shapes.
+  - **pihole's golden hand-edited** to carry a route it never declared: **rc 1, 2 failures** (the
+    catalog↔golden mismatch, and "declares nothing must render nothing").
+  - **radarr's golden widened** to `--api-route=^/`: **rc 1, 2 failures** (unanchored, plus the
+    mismatch).
+  - **Renderer turned into a constant and EVERY golden regenerated to match: the regen exits 0** —
+    i.e. `render-golden` is **happy** — and the new check exits **1 with 15 failures**, naming
+    pihole, qbittorrent, radarr and sonarr individually. **This is the case that justifies the
+    check's existence** and I provoked it rather than argued it.
+  - **Schema `items.pattern` deleted**: **rc 1, 1 failure** — the three-surface drift guard.
+  - **Unmutated control: rc 0.**
+- **The selftest diagnostics fix was PROVOKED, not reasoned.** Both mutations above were run
+  against throwaway copies of the renderer outside the repo (`origin/main`'s `render.py` for the
+  "before" column, mine for the "after"), never by mutating the branch. The pre-fix StopIteration
+  and the post-fix 3-and-20 named assertions are quoted from those runs. `--selftest` on the
+  unmutated branch: **PASS**. Goldens after the fix: **unchanged** (it touches only `_selftest()`).
+- **The check is selected by the whole-repo union.** `./conformance/run.sh --layer l0 --requirements
+  r-nosuchreq-99 --include-scope whole-repo --print-selection` lists
+  `tests/l0-static/oauth2-proxy-api-routes-per-app` among **21** tests (it was 20), i.e. it runs
+  even when the touched-requirement filter matches nothing.
+- **THE SCHEMA GUARD IS PROVEN WITH THE PINNED VALIDATOR `ci/l0.sh` ACTUALLY DRIVES**, not by
+  reading the pattern. `./schemas/validate.sh` (`check-jsonschema, version 0.29.4`) against seven
+  throwaway ServiceDefinitions written outside the repo:
+
+      good            ["^/api/", "^/signalr/"]   rc 0
+      bad-root        ["^/"]                     rc 1  apiRoutes ... does not match
+      bad-dotstar     ["^/.*"]                   rc 1  apiRoutes ... does not match
+      bad-unanchored  ["/api/"]                  rc 1  apiRoutes ... does not match
+      bad-alt         ["^/api/|^/"]              rc 1  apiRoutes ... does not match
+      bad-everything  [".*"]                     rc 1  apiRoutes ... does not match
+      bad-mode        ["^/api/"] on mode: oidc   rc 1  const 'forward-auth'
+
+  So a too-broad regex is rejected **where a human types it**, on every gate run, by the real tool
+  — and the shape invariant fires too.
+- **THE RENDERER'S REFUSAL REACHES `render-golden` AS A READABLE FAILURE, NOT A TRACEBACK.** With
+  `^/signalr/` swapped for `^/` in a *copy* of the catalog (outside the repo), both invocation
+  shapes exit **rc 1 with 0 tracebacks** — the single-SD form and `--all <dir> -o <dir>
+  --wave-layout --no-image-automation`, which is the one `ci/render-golden.sh` drives:
+
+      render.py: radarr: auth.apiRoutes entry '^/' is not an anchored path regex. Each entry
+      must start with '^/' and continue with literal path characters (e.g. '^/api/',
+      '^/signalr/', '^/api/v3/.*'). ... A too-broad --api-route answers a PAGE LOAD with 401, so
+      a logged-out operator sees a bare error page and is never offered a sign-in: the service
+      looks permanently broken rather than merely unauthenticated.
+
+  So the breadth guard is gated by `render-golden` even on a run where the schema stage skipped,
+  and the operator is told *why* rather than handed a stack trace.
+- **The schema's new shape invariant.** A non-empty `auth.apiRoutes` requires
+  `auth.mode: forward-auth`, because the flag renders onto the per-app sidecar and no sidecar exists
+  for `oidc`/`native`/`none` — declaring it elsewhere would be a **silent no-op**, which is the
+  failure class this repo keeps paying for.
+- **Gate:** toolbox `./ci/gate.sh` with the two-mount ENGINE shim — exit code and stage list in the
+  next section, because it is not a pass and that belongs there.
+- **No mutating cluster access.** `kubectl get/logs/jsonpath` only, `--context k3s-prod` named on
+  every call. No `apply`/`edit`/`patch`, no `flux reconcile`. No Secret `.data` or `.stringData` was
+  read or printed; I did not need to touch a Secret at all this session. The one container I ran
+  locally was the pinned oauth2-proxy image, to read its `--help`.
+- **`~/.git-credentials` was never read.** PR creation used `git credential fill`, which does not
+  echo the value.
+
+## What I did NOT verify
+
+- **NO BROWSER ROUND-TRIP. BY ANYONE. FOR THIS FLAG.** This is the most important line here. The
+  302/401 split, the dropped POST bodies and the orphaned SignalR poll are all read from access
+  logs; that a **401 repairs** the operator's experience is reasoned from HTTP semantics, not
+  watched. Nobody has added a Jackett indexer or recovered a Radarr tab through this
+  configuration. The human test below is the first real evidence this change will ever have.
+- **Local gate: rc 3 (UNDETERMINED), not 0.** Green: `spec-immutability`, `decisions-log`, `lint`,
+  `l0`, `render-golden`, `l1`. **Skipped:** `gitea-settings` (no `SETTINGS_API_TOKEN` locally) and
+  **L2/L3 conformance** (`CLUSTER_AVAILABLE` unset). Nothing FAILED; nothing runtime was certified
+  either. **rc 3 is UNDETERMINED, never green.**
+- **MY LOCAL `approval-policy: green` WAS MEANINGLESS, AND I ALMOST REPORTED IT AS EVIDENCE.**
+  `ci/approval-policy.sh:47` reads `GITHUB_EVENT_NAME`, which is unset outside CI, so the stage
+  short-circuits at line 50: `ok "not a pull_request event (event='none') — approval policy N/A"`.
+  **A clean local gate therefore CANNOT catch a sensitive-path problem — structurally, not by
+  accident.** Flagged to me by the coordinator from Ines Dubois's round, and I verified it by
+  re-running the gate with `GITHUB_EVENT_NAME=pull_request PR_NUMBER=34`: the stage then
+  **evaluates** and fails closed on the missing token —
+
+      [fail] no SETTINGS_API_TOKEN — cannot read this PR's changed files or its reviews.
+      [fail] stage approval-policy: FAIL
+      [fail] approval-policy: a sensitive change lacks a non-author approval -> blocks merge
+      [fail] MERGE GATE: BLOCKED        (rc 1, not 3)
+
+  — while **every other stage stays green** (`spec-immutability`, `decisions-log`, `lint`, `l0`,
+  `render-golden`, `l1`). That is the useful result: it bounds the CI failure to this one stage
+  rather than leaving "something is red" as the whole report. Fail-closed is correct here and
+  deliberate: unlike `gitea-settings`, where a missing credential is a lock-out, a skip of this
+  stage would be a bypass (CLAUDE.md §1).
+- **CI, read from each commit's `/statuses` rather than from a tick.** Both contexts report, and
+  **only one of them runs**. Identical on `99560c2d`, `df1b25b6` and `f1d16f44`:
+
+      gate / gate (pull_request)         state=failure   desc='Failing after ~1m43-1m45s'
+      gate / conformance (pull_request)  state=success   desc='Has been skipped'
+
+  **That second line is §2b's trap in the wild**: a *skipped* Gitea job reports
+  `state=success`. The conformance job did not execute, so its green tick certifies nothing —
+  read the description, not the colour. The duration is given as a range on purpose: it differs
+  by a second or two per run, and pinning one number inside the file whose commit triggers the
+  next run is a self-reference, not a measurement. The load-bearing fields are the **state** and
+  the **description**, and those are stable across all three.
+
+  Worth recording that the **same context changed its answer over time**: on the first push it
+  briefly read `state=pending desc='Blocked by required conditions'` — still waiting on the gate
+  job it depends on — and resolved to `success` only once that job settled. So sampling a status
+  once can show either a pending **or** a green tick for a job that never executed.
+  **I could NOT read the CI job log to confirm the failing stage from CI's
+  own output**: this Gitea's `/api/v1/repos/{owner}/{repo}/actions/runs` endpoint answers
+  **404** (only `/actions/tasks` exists, which lists tasks without logs), and
+  `/actions/jobs/{task_id}/logs` answers `404 Not found` for the task id. So my attribution of the
+  CI failure to `approval-policy` rests on the local event-set reproduction above plus the
+  sensitive-path rule, **not** on CI's own log. That is weaker evidence than PR #28 had and I am
+  naming it rather than borrowing its wording.
+- **Branch protection and the runner, RE-MEASURED this session against the live API** (§2b),
+  not inherited from CLAUDE.md §2's table:
+
+      GET /repos/Home-Network/home-network-k3s-impl/branches/main
+        protected              True
+        user_can_push          False
+        enable_status_check    True
+        status_check_contexts  ['gate / gate (pull_request)']
+        required_approvals     1
+      GET /admin/actions/runners
+        id=1 name=incluster-dind status=online
+
+  So `required_approvals=1` applies to **every** PR here regardless of path — this needs a human
+  even without the sensitive-path rule. I did **not** provoke a rejected push to `main` this
+  session; `user_can_push: False` is the API's word, which is exactly the weaker form of evidence
+  CLAUDE.md §2 warns about, and I did not strengthen it.
+- **`^/api/` ON RADARR IS NOT MY MEASUREMENT.** It is PR #30's, taken on the **pre-fix** pod. The
+  current pod has logged **zero** `/api/` traffic in 23 h, so I could not reproduce it. I judge the
+  breadth risk nil (`/api/v3/` is pure JSON; the UI is at `/`), but if someone wants a strictly
+  self-measured configuration, drop `^/api/` and keep `^/signalr/` — the 232 are entirely SignalR.
+- **SONARR AND QBITTORRENT ARE UNMEASURED, NOT CLEARED.** Zero traffic in 23 h is zero evidence.
+  qBittorrent's WebUI drives everything through `/api/v2/` and is *plausibly* affected the same way
+  Jackett is; Sonarr is nearly Radarr's codebase. **Both are left alone on purpose** — plausibility
+  is what put a renderer constant on the table in the first place. Expect to add `^/api/` to
+  qBittorrent and `^/api/` + `^/signalr/` to Sonarr once someone opens those UIs and records the
+  split. **This PR does not fix them.**
+- **PiHole is asserted to serve no `/api/` on inherited evidence, not mine.** Its log shows 0
+  unauthenticated requests, which says nothing about what paths it serves. The claim comes from
+  PR #28/#30. Note the pinned `update.range` is `>=2024.0.0`, and PiHole v6 *does* introduce a REST
+  API at `/api/` — so if PiHole is ever upgraded across that line, this needs re-measuring. It is
+  still correct to declare nothing today, because declaring an unmeasured route is the mistake.
+- **`--cookie-refresh=2m30s` has STILL never been exercised** — 0 `AuthSuccess` on all five pods.
+  PR #28 named this as the fix most likely to need a second pass and it remains unobserved. My
+  change does not touch it, but a sign-in during the human test below will exercise both at once,
+  so a failure there must be attributed carefully.
+- **The regexes are not validated against the apps' real route tables.** I assert `/api/v3/` and
+  `/signalr/` are JSON/RPC namespaces a browser never navigates to from Radarr's and Jackett's
+  observed traffic and from how those UIs are built — **not** from reading their routing code or
+  enumerating their endpoints. A path I have not thought of that matches `^/api/` and *is*
+  browser-navigated would 401.
+- **`OPTIONS` preflight and non-browser clients are unconsidered.** `--skip-auth-preflight` exists
+  and is not set; nothing in these logs shows a CORS preflight, and Sonarr/Radarr talk to Jackett
+  **directly on `:9117`**, bypassing the proxy entirely — so I do not expect impact. Unverified.
+- **The live pods do not carry this change and nothing re-measures that they will.** git is not the
+  cluster (§2b). **Merging is what deploys**, and it will roll **jackett** and **radarr** only
+  (`strategy: Recreate` on RWO Longhorn volumes, so a brief gap per app, and radarr/sonarr lose
+  direct `:9117` access to Jackett while its pod restarts). Existing sessions survive — no cookie
+  flag changed — so unlike PR #28 nobody is signed out.
+- **No L2/L3 assertion reads any of this.** There is no runtime check that a `/api/` request to a
+  gated app answers 401, and I did not write one; `CLUSTER_AVAILABLE` is unset so it could not run
+  in CI anyway. The whole guard is static.
+- **`catalog/render.py --selftest` is UNGATED IN MY BASE, and I did not wire it.** Desmond's PR #33
+  does that (`ci/render-golden.sh`) and is in review, not merged — `origin/main` at `cdf3e14`
+  contains no stage that drives it, which I checked rather than assumed (`grep -n selftest
+  ci/render-golden.sh` → no match). **So until #33 lands, my new selftest assertions are
+  dev-time only**, and the *gated* half of my breadth ratchet is the schema `pattern` (via
+  `ci/l0.sh`'s schema stage), the renderer's own `raise` (via `render-golden`), and the new L0
+  check. I deliberately did not wire the selftest myself: #33 already does it, and duplicating it
+  here would collide with his branch in the one file he owns. Note the two PRs are **independent
+  but complementary** — his makes the stage real, mine makes its output readable when it fires.
+- **I did NOT verify PR #33's own claims.** The 106-assertion count, the
+  selftest→stage→`gate.sh`→`BLOCKED` propagation chain, and the `--cookie-refresh 2m30s → 10m`
+  mutation being caught by the selftest but not by `render-golden` are all his measurements,
+  relayed to me by the coordinator. I reproduced only the StopIteration defect itself, because that
+  is the part I was asked to fix. If his branch is reworked, nothing in mine depends on it.
+- **The new check does not read the spec repo** and asserts nothing about `../home-network-k3s`
+  (R-TEST-06, read-only). `schemas/renderer-contract.md` is updated in **this** repo only; the
+  canonical spec `docs/13-service-catalog.md` is **not** updated, because it is immutable from here.
+  Someone with spec write access should mirror `auth.apiRoutes` into docs/13 and docs/15.
+- **THE OAUTH2-PROXY SIDECARS' METRICS ARE STILL UNSCRAPABLE, AND THIS PR DOES NOT FIX IT.** See
+  the declined-bundle section above. #28 shipped `--metrics-address` as a prerequisite and said
+  plainly that it was not scraped; Ines has now measured that the listener **serves** and that the
+  scrape is **dropped by each app namespace's default-deny**. Nothing in my change touches that.
+  Anyone reading this entry as "the fleet gates are observable now" is wrong, and the next PR is
+  where that changes.
+- **Alternation is refused, and that is a judgement I may have to revisit.** No app in this catalog
+  needs it, and the flag is repeatable — but if some future app genuinely needs a regex the ratchet
+  rejects, the fix is to widen the pattern **with a reason** on all three surfaces at once, not to
+  delete the ratchet on one.
+- **CI status: two contexts, and only one of them ran.** Quoted in the PR body and in the report to
+  the PM. `approval-policy` **fails by design** until a non-author approving review exists, because
+  the diff touches `conformance/**`, which `infra/gitea/approval-policy.json` lists as sensitive
+  ("the tests and the harness; changing them changes what green MEANS"). That is CLAUDE.md §1
+  working, not a defect. `required_approvals=1` applies to every PR here regardless of path, so this
+  needs a human either way.
+
+### The human test — and how to tell a too-broad regex from a working one
+
+After merge, wait for **new** pod names at 2/2 in `jackett` and `radarr`.
+
+1. **The breadth test FIRST, before anything else, from a signed-out browser** (private window).
+   Open `https://radarr.home.keithjackson.dev/` and `https://jackett.home.keithjackson.dev/UI/Dashboard`.
+   **Each must bounce through Keycloak and land on the app's UI.** If either shows a bare
+   `401 Unauthorized` instead of a sign-in, **the regex is too broad — stop and revert.** That is
+   the one failure this change can introduce, and it is the only test that can see it. A working
+   regex is invisible on a page load; a broken one is total and immediate.
+2. **Jackett, the original report:** signed in, *+ Add indexer* → pick one → *Configure* → *Okay*.
+   **It must save.** Previously 4 of 6 POSTs never reached Jackett at all.
+3. **Radarr, the live symptom:** open Radarr and leave the tab open. DevTools → Network, filter
+   `signalr`. Today those negotiate calls are **302** and the UI never complains. They must now be
+   **200** while signed in — and if the session is ever lost, **401**, with the UI showing a visible
+   connection error instead of polling silently forever.
+4. **The loud-failure test:** in DevTools, delete the `_oauth2_proxy_radarr` cookie and let the tab
+   poll. The negotiate call must answer **401** and Radarr must *show* a problem. Then **reload** —
+   `/` matches neither regex, so the reload must sign you back in silently. That round trip is the
+   entire fix: fail loudly on the API, recover on the page load.
+5. **The six-minute test (PR #28's open item, still open):** stay on a dashboard for >6 minutes,
+   then save something. That exercises `--cookie-refresh=2m30s` past the 300 s access-token cliff.
+   If it fails, suspect the realm's refresh/SSO-idle timeouts — not this change.
+6. **If something still fails, it must now fail LOUDLY.** A silent success-looking no-op means the
+   change did not take: check the running pod's args for `--api-route` before assuming anything
+   else.

decisions/2026-10-01-quicksync-via-device-plugin.md

@@ -0,0 +1,211 @@
+---
+date: 2026-10-01
+branch: feat/quicksync-via-device-plugin
+title: "`hardware.gpu: intel-quicksync` is now satisfied by `gpu.intel.com/i915`, not a `/dev/dri` hostPath — the hostPath was measured not to work (device-cgroup EPERM), and the swap took ZERO ServiceDefinition edits"
+---
+
+## What changed
+
+The mechanism behind `hardware.gpu: intel-quicksync` was replaced. **No ServiceDefinition field
+moved** — that was the whole design bet when this field was made an enum of capability classes
+instead of a device path, and this is the change that collects on it.
+
+**Why the old mechanism had to go.** The `/dev/dri` hostPath shipped on 2026-09-30 was measured
+**non-functional** in the live Plex pod on k3s-prod-2 on 2026-10-01. The device node was *visible*
+in the container — `crw-rw---- root video2 226,128 renderD128` — and could not be opened:
+
+| attempt | result |
+|---|---|
+| `uid 0`, `O_RDWR` | **`EPERM`** "Operation not permitted" |
+| `uid 1000`, `O_RDWR` | **`EPERM`** "Operation not permitted" |
+
+`EPERM` is the **device cgroup**, not `EACCES` from file permissions, proven three ways: root on the
+**host** opens the same device `rc=0`; the node is unified `cgroup2fs`; and the container's runc
+`config.json` read `linux.resources.devices: [{allow: False, access: 'rwm'}]` with
+`linux.devices: None` — one deny-all rule, no allow entry for major 226. A hostPath is a *filesystem*
+operation: it binds the device node in and adds **no cgroup allow rule**, because Kubernetes cannot
+infer one from a volume. `uid 0` failing matters: root does not bypass the cgroup device gate, so
+"run it as root" was never going to work either, and `privileged: true` is unavailable (Kyverno
+`restrict-host-namespaces` is Enforce on app namespaces).
+
+**`catalog/render.py`** — retired `DRI_VOLUME_NAME` / `DRI_DEVICE_DIR` for `QUICKSYNC_RESOURCE`
+(`gpu.intel.com/i915`). The `hardware.gpu == intel-quicksync` branch now emits exactly one thing on
+the **app container only** (never a sidecar):
+
+```yaml
+resources:
+  limits:
+    gpu.intel.com/i915: 1
+```
+
+and no volume, no volumeMount, no `securityContext`, no `nodeSelector`. `limits` rather than
+`requests` is mandatory, not stylistic: Kubernetes mirrors `requests` from `limits` automatically and
+a requests-only extended resource is invalid. This is the **only** app-container `resources` render
+path in the file and it is deliberately narrow — no general CPU/memory resources feature was invented
+along the way.
+
+**Self-tests** — the three existing ones updated plus one new:
+- the emitted `resources` dict is asserted **exactly** (`{"limits": {"gpu.intel.com/i915": 1}}`), so
+  a stray CPU/memory addition through this field fails loudly;
+- the retired hostPath volume and `/dev/dri` mount are asserted **absent**;
+- the no-`hardware` negative extended to cover `resources` (absence still renders nothing at all);
+- the `podSecurityContext` pass-through still holds, and the "no synthesised `securityContext`"
+  assertion is kept verbatim;
+- **new:** the i915 limit must never reach the oauth2-proxy sidecar.
+
+**`schemas/renderer-contract.md`** — the mechanism half of `### hardware` and invariant 8 rewritten.
+The intent half is unchanged and still correct. The `EPERM` measurement is recorded as the *reason*
+the design is what it is, so a future reader cannot "simplify" it back without knowing it was tried
+and measured. The `DirectoryOrCreate` rationale is kept, explicitly labelled historical.
+
+**`catalog/plex.yaml`** — comment correction only; the `hardware:` field is byte-identical.
+`git diff` on that file contains **no non-comment lines**. Its old comment restated the mechanism
+("exactly two things — a `/dev/dri` hostPath volume…"), which is now false; it states intent and
+points at the contract, and keeps the measured facts that remain true (device on all three nodes,
+`i915` bound, the cont-init hook / `supplementalGroups` explanation).
+
+**`runbooks/plex-quicksync.md`** — the diagnostic sequence now checks
+`.status.allocatable["gpu.intel.com/i915"]` and the plugin DaemonSet instead of a hostPath; the
+decisive open test is specified **as uid 1000 via `s6-setuidgid plex`**; the cgroup `EPERM` is
+recast from a *risk* to the *reason for the design*; a new top-of-page "STOP HERE IF PLEX IS
+`Pending`" section and a "Plex will not schedule" procedure carry the new failure mode and the escape
+hatch. The manual-toggle section (`HardwareAcceleratedCodecs`, Settings → Transcoder, still not
+settable by any env var) is unchanged.
+
+**`apps/wave-0/plex.yaml`** — re-rendered: six hostPath lines removed, a three-line `resources` block
+added. Nothing else in `apps/` changed.
+
+### The one real regression, stated rather than sold
+
+This trades failure modes, and the new one is **worse in blast radius**:
+
+| | device/resource unavailable ⇒ |
+|---|---|
+| **Before** | `DirectoryOrCreate` mounted an empty dir; Plex ran and **silently software-transcoded**. Invisible, harmless to everything else. |
+| **After** | Plex is **Unschedulable**. `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so this **stalls the app tier**. |
+
+It is **unavoidable**: an extended resource is a hard scheduling constraint, and it is also the only
+thing that produces the cgroup allow rule. There is no graceful-degradation variant. Plex's
+schedulability is now coupled to the `intel-gpu-plugin` DaemonSet — broken, removed, or merely not
+yet reconciled, it takes the app tier with it. That is the price of hardware transcoding working at
+all.
+
+**Escape hatch — one field, no code change:** delete the `hardware:` block from `catalog/plex.yaml`
+and re-render. Plex returns to software transcoding and the tier unblocks. It is in the contract, in
+the runbook's "Plex will not schedule → remedy B" with exact commands, and here, because it is what
+someone will need at 2am.
+
+## What I verified
+
+**The precondition, re-read from the branch rather than assumed:** the plugin is a standalone
+DaemonSet at `platform/intel-gpu-plugin/` with its own Flux Kustomization
+(`clusters/base/intel-gpu-plugin.yaml`, named `intel-gpu-plugin` in `flux-system`), namespace
+`intel-gpu-plugin`, DaemonSet `intel-gpu-plugin`, pinned `intel/intel-gpu-plugin:0.37.0`. Every
+`kubectl` command I wrote into the runbook names those exact objects. The node/DaemonSet readings
+themselves (`i915=1` on all three nodes; 3 desired / 3 ready / 3 available) were **measured by the
+PM, not by me** — see "What I did NOT verify".
+
+**`render.py --selftest`: PASS**, and every new assertion is **mutation-tested — 8/8 mutants caught**
+(a test that cannot fail is not a test):
+
+| mutant | caught by |
+|---|---|
+| `requests` instead of `limits` | exact-resources assertion |
+| emission dropped entirely | exact-resources assertion |
+| `/dev/dri` hostPath + mount re-added | hostPath-absent assertion |
+| emitted unconditionally (no `hardware` key) | no-`hardware` negative |
+| `supplementalGroups` synthesised | no-pod-`securityContext` assertion |
+| resource renamed `gpu.intel.com/xe` | exact-resources assertion |
+| quantity as the string `"1"` | exact-resources assertion |
+| limit leaked onto the oauth2-proxy sidecar | new app-container-only assertion |
+
+**Re-render is clean and scoped.** `python3 catalog/render.py --all catalog -o apps --wave-layout
+--no-image-automation` then `git status --porcelain apps/` ⇒ `M apps/wave-0/plex.yaml` and nothing
+else. `git diff --stat apps/` ⇒ `1 file changed, 3 insertions(+), 6 deletions(-)` — exactly the six
+hostPath lines out, the three-line `resources` block in. **`apps/wave-1/keycloak.yaml` and
+`apps/wave-0/vm-demo.yaml` are untouched** (checked explicitly: they are git-tracked but excluded
+from the golden comparison by `RENDER_GOLDEN_NOT_ONBOARDED` in `ci/render-golden.sh`, so a spurious
+diff there would land ungated). `clusters/` is untouched; `--emit-waves` was not run.
+
+**Invariant 8 holds, mechanically checked rather than eyeballed.** `git diff -U0 catalog/plex.yaml`
+filtered to non-comment changed lines returns **empty**, and `yaml.safe_load` gives
+`hardware == {'gpu': 'intel-quicksync'}`. Zero ServiceDefinition field edits.
+
+**The gate: `rc 3` UNDETERMINED, which is this environment's calibrated best — NOT green, and I am
+not calling it green.** Run from this worktree in the pinned toolbox
+(`DEV_KUBECONFIG=/dev/null`, plus an `ENGINE` shim bind-mounting the main checkout's `.git`
+read-only, per CLAUDE.md §3; `infra/ansible/.cache/prod.kubeconfig` was **not** copied):
+
+| stage | result |
+|---|---|
+| `spec-immutability` | **SKIPPED** — spec repo not present at `/work/../home-network-k3s` (R-TEST-06 unverified) |
+| `decisions-log` | green |
+| `lint` | green |
+| `l0` | green |
+| `render-golden` | green |
+| `l1` | green |
+| `approval-policy` | green (`event='none'`, N/A off a PR event) |
+| `gitea-settings` | **SKIPPED** — `SETTINGS_API_TOKEN` unset |
+| `conformance` | **SKIPPED** — `CLUSTER_AVAILABLE != true`, L2/L3 not run |
+
+Three skips, the three expected ones; **nothing failed**. `diff base: origin/main @ 8214642d87de …
+HEAD @ 78bf68493d7d`, touched requirements resolved (`R-PLAT-02` among them), so the R-TEST-03
+coverage gate actually applied — an earlier run on the uncommitted tree reported the touched set
+"NOT DETERMINABLE (on-base)" and that is *not* the run I am quoting.
+
+## What I did NOT verify
+
+**HARDWARE TRANSCODING IS STILL NOT OBSERVED WORKING, and nothing in this branch could establish
+it.** A green gate proves the **manifest renders**, nothing more.
+
+- **The `open()` test has not been re-run under the new mechanism, and cannot be until this merges
+  and Flux reconciles.** The whole case for this change is that the device plugin's
+  `DeviceSpec{… Permissions: "rw"}` produces the cgroup allow rule the hostPath could not. That is
+  read from the v0.37.0 **source**, not observed on this cluster. If the plugin does not in fact
+  deliver it here, this change buys nothing and costs schedulability.
+- **The Plex-side toggle is still unset.** `HardwareAcceleratedCodecs` was **absent** from
+  `Preferences.xml` as of 2026-09-30 and nothing in this branch touches it; no env var, ConfigMap or
+  `postBuild` substitution can. It is a manual UI click, and until it happens Plex software-transcodes
+  no matter how the device arrives.
+- **I did not measure the cluster at all.** `i915=1` on all three nodes and the DaemonSet's
+  3/3/3 come from the PM's brief, dated 2026-10-01. Per CLAUDE.md §2b this repo is not evidence about
+  the live platform, and neither is a handoff entry: **re-measure before relying on them** (the
+  commands are first-command #1 in the runbook). The DaemonSet was 2m15s old when read — young enough
+  that it had not been observed stable for any length of time.
+- **The new failure mode has not been exercised.** "Plugin unavailable ⇒ Plex Unschedulable ⇒
+  `apps-wave-0` stalls ⇒ `apps-wave-1` blocked" is derived from `wait: true` and `dependsOn` in the
+  Flux wiring, not from a drill. The escape hatch (delete `hardware:`, re-render) is likewise
+  reasoned, not rehearsed — though it is a strict subset of the renderer path the selftest covers.
+- **No L2/L3 conformance.** `CLUSTER_AVAILABLE` is unset, so the gate's runtime layer did not run
+  here and does not run in CI either. `spec-immutability` and `gitea-settings` also skipped.
+- **CI's own tick, when it appears, will not distinguish rc 0 from rc 3**:
+  `.gitea/workflows/gate.yml:138` is `./ci/gate.sh || [ "$?" -eq 3 ]`. Read the stage output, not the
+  colour — and take the newest row per context from `GET /statuses/{sha}`, which appends.
+- **`imagePullPolicy: Always` on `plexinc/pms-docker:latest` is still a live hazard** to the
+  cont-init-hook reasoning this feature leans on. Unchanged by this branch, recorded in the runbook's
+  floating-image caveat.
+
+### Post-merge sequence for the operator (all mutating steps are yours, not an agent's)
+
+1. Merge the PR once the gate reports. Confirm Flux applied it — compare
+   `kubectl --context k3s-prod -n flux-system get kustomization apps-wave-0
+   -o jsonpath='{.status.lastAppliedRevision}'` against the merge commit.
+2. Confirm Plex **scheduled** (this is the new risk):
+   `kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex -o wide`.
+   `Pending` ⇒ runbook "Plex will not schedule".
+3. Confirm it consumed the resource:
+   `kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex
+   -o jsonpath='{.items[0].spec.containers[0].resources.limits}'`.
+4. **The decisive test — as uid 1000, via `s6-setuidgid plex`. `rc=0` is the pass condition:**
+   `kubectl --context k3s-prod -n plex exec deploy/plex -- s6-setuidgid plex sh -c
+   'exec 3<>/dev/dri/renderD128 && echo RW-OPEN-OK || echo RW-OPEN-DENIED'`.
+   **Passing as `uid 0` is weaker evidence and must not be accepted as proof** — Plex runs as 1000,
+   and uid 0 was one of the two `EPERM`s that killed the hostPath.
+5. **Only then** the Plex UI toggle: Settings → Transcoder → "Use hardware acceleration when
+   available" (under *Show Advanced*) → Save. Then force a transcode and read Dashboard → Activity:
+   `Transcode (hw)` is the proof; plain `Transcode` is software.
+
+Step 4 failing means the plugin is not delivering the cgroup allow rule on this cluster. Do **not**
+reach for `privileged: true` (Kyverno Enforce rejects it, and uid 0 failed anyway) and do **not**
+re-add a `/dev/dri` hostPath (that is the mechanism this replaced, measured non-functional). The
+escape hatch is to delete the `hardware:` block from `catalog/plex.yaml` and re-render.

decisions/2026-10-01-render-selftest-runs-in-gate.md

@@ -0,0 +1,205 @@
+---
+date: 2026-10-01
+branch: fix/render-selftest-runs-in-gate
+title: catalog/render.py's 106 assertions were dev-time only; the gate now runs them and blocks on them
+---
+
+## What changed
+
+Single subject, one file: **`ci/render-golden.sh` now invokes `catalog/render.py --selftest`**
+and exits 1 when it fails. Nothing else. No renderer change, no new conformance test, no new
+gate stage.
+
+This closes the top follow-up of `decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md`:
+
+> Nothing in `ci/` or `.gitea/` runs `catalog/render.py --selftest` — I grepped for it. So the
+> twenty-odd assertions in that selftest, including the six I just added, are dev-time only: a
+> future edit that drops either flag *and* re-renders the goldens would pass the entire merge
+> gate.
+
+Re-measured here and confirmed: before this commit, `grep -rl -- --selftest` over `ci/` and
+`.gitea/` returned **nothing** for the renderer. The selftest executes **106** assertions
+(`check()` invocations counted at runtime via `sys.settrace`, without editing `render.py`; 109
+static call sites, three of them on failure-only branches). All 106 were unexercised by the
+merge gate.
+
+### The lead was WRONG, and here is why
+
+The brief named `conformance/tests/l0-static/auditability-proofs-producible/check.sh:291` as the
+one-line fix. That line is real and it does drive selftests:
+
+    for tool in ("record_run.py", "status.py"):
+        r = run([sys.executable, str(LIB / tool), "--selftest"])
+
+But it is **the wrong hook**. That loop is section (d) of a check that serves **R-SEC-03**, and
+its subject is the *evidence chain* — can a self-skipped run, or a stale ledger row, become an
+audit proof? `record_run.py` and `status.py` are **in** that chain; `catalog/render.py` is not.
+Hanging the renderer there would have made the check's name, its `intent` annotation and its
+requirement mapping (`R-SEC-03`) all untrue, and a future decision about the renderer could then
+only be expressed by weakening a *security-auditability* check. That is the precise coupling the
+cookie-collision entry refused for its own check, and it applies here.
+
+The second option that entry offered — a new scope-declaring L0 test — is the supported mechanism
+for a **whole-repo invariant** (`ci/l0.sh` Stage 3b's `--include-scope whole-repo` union, ratcheted
+by `tests/l0-static/whole-repo-invariants-wired`). It is the right shape for an invariant
+quantified over the tracked tree. It is the wrong shape for *"a script in this repo must pass its
+own unit test"*: it would need to declare requirements it does not serve, and it would be two new
+files of several hundred lines by this tree's conventions for what is a six-line invocation.
+
+**The honest home is `ci/render-golden.sh`** — the gate stage whose subject *already is*
+`catalog/render.py`. It already resolves `$PYTHON` and `$RENDER_PY`, already reports rc 3 when
+either is missing, already exits 1 on renderer error, and `ci/gate.sh` already maps it through
+`for s in lint l0 render-golden l1` where a FAIL is BLOCKED and a SKIP is UNDETERMINED. The
+stage's two assertions are independent and both are needed:
+
+- `render-golden` catches golden-vs-renderer drift, but is satisfied by **any** renderer whose
+  output was re-rendered — including one that just lost a contract invariant and had all ten
+  goldens regenerated in the same commit. That is exactly the hole named in the quote above.
+- `--selftest` catches that, and is blind to golden drift.
+
+The file header and the stage name were updated so neither is a lie: the stage is now "the
+renderer is trustworthy", with two numbered assertions, and the step summary emits its own
+`- render-selftest:` line so the gate output still NAMES what ran.
+
+The selftest runs **before** the empty-catalog `exit 3` on purpose: it renders synthetic
+ServiceDefinitions in memory and is catalog-independent, so an empty catalog must not be able to
+skip it.
+
+`catalog/render.py` itself was **not touched** (owned by a concurrent branch this session), and
+neither was any file under `conformance/`.
+
+## What I verified
+
+- **The selftest runs and passes: rc 0, `render.py selftest: PASS`, 106 assertions executed.**
+  Counted at runtime, not from the source: a `sys.settrace` hook counting `check()` calls, in a
+  throwaway script outside the repo, with no edit to `render.py`.
+- **The stage passes with it wired in**, host run of the single stage:
+
+      [gate] Stage: render==golden reproducibility (canonical render vs committed apps/)
+      [gate] Stage: catalog/render.py --selftest (the renderer's own output-contract assertions)
+      [ ok ] render.py --selftest passed
+      [ ok ] render==golden: all 10 onboarded service golden(s) reproduce from the canonical render
+
+  rc **0**. The golden comparison still reports all 10 goldens, so the new invocation did not
+  displace what the stage already asserted.
+- **IT CAN FAIL — provoked twice, against a mutated COPY of the renderer, never against the
+  tree.** `ci/render-golden.sh` already honours `RENDER_PY`, so each mutant lived in
+  `/tmp/claude-1000/desmond-selftest/` and the repo's `catalog/render.py` was never modified —
+  which matters, because that file belongs to a concurrent branch this session.
+  - **Mutation 1 — the regression this selftest was written for.** Deleted the
+    `f"--cookie-name=_oauth2_proxy_{s['name']}"` emission from `_oauth2_sidecar_container()`,
+    i.e. reverted the fleet-wide cookie-collision fix of PR #28. Stage rc **1**. Note the shape
+    of the failure, because it is a real (minor) weakness and is listed as a follow-up: the
+    selftest died with an uncaught `StopIteration` out of `next(a for a in container["args"] if
+    a.startswith("--cookie-name="))` at `render.py:2641`, rather than printing its collected
+    `SELFTEST FAIL:` lines. It is correctly non-zero and the stage correctly blocks, but the
+    operator gets a traceback instead of the assertion message.
+  - **Mutation 2 — a clean, named assertion failure.** `OAUTH2_COOKIE_REFRESH = "2m30s"` ->
+    `"10m"`, widening the refresh past the measured 300s access-token lifetime. Stage rc **1**:
+
+        [fail] catalog/render.py --selftest FAILED — the renderer violates its own output contract.
+        [fail]   Every golden under apps/ is produced by this renderer, so the goldens below cannot
+        [fail]   be trusted even if they reproduce byte-for-byte. Reproduce with:
+        [fail]     python3 catalog/render.py --selftest
+            SELFTEST FAIL: --cookie-refresh=10m parses to 600.0s, which is not inside (0, 300s). A
+            refresh at or past the access-token lifetime never renews in time, which is the same
+            logout it was added to stop
+
+    This is the mutation that proves the wiring's value most directly: mutation 2 would **not**
+    be caught by `render-golden`'s diff if the goldens were regenerated in the same commit, and
+    it is one of the two flags the cookie-collision entry explicitly recorded as **ungated**.
+- **THE NON-ZERO PROPAGATES ALL THE WAY TO `gate.sh`'s rc — provoked, in the pinned toolbox, not
+  reasoned about.** `RENDER_PY=/desmond-selftest/render-mutant2.py ./ci/gate.sh`:
+
+      [ ok ] stage spec-immutability: pass
+      [ ok ] stage decisions-log: pass
+      [ ok ] stage lint: pass
+      [ ok ] stage l0: pass
+      [gate] Stage: catalog/render.py --selftest (the renderer's own output-contract assertions)
+      [fail] catalog/render.py --selftest FAILED — the renderer violates its own output contract.
+          SELFTEST FAIL: --cookie-refresh=10m parses to 600.0s, which is not inside (0, 300s). ...
+      [fail] stage render-golden: FAIL
+      [ ok ] stage l1: pass
+      ...
+      [fail] render-golden: FAILED -> blocks merge
+      [fail] MERGE GATE: BLOCKED
+
+  `gate.sh` rc **1**. Three links in the chain, each measured: selftest rc 1 -> stage rc 1 ->
+  gate rc 1 / `MERGE GATE: BLOCKED`. This mattered enough to provoke rather than assume: this
+  repo has a documented habit of exit codes getting flattened, and
+  `.gitea/workflows/gate.yml:138` runs `./ci/gate.sh || [ "$?" -eq 3 ]`, which tolerates rc 3 —
+  so a stage that degraded a failure to a skip would be swallowed by CI. rc 1 is not tolerated
+  there, and rc 1 is what this produces.
+- **`spec-immutability` genuinely RAN and passed, it did not skip.** The two-mount worktree shim
+  plus `-e SPEC_REPO=/home-network-k3s` (the fix recorded in the 2026-09-30 cookie entry) works:
+  `stage spec-immutability: pass`, not SKIPPED. R-TEST-06 is therefore actually covered by these
+  runs.
+- **No collision with PR #29.** The brief warned that `feat/oauth2-proxy-auth-failure-visible`
+  had added checks under `conformance/tests/l0-static/`. Measured —
+  `git diff origin/main...origin/feat/oauth2-proxy-auth-failure-visible --name-only` — it touches
+  **no `conformance/` path at all**: `decisions/`, `platform/oauth2-proxy/` (a promtool test, an
+  alert rule, its kustomization), `runbooks/`. Disjoint from this branch either way, since this
+  branch touches only `ci/render-golden.sh` and this file.
+- **Priya's files were not touched.** `catalog/render.py`, `schemas/servicedefinition.schema.json`,
+  `apps/wave-*` and `conformance/tests/l1-rendering/` are all untouched by this branch — confirmed
+  by `git diff --name-only`, which lists exactly two paths.
+- **`catalog/render.py --selftest` was the LAST unwired selftest in the tree.** Measured:
+  `grep -rl -- --selftest` across the repo finds selftest-capable artifacts in exactly four
+  places, and after this commit every one is driven by the gate — `conformance/lib/record_run.py`
+  and `conformance/lib/status.py` by `auditability-proofs-producible/check.sh:291`, the Keycloak
+  client-secret reconciler (embedded in `platform/keycloak/client-secret-reconciler.yaml`) by
+  `keycloak-client-secret-reconciler/check.sh:633`, and `catalog/render.py` by this change.
+
+## What I did NOT verify
+
+- **Local gate: rc 3 (UNDETERMINED), NOT 0. This is not green and must not be reported as
+  green.** Measured in the pinned toolbox on `eeb6211`:
+
+      [ ok ] spec-immutability: green
+      [ ok ] approval-policy: green
+      [ ok ] decisions-log: green
+      [ ok ] lint: green
+      [ ok ] l0: green
+      [ ok ] render-golden: green
+      [ ok ] l1: green
+      [fail] MERGE GATE: UNDETERMINED — greenness was NOT ESTABLISHED.
+      [fail]   Stages that did not run: gitea-settings
+
+  Nothing FAILED. The single cause is `gitea-settings` **SKIPPED** — `SETTINGS_API_TOKEN` is not
+  set in my environment, so branch protection and runner registration were not read; CI has that
+  token and the stage is expected to run there. **L2/L3 `conformance`** also skipped
+  (`CLUSTER_AVAILABLE` unset) but is a *declared* gap that `gate.sh` treats as not-applicable
+  rather than undetermined, so it is not what produced the 3. Touched requirements
+  **R-GITOPS-01 R-TEST-02 R-TEST-03 R-TEST-06** are therefore **UNVERIFIED at L2/L3**, all four.
+  The stage this branch changes, `render-golden`, is green **and** was independently proven able
+  to turn the whole gate red (above).
+- **`approval-policy` passed LOCALLY only because there is no pull_request event**
+  (`not a pull_request event (event='none') — approval policy N/A`). That is not evidence that it
+  passes. This diff touches `ci/**`, a sensitive path, so in CI the stage is expected to **FAIL**
+  until a non-author approving review exists. That is §1 working, fail-closed, not a defect.
+- **The selftest's own quality is unchanged and ungated.** I wired its *invocation*; I did not
+  review its 106 assertions, and nothing ratchets them. Deleting the body of `_selftest()` would
+  still pass this stage, because an empty selftest returns 0. The repo already has the right
+  answer for that — `conformance/tests/l0-static/keycloak-client-secret-reconciler/mutations.py`
+  injects regressions one at a time and requires `--selftest` to catch every one — and applying
+  that pattern to `catalog/render.py` is the follow-up, deliberately not done here: it is a
+  second subject, it touches `conformance/`, and it would need to live beside the renderer whose
+  file another branch owns this session. **Until it exists, "the selftest runs" is proven and
+  "the selftest is falsifiable" rests on my two hand-run mutations, which are not in the gate.**
+- **Mutation 1's traceback is not fixed.** `render.py:2641`'s `next(...)` has no default, so the
+  absence of `--cookie-name` raises `StopIteration` and the selftest's collected `SELFTEST FAIL:`
+  lines are never printed. Correct verdict, poor diagnostics, and it suppresses the other
+  assertions that mutation also broke. The one-line fix (`next(..., None)` plus a `check()`) is in
+  `catalog/render.py`, which is another branch's file this session, so it is a follow-up and not
+  a thing I touched.
+- **Nothing about the live cluster.** No `kubectl` of any kind was run on this branch — not even
+  read-only; this change is offline CI wiring and needed none. No `apply`/`edit`/`patch`, no
+  `flux reconcile`. Merging this deploys nothing: it changes only what the gate refuses.
+- **CI's own run of this branch.** Quoted in the PR body from the commit's `/statuses`, with
+  descriptions read rather than colours. A *skipped* Gitea job reports `success` with
+  "Has been skipped" and certifies nothing.
+- **`vars.CLUSTER_AVAILABLE` is still unset in CI**, so even a fully green PR proves the L0/L1
+  offline subset and nothing about runtime behaviour.
+- **I did not re-measure branch protection, required approvals, or the runner.** Those are
+  CLAUDE.md §2 claims and §2b says re-measure before relying on them; this branch relies on none
+  of them beyond the PR flow itself, and I read the gate's own output rather than the API.

infra/ansible/roles/node_health/README.md

@@ -19,9 +19,9 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
 
 ## Contract
 
-- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu`,
-  and — only when it can escalate — `smartctl`. It writes nothing to the node and
-  **installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is
+- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu`
+  (and `lspci` only if it happens to be installed), and — only when it can escalate —
+  `smartctl`. It writes nothing to the node and **installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is
   absent the SMART checks SKIP with that as the stated reason (operator ruling 2026-08-31:
   package installation is `node_baseline`'s job, never a health check's, never a manual
   `apt-get`).
@@ -50,6 +50,9 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
 | `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite **and whether it survives a reboot** (see [Swap off is not the same as swap disabled](#swap-off-is-not-the-same-as-swap-disabled)), EDAC correctable/uncorrectable counters |
 | `memory.physical_total` | **is this machine using the RAM it has?** `/proc/zoneinfo` present-vs-managed pages (no root, metal or VM) and `MemTotal` vs the DIMM capacity SMBIOS reports (`dmidecode -t 17`, root, bare metal). Cause-independent by design — see [The EFI memattr check](#the-efi-memattr-check) |
 | `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) |
+| `graphics.render_device` | is there a usable `/dev/dri/renderD*` character device — the path Plex hardware-transcodes through. See [The render-device check](#the-render-device-check) |
+| `graphics.drm_driver` | which DRM driver is bound to the display device, against `node_health_graphics_expected_driver` (`i915` here) |
+| `graphics.media_stack` | **INFO only**: `vcs*`/`vecs*` media engines, GuC/HuC firmware state, host VAAPI userspace. None of these is ever graded — see below for why |
 | `thermal.max` | hottest sensor across thermal zones and hwmon chips |
 | `pressure.cpu` / `.memory` / `.io` | PSI `some avg60` — dev learned to care about CPU PSI specifically |
 | `network.link`, `network.errors`, `network.resolvers` | link state/speed, NIC error counters, DNS upstream redundancy per address family |
@@ -91,6 +94,78 @@ used to blame that state on "this node has not been baselined yet", which sent t
 back to re-run the role that had already run.
 
 
+## The render-device check
+
+Plex transcodes on the iGPU through `/dev/dri/renderD128`. **When that node stops existing,
+nothing fails loudly.** Plex falls back to software transcoding by itself; the only symptom
+is a hot CPU and stuttering clients. A node rebuild, a BIOS change that disables the onboard
+IGD, a kernel that binds a different DRM driver, or a driver regression all produce exactly
+that silent degradation.
+
+The state was measured by hand on all three prod nodes on 2026-09-30 and found working. **A
+fact measured once decays silently**, so it is re-measured on every health run instead of
+living in a runbook. Nothing is installed, no module is loaded, no group is changed — the
+role's read-only contract is unchanged.
+
+| fact | where it comes from | root? |
+|---|---|---|
+| `/dev/dri` contents, ownership, mode, char-vs-not | `lstat()` on each entry | no |
+| owning group **name** for the render node's gid | `grp.getgrgid()` | no |
+| bound DRM driver | `/sys/class/drm/card*/device/driver` symlink target | no |
+| is the iGPU on the bus at all | `/sys/bus/pci/devices/*/class` (PCI base class `0x03`) | no |
+| media engines (`vcs*`, `vecs*`) | `/sys/class/drm/card*/engine/*` | no |
+| module load state | `/proc/modules` | no |
+| GuC/HuC | `/sys/module/i915/parameters/enable_guc` + `/proc/cmdline` | **parameter file is mode 0400** |
+
+### The PCI scan is separate from the DRM scan on purpose
+
+`/sys/class/drm/card*` exists only once a driver has **bound**. A machine whose iGPU sits on
+the bus with nothing attached has no `card0` at all — and that is precisely the failure being
+hunted. So the display device is looked for on the PCI bus independently, from sysfs, which
+is always there. That is what lets `graphics.render_device` distinguish
+
+- *no GPU in this machine* → **SKIP** (a VM guest; nothing to grade, the same convention
+  `thermal.max` uses for "no sensor on this hardware"), from
+- *a GPU is on the bus and there is still no render node* → **FAIL**.
+
+`lspci` is used **only** to decorate the PCI entry with a readable name, and it is treated as
+optional because it is: `pciutils` is **not** declared in `node_baseline_packages`, so nothing
+in this repo guarantees it. It did turn out to be present on all three prod nodes
+(`/usr/bin/lspci`, measured 2026-09-30 — Debian installs `pciutils` by default), but depending
+on that would mean depending on an undeclared package. Absent, `description` is `null`, the
+numeric IDs are printed instead, and no verdict changes.
+
+### The render node's GID is reported, never asserted
+
+The render device was `root:render`, **gid 992**, mode `0660` on all three prod nodes on
+2026-09-30. The check does **not** compare against `992`, and must not be changed to.
+`render` is a **systemd dynamic allocation** (`systemd-sysusers`), not a Debian static gid
+from `base-passwd`, so a rebuilt node can legitimately land on a different number. Plex's own
+container entrypoint stats the device and joins whichever group owns it, so drift genuinely
+is not a fault. Recording the observed group name and gid in the detail line makes drift
+**visible** without making it an error — a hard `!= 992` would be a false alarm waiting for a
+reinstall.
+
+What *is* configurable is the expected DRM driver, `node_health_graphics_expected_driver`
+(default `i915`), so the verdict stays out of task logic. `xe` bound instead of `i915` is a
+**WARN**, not a FAIL: the device still works, but the two drivers do not share a userspace
+media stack and a libva/iHD build that transcodes correctly against one is not automatically
+right for the other. On these nodes `xe.ko.xz` exists on disk and is bound to nothing.
+
+### Two things that are informational and must stay that way
+
+- **GuC/HuC firmware not loaded** (`i915.enable_guc=0`) is a known, accepted state here. It
+  costs low-power H.264 encode *quality* only; decode and encode both still run on the media
+  engines. Grading it would put a permanent WARN on a chosen configuration.
+- **No host VAAPI userspace / no `vainfo`** is **not a defect**. Plex ships its own libva and
+  media driver inside its container, so the host needs neither. Flagging it would invite
+  someone to `apt install vainfo` on a node to clear a health check — and this role installs
+  nothing.
+
+`enable_guc` is one of i915's `0400` module parameters, so an unprivileged pass usually
+cannot read it. It reports as an honest `NOT READ — <reason>` rather than a fabricated `0`,
+with `/proc/cmdline` as the world-readable second source for an explicit boot override.
+
 ## The EFI memattr check
 
 `infra/PROD-PROVISIONING.md` §0b is authoritative; this is only what the check needs.
@@ -273,3 +348,10 @@ stays `null` — no claim is made either way.
   but the wear/endurance context SKIPs rather than guessing.
 - `ioerr_cnt` counts commands the kernel saw fail, which includes benign rejected or
   unsupported commands during discovery. Attribution needs `dmesg`, which needs root.
+- `graphics.render_device`'s **FAIL path has never fired against a real node** — every prod
+  node has a working render device, and the role is read-only so one cannot be taken away to
+  prove it. The failure branches were exercised against synthetic fact sets only. The same
+  applies to the `xe`-bound WARN: no node here has `xe` bound.
+- `graphics.*` proves the render **device** exists and which driver owns it. It does **not**
+  prove a transcode succeeds — that needs a VAAPI round trip inside Plex's own container,
+  with Plex's own libva, which is out of scope for a node-level check that installs nothing.

infra/ansible/roles/node_health/defaults/main.yml

@@ -35,6 +35,24 @@ node_health_try_privileged: true
 # --nocheck=standby: never spin up a sleeping disk just to read it. Read-only hygiene.
 node_health_smartctl_args: "--json=c --all --nocheck=standby"
 
+# --- Graphics / render device ---------------------------------------------------------
+# The DRM driver this site EXPECTS to be bound to its display device. Measured on all three
+# prod nodes 2026-09-30: `i915` on the CoffeeLake-S GT2 [UHD Graphics 630] iGPU of every HP
+# EliteDesk 800 G5 Mini. The newer `xe` module is present on disk as xe.ko.xz but is bound to
+# nothing, and that difference matters enough to surface: the two drivers do not share a
+# userspace media stack (a libva/iHD build that works against i915 is not automatically the
+# right one for xe), so a silent switch would change Plex's transcoding path. A site on AMD
+# or NVIDIA sets this to `amdgpu` / `nouveau` / `nvidia` rather than editing task logic.
+#
+# WHAT IS DELIBERATELY *NOT* A SETTING HERE: the numeric GID of the render node's owning
+# group. `render` is a systemd DYNAMIC allocation (systemd-sysusers), not a Debian static
+# one, so a rebuilt node may legitimately land on a different number — it was 992 on all
+# three prod nodes on 2026-09-30, and an assertion against that literal would be a false
+# alarm waiting to happen. Plex's own container entrypoint stats the device and joins
+# whichever group owns it, so drift genuinely is not a fault. The observed group name and GID
+# are therefore REPORTED in the check detail and never graded.
+node_health_graphics_expected_driver: i915
+
 # --- Thresholds ---------------------------------------------------------------------
 # All verdicts come from here, so a site can retune without touching task logic.
 node_health_thresholds:

infra/ansible/roles/node_health/files/node_health_collect.py

@@ -35,10 +35,13 @@
 # =============================================================================
 
 import glob
+import grp
 import json
 import os
+import pwd
 import re
 import shutil
+import stat
 import subprocess
 import sys
 import time
@@ -756,6 +759,212 @@ def collect_pressure():
     return out
 
 
+# --- Graphics / DRM render device -----------------------------------------------------
+# WHY THIS IS HERE AT ALL. Plex transcodes on the iGPU through /dev/dri/renderD128. If that
+# node stops existing — a node rebuild, a BIOS change that disables the IGD, a kernel that
+# binds a different DRM driver — nothing breaks loudly: Plex silently falls back to software
+# transcoding and the only symptom is a hot CPU. So the render device is measured on every
+# health run rather than being a fact someone established once by hand.
+#
+# EVERYTHING BELOW IS UNPRIVILEGED. /dev/dri is world-traversable, the device nodes are
+# stat()able by anyone, /sys/class/drm and /sys/bus/pci/devices are world-readable, and so
+# are the module parameters under /sys/module. Nothing here needs root, so nothing here
+# belongs in the privileged pass. (GuC/HuC *load* status lives in debugfs, which does need
+# root AND debugfs mounted; the module PARAMETER below is the unprivileged stand-in and is
+# what actually decides whether the firmware is loaded at all.)
+#
+# THE PCI SCAN IS DELIBERATELY SEPARATE FROM THE DRM SCAN. /sys/class/drm/card* only exists
+# once a driver has BOUND. A machine whose iGPU is on the bus with no driver bound has no
+# card0 at all, and that is precisely the failure this is meant to catch — so the display
+# device is looked for on the PCI bus independently, from sysfs, which is always there.
+# `lspci` is used ONLY to decorate that with a human-readable name, and it is treated as
+# optional because it IS optional: pciutils is not declared in node_baseline_packages, so
+# nothing in this repo guarantees it. It did turn out to be present on all three prod nodes
+# (/usr/bin/lspci, measured 2026-09-30 — Debian installs pciutils by default), but relying on
+# that would make the collector depend on an undeclared package. A missing lspci leaves
+# `description` null, the numeric IDs are reported instead, and no verdict changes.
+DISPLAY_PCI_CLASS_PREFIX = "0x03"  # PCI base class 03 = display controller (VGA/3D/other)
+# DRM drivers worth reporting the load state of. The bound-driver symlink is authoritative
+# (a built-in driver never appears in /proc/modules); this is corroboration only, and it is
+# how "xe.ko.xz exists on disk but is bound to nothing" becomes a visible fact.
+DRM_MODULES = ("i915", "xe", "amdgpu", "radeon", "nouveau", "nvidia")
+
+
+def _uid_name(uid):
+    try:
+        return pwd.getpwuid(uid).pw_name
+    except (KeyError, OverflowError, TypeError, OSError):
+        return None
+
+
+def _gid_name(gid):
+    """Group NAME for a numeric gid, or None when nothing resolves it.
+
+    Reported as an OBSERVED value, never asserted against a literal. The `render` group is
+    a systemd DYNAMIC allocation (it is created by systemd-sysusers, not fixed by Debian's
+    static base-passwd list), so its gid can legitimately differ after a rebuild. Plex's own
+    container init stats the device and joins whatever group owns it, so a moved gid is
+    drift to notice, not a fault to fail on."""
+    try:
+        return grp.getgrgid(gid).gr_name
+    except (KeyError, OverflowError, TypeError, OSError):
+        return None
+
+
+def _module_param(mod, param):
+    """One /sys/module/<mod>/parameters/<param>, with WHY it is missing when it is.
+
+    Three outcomes that must not be conflated: a real value; "the module exposes no such
+    parameter" (not loaded, built in, or a driver that simply has no such knob — xe has no
+    enable_guc, GuC is mandatory there); and "the file exists but is mode 0400". Most i915
+    module parameters are `i915_param_named(..., 0400, ...)` — root-only — so an
+    unprivileged pass legitimately cannot read them. Returning null with the reason keeps
+    that an honest UNKNOWN instead of a fabricated 0. read_text() also records the
+    permission denial in UNREADABLE, so the summary lists it with everything else root
+    would add."""
+    path = "/sys/module/%s/parameters/%s" % (mod, param)
+    if not os.path.exists(path):
+        return {"value": None, "reason": "%s exposes no %s parameter (module not loaded, "
+                                         "built in, or no such knob on this driver)" % (mod, param)}
+    value = read_text(path)
+    if value is None:
+        return {"value": None, "reason": "%s is not readable without root (i915 module "
+                                         "parameters are mode 0400)" % path}
+    return {"value": value, "reason": None}
+
+
+def _dev_node(path):
+    """stat() one /dev/dri entry. lstat, so a symlink is reported as the symlink it is."""
+    try:
+        st = os.lstat(path)
+    except OSError as exc:
+        UNREADABLE.append({"path": path, "reason": "stat failed: %s" % exc})
+        return None
+    return {
+        "path": path,
+        "name": os.path.basename(path),
+        "char_device": stat.S_ISCHR(st.st_mode),
+        "mode": "%04o" % stat.S_IMODE(st.st_mode),
+        "uid": st.st_uid,
+        "gid": st.st_gid,
+        "owner": _uid_name(st.st_uid),
+        "group": _gid_name(st.st_gid),
+    }
+
+
+def collect_graphics():
+    dri_dir = "/dev/dri"
+    dri_present = os.path.isdir(dri_dir)
+    devices = []
+    if dri_present:
+        for path in sorted(glob.glob(dri_dir + "/*")):
+            # /dev/dri/by-path/ is a directory of symlinks, not a device. Skipping
+            # directories rather than filtering to char devices is deliberate: anything
+            # in here that is NOT a character special file is an anomaly worth seeing, and
+            # `char_device: false` is how it becomes visible.
+            if os.path.isdir(path):
+                continue
+            entry = _dev_node(path)
+            if entry is not None:
+                devices.append(entry)
+    render_nodes = [d for d in devices if d["name"].startswith("renderD")]
+
+    # Bound DRM devices. /sys/class/drm also holds one entry per CONNECTOR
+    # (card0-HDMI-A-1, card0-DP-1 ...) — those are outputs, not devices, and are skipped.
+    cards = []
+    for cdir in sorted(glob.glob("/sys/class/drm/card[0-9]*")):
+        name = os.path.basename(cdir)
+        if "-" in name:
+            continue
+        driver = None
+        drvlink = cdir + "/device/driver"
+        if os.path.islink(drvlink):
+            driver = os.path.basename(os.path.realpath(drvlink))
+        address = None
+        devlink = cdir + "/device"
+        if os.path.islink(devlink):
+            address = os.path.basename(os.path.realpath(devlink))
+        # i915/xe publish one directory per hardware engine. vcs* is a video (H.264/HEVC)
+        # engine and vecs* the video-enhancement engine: those two are what a transcode
+        # actually runs on, so their presence is the difference between "a GPU is bound"
+        # and "this GPU can do media work".
+        engines = sorted(
+            {
+                os.path.basename(p)
+                for p in (glob.glob(cdir + "/engine/*") + glob.glob(cdir + "/device/engine/*"))
+                if os.path.isdir(p)
+            }
+        )
+        cards.append(
+            {
+                "card": name,
+                "driver": driver,
+                "pci_address": address,
+                "vendor_id": read_text(cdir + "/device/vendor"),
+                "device_id": read_text(cdir + "/device/device"),
+                "engines": engines,
+                "media_engines": [e for e in engines if e.startswith(("vcs", "vecs"))],
+            }
+        )
+
+    # Display-class devices on the PCI bus, driver bound or not.
+    pci_display = []
+    for pdir in sorted(glob.glob("/sys/bus/pci/devices/*")):
+        class_id = read_text(pdir + "/class") or ""
+        if not class_id.startswith(DISPLAY_PCI_CLASS_PREFIX):
+            continue
+        address = os.path.basename(pdir)
+        driver = None
+        if os.path.islink(pdir + "/driver"):
+            driver = os.path.basename(os.path.realpath(pdir + "/driver"))
+        description = None
+        rc, out, _ = run(["lspci", "-nn", "-s", address])
+        if rc == 0 and out.strip():
+            description = out.strip().splitlines()[0]
+        pci_display.append(
+            {
+                "address": address,
+                "class_id": class_id or None,
+                "vendor_id": read_text(pdir + "/vendor"),
+                "device_id": read_text(pdir + "/device"),
+                "revision": read_text(pdir + "/revision"),
+                "driver": driver,
+                "description": description,
+            }
+        )
+
+    loaded = set()
+    for line in (read_text("/proc/modules") or "").splitlines():
+        fields = line.split()
+        if fields:
+            loaded.add(fields[0])
+    modules = {m: (m in loaded) for m in DRM_MODULES}
+
+    # GuC/HuC. i915's `enable_guc` is a bitmask: 0 = neither, 1 = GuC submission, 2 = HuC
+    # firmware, 3 = both, -1 = whatever the driver defaults to on this platform. It is the
+    # one place the state is visible without root AND without debugfs — but the sysfs file
+    # is mode 0400, so unprivileged it usually reads back as an honest UNKNOWN. The kernel
+    # command line is the world-readable second source: an explicit `i915.enable_guc=` there
+    # is authoritative about what was ASKED for, even when the parameter file cannot be read.
+    uc = {mod: {"enable_guc": _module_param(mod, "enable_guc")} for mod in ("i915", "xe")}
+    cmdline_params = [
+        token
+        for token in (read_text("/proc/cmdline") or "").split()
+        if token.startswith(tuple("%s." % m for m in DRM_MODULES))
+    ]
+
+    return {
+        "dri_present": dri_present,
+        "devices": devices,
+        "render_nodes": render_nodes,
+        "drm_cards": cards,
+        "pci_display": pci_display,
+        "modules": modules,
+        "uc": uc,
+        "cmdline_params": cmdline_params,
+    }
+
+
 # --- Storage ------------------------------------------------------------------------
 PSEUDO_FS = {
     "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "cgroup2", "pstore",
@@ -1095,7 +1304,15 @@ def collect_os():
 # we only DETECT, and the role skips the SMART checks with an accurate reason.
 def collect_tools():
     tools = {}
-    for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu"):
+    # lspci (pciutils) and vainfo (libva-utils) are BOTH optional and NEITHER is declared in
+    # node_baseline_packages, so nothing in this repo guarantees either. Measured on prod
+    # 2026-09-30: lspci IS there (/usr/bin/lspci — Debian ships pciutils by default) and
+    # vainfo is NOT. lspci only decorates the PCI display entry with a readable name; vainfo
+    # is not needed on the host at all, because Plex ships its own libva and media driver
+    # inside its container. They are probed so the report can state their presence or absence
+    # as a fact instead of leaving the reader to wonder — and so nothing here is ever tempted
+    # to install them. This role installs nothing.
+    for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu", "lspci", "vainfo"):
         tools[tool] = which(tool)
     return tools
 
@@ -1127,6 +1344,7 @@ def main():
         "os": collect_os(),
         "dmi": collect_dmi(),
         "cpu": collect_cpu(),
+        "graphics": collect_graphics(),
         "memory": collect_memory(),
         "thermal": collect_thermal(),
         "pressure": collect_pressure(),

infra/ansible/roles/node_health/tasks/assess.yml

@@ -652,6 +652,204 @@
          ~ (', lscpu Virtualization=' ~ c.virtualization if c.virtualization else '')
          ~ (', running as a guest of ' ~ c.hypervisor_vendor if c.is_virtual_guest | bool else '') }}
 
+# --- Graphics / render device ----------------------------------------------------------
+# WHY A HEALTH CHECK AND NOT A NOTE IN A RUNBOOK. Plex transcodes on the iGPU through
+# /dev/dri/renderD128. When that node stops existing nothing fails loudly — Plex falls back
+# to software transcoding on its own and the only symptom is a hot CPU and stuttering
+# clients. A node rebuild, a BIOS setting that disables the onboard IGD, or a kernel that
+# binds a different DRM driver all produce exactly that silent degradation. It was measured
+# by hand on all three prod nodes on 2026-09-30 and found working; a fact measured once
+# decays silently, so it is re-measured on every health run instead.
+#
+# READ-ONLY, AND UNPRIVILEGED. Every fact behind these three checks comes from the
+# unprivileged pass: stat() on /dev/dri, the /sys/class/drm bound-driver symlink,
+# /sys/bus/pci/devices class codes, /proc/modules and /proc/cmdline. The role installs
+# nothing (no vainfo, no pciutils), loads no module and changes no group — see the role
+# README. The ONE fact that is root-gated is i915's enable_guc module parameter (mode 0400),
+# and it reports as an honest UNKNOWN with the reason rather than as a 0.
+#
+# THE THREE VERDICTS ARE SPLIT ON PURPOSE:
+#   graphics.render_device  the OUTCOME — can anything hardware-transcode on this node.
+#   graphics.drm_driver     the CAUSE side — what is bound, and is it what this site expects.
+#   graphics.media_stack    INFO only. GuC/HuC and host VAAPI userspace both belong here and
+#                           NEITHER is ever a failure; see the task's own comment for why.
+- name: Assess — locate the display device and its bound driver (once, shared by three checks)
+  ansible.builtin.set_fact:
+    node_health_render:
+      # Display-class PCI devices (base class 0x03), driver bound or not. This is the
+      # question "is there a GPU in this machine at all", and it is answered from the PCI
+      # bus rather than from /sys/class/drm because a card* entry only exists once a driver
+      # has BOUND — so a GPU with nothing bound to it has no card* and would otherwise look
+      # identical to a machine with no GPU.
+      display_devices: "{{ g.pci_display }}"
+      bound_drivers: >-
+        {{ g.pci_display | map(attribute='driver') | select('string') | unique | list }}
+      # Only a CHARACTER SPECIAL file is a usable render node. Anything else called renderD*
+      # is an anomaly, not a device, and must not be counted as one.
+      render_chardevs: "{{ g.render_nodes | selectattr('char_device') | list }}"
+      # Video (vcs*) and video-enhancement (vecs*) engines across every bound card — the
+      # difference between "a GPU is bound" and "this GPU can do media work".
+      media_engines: >-
+        {{ g.drm_cards | map(attribute='media_engines') | flatten | unique | sort | list }}
+  vars:
+    ansible_become: false
+    g: "{{ node_health_facts.graphics }}"
+
+- name: Assess — DRM render node for hardware transcoding
+  ansible.builtin.set_fact:
+    node_health_checks: "{{ node_health_checks + [{
+      'id': 'graphics.render_device',
+      'title': 'DRM render node present for hardware transcoding (/dev/dri/renderD*)',
+      'status': render_status,
+      'detail': render_detail }] }}"
+  vars:
+    ansible_become: false
+    g: "{{ node_health_facts.graphics }}"
+    r: "{{ node_health_render }}"
+    # SKIP, not PASS and not FAIL, when the machine has no display device at all: there is
+    # then nothing to grade, and the same convention thermal.max already uses for "no sensor
+    # on this hardware" applies. FAIL is reserved for the state that actually breaks Plex —
+    # a GPU IS on the bus and there is still no render node.
+    render_status: >-
+      {{ 'SKIP' if (r.display_devices | length) == 0
+         else ('PASS' if (r.render_chardevs | length) > 0 else 'FAIL') }}
+    dri_listing: >-
+      {{ (g.devices | map(attribute='name') | join(', ')) if (g.devices | length) > 0
+         else 'nothing' }}
+    render_detail: >-
+      {%- if (r.display_devices | length) == 0 -%}
+      this machine exposes no display-class PCI device (base class 0x03) at all, so there is
+      no GPU here for a render node to belong to and hardware transcoding is not available on
+      this node by construction. Normal for a VM guest. Measured from /sys/bus/pci/devices,
+      not assumed.
+      {%- elif (r.render_chardevs | length) > 0 -%}
+      {{ r.render_chardevs | length }} render node(s):
+      {% for d in r.render_chardevs %}{{ d.path }} (character special, mode {{ d.mode }},
+      owner {{ d.owner | default(d.uid, true) }}, group {{ d.group | default('unresolved', true) }}
+      gid {{ d.gid }}){{ ', ' if not loop.last else '' }}{% endfor %}.
+      /dev/dri holds: {{ dri_listing }}.
+      THE OWNING GROUP AND GID ARE REPORTED AS OBSERVED, NEVER ASSERTED against a literal:
+      `render` is a systemd DYNAMIC allocation rather than a Debian static one, so a rebuilt
+      node may legitimately land on a different number (all three prod nodes read gid 992 on
+      2026-09-30), and Plex's container entrypoint stats the device and joins whichever group
+      owns it. Drift is therefore visible in this line without being a failure.
+      {%- else -%}
+      HARDWARE TRANSCODING IS BROKEN ON THIS NODE. A display-class device IS present on the
+      PCI bus ({{ r.display_devices | map(attribute='address') | join(', ') }}) but there is
+      no usable character-special render node: /dev/dri
+      {{ 'does not exist' if not (g.dri_present | bool) else 'holds ' ~ dri_listing }}.
+      {{ ('Note that /dev/dri holds ' ~ (g.render_nodes | map(attribute='name') | join(', '))
+      ~ ' — present, but NOT character special, which is an anomaly in its own right. ')
+         if (g.render_nodes | length) > 0 else '' }}
+      Nothing fails loudly in this state — Plex silently falls back to software transcoding
+      and the only symptom is CPU load. Check graphics.drm_driver below for whether a DRM
+      driver bound at all, then whether the onboard IGD is still enabled in BIOS.
+      {%- endif -%}
+
+- name: Assess — DRM driver bound to the display device
+  ansible.builtin.set_fact:
+    node_health_checks: "{{ node_health_checks + [{
+      'id': 'graphics.drm_driver',
+      'title': 'DRM driver bound to the display device (expected ' ~ expected ~ ')',
+      'status': driver_status,
+      'detail': driver_detail }] }}"
+  vars:
+    ansible_become: false
+    g: "{{ node_health_facts.graphics }}"
+    r: "{{ node_health_render }}"
+    expected: "{{ node_health_graphics_expected_driver }}"
+    # WARN rather than FAIL when a DIFFERENT driver is bound (the `xe` case): the device
+    # still works, but the userspace media stack is not the same one, so it is a change the
+    # operator must see before a transcode regression surfaces as a user complaint. FAIL is
+    # for nothing bound at all, which is the cause of a missing render node.
+    driver_status: >-
+      {{ 'SKIP' if (r.display_devices | length) == 0
+         else ('FAIL' if (r.bound_drivers | length) == 0
+               else ('PASS' if expected in r.bound_drivers else 'WARN')) }}
+    driver_detail: >-
+      {%- if (r.display_devices | length) == 0 -%}
+      no display-class PCI device on this machine, so no DRM driver can be bound to one.
+      Measured from /sys/bus/pci/devices.
+      {%- else -%}
+      {% for p in r.display_devices %}{{ p.address }}
+      {{ p.description | default(((p.vendor_id | default('?', true)) ~ ':' ~ (p.device_id | default('?', true))), true) }}
+      -> driver {{ p.driver | default('NONE BOUND', true) }}{{ '; ' if not loop.last else '' }}{% endfor %}.
+      {{ ('DRM cards: ' ~ (g.drm_cards | map(attribute='card') | zip(g.drm_cards | map(attribute='driver'))
+           | map('join', '=') | join(', ')) ~ '. ') if (g.drm_cards | length) > 0
+         else 'No /sys/class/drm/card* entry exists, which is consistent with no driver being bound. ' }}
+      Module load state (/proc/modules, corroboration only — a built-in driver never appears
+      there, the bound-driver symlink above is what decides):
+      {{ g.modules | dict2items | map(attribute='key') | zip(g.modules | dict2items | map(attribute='value') | map('string'))
+         | map('join', '=') | join(', ') }}.
+      {%- if (r.bound_drivers | length) == 0 %}
+      NOTHING IS BOUND to the display device. That is why there is no render node: the DRM
+      subsystem creates /dev/dri/renderD* only when a driver attaches. Check whether the
+      module exists for this kernel and whether it was blacklisted or failed to probe.
+      {%- elif expected in r.bound_drivers %}
+      This is the expected driver for this site (node_health_graphics_expected_driver).
+      {%- else %}
+      EXPECTED `{{ expected }}`, GOT `{{ r.bound_drivers | join(', ') }}`. The device still
+      works, so this is not an outage — but the two DRM drivers do not share a userspace
+      media stack, and a libva/iHD build that transcodes correctly against one is not
+      automatically right for the other. Confirm Plex still hardware-transcodes before
+      treating this as benign.
+      {%- endif %}
+      {%- endif -%}
+
+# INFO ONLY, AND THAT IS A DELIBERATE RULING, NOT AN OVERSIGHT:
+#
+#   GuC/HuC firmware not loaded (i915.enable_guc=0) — the accepted state on these nodes. It
+#   costs low-power H.264 encode QUALITY, nothing else; decode and encode both still work
+#   through the media engines. Grading it would put a permanent WARN on a configuration the
+#   operator has chosen.
+#   No host VAAPI userspace / no `vainfo` — NOT a defect. Plex ships its own libva and media
+#   driver inside its container, so the host needs neither. Flagging it would invite someone
+#   to `apt install vainfo` on a node to clear a health check, and this role installs nothing.
+#
+# The media ENGINES are the fact worth watching in this line: vcs* (video) and vecs*
+# (video-enhancement) are what a transcode actually runs on.
+- name: Assess — media engines, GuC/HuC firmware and host VAAPI userspace (informational)
+  ansible.builtin.set_fact:
+    node_health_checks: "{{ node_health_checks + [{
+      'id': 'graphics.media_stack',
+      'title': 'Media engines, GuC/HuC firmware and host VAAPI userspace (informational)',
+      'status': ('SKIP' if (node_health_render.display_devices | length) == 0 else 'INFO'),
+      'detail': media_detail }] }}"
+  vars:
+    ansible_become: false
+    g: "{{ node_health_facts.graphics }}"
+    r: "{{ node_health_render }}"
+    # `.get()` rather than a subscript: uc only carries the two Intel drivers, and a site
+    # that set the expected driver to amdgpu must get a clean "no such parameters" line
+    # instead of an undefined-key traceback.
+    guc: "{{ g.uc.get(node_health_graphics_expected_driver) }}"
+    guc_text: >-
+      {{ ('enable_guc=' ~ guc.enable_guc.value) if (guc is mapping and guc.enable_guc.value is not none)
+         else ('NOT READ — ' ~ (guc.enable_guc.reason | default('no reason recorded', true))
+               if guc is mapping
+               else 'no /sys/module/' ~ node_health_graphics_expected_driver ~ '/parameters at all') }}
+    media_detail: >-
+      {%- if (r.display_devices | length) == 0 -%}
+      no display-class PCI device on this machine, so there is no media engine, no GuC/HuC
+      firmware and nothing to transcode with.
+      {%- else -%}
+      media engines:
+      {{ (r.media_engines | join(', ')) if (r.media_engines | length) > 0
+         else 'NONE exposed under /sys/class/drm/card*/engine — a bound driver with no vcs*/vecs* engine cannot do media work' }}
+      (all engines: {{ (g.drm_cards | map(attribute='engines') | flatten | unique | sort | join(', ')) | default('none', true) }}).
+      GuC/HuC: {{ guc_text }}.
+      {{ ('Kernel cmdline carries ' ~ (g.cmdline_params | join(' '))) if (g.cmdline_params | length) > 0
+         else 'No i915/xe parameter on the kernel cmdline' }}.
+      GuC/HuC being off is a KNOWN, ACCEPTED state here and is never graded: it costs
+      low-power H.264 encode quality only, and both decode and encode still run on the
+      engines above.
+      Host VAAPI userspace: vainfo
+      {{ ('present at ' ~ node_health_facts.tools.vainfo) if node_health_facts.tools.vainfo
+         else 'NOT installed on this host' }}
+      — also never graded, because Plex ships its own libva and media driver inside its
+      container, so the host needs none. This role installs nothing.
+      {%- endif -%}
+
 # --- Thermals -----------------------------------------------------------------------
 - name: Assess — temperatures
   ansible.builtin.set_fact:

infra/ansible/roles/node_health/templates/summary.txt.j2

@@ -136,6 +136,37 @@
 {% endfor %}
 {% endfor %}
 
+--------------------------------------------------------------------------------
+ GRAPHICS / RENDER DEVICE (the Plex hardware-transcoding path)
+--------------------------------------------------------------------------------
+{% set gfx = node_health_facts.graphics %}
+{% if gfx.pci_display %}
+{% for p in gfx.pci_display %}
+ pci  {{ '%-12s' | format(p.address) }} {{ p.description | default(((p.vendor_id | default('?', true)) ~ ':' ~ (p.device_id | default('?', true)) ~ ' class ' ~ (p.class_id | default('?', true))), true) }}
+              driver {{ p.driver | default('NONE BOUND', true) }}
+{% endfor %}
+{% else %}
+ pci         no display-class device (PCI base class 0x03) on this machine — no GPU here
+{% endif %}
+{% for c in gfx.drm_cards %}
+ drm  {{ '%-12s' | format(c.card) }} driver {{ '%-8s' | format(c.driver | default('-', true)) }} at {{ c.pci_address | default('-', true) }}  engines {{ c.engines | join(' ') | default('none exposed', true) }}
+{% endfor %}
+{% if gfx.dri_present %}
+{% for d in gfx.devices %}
+ dev  {{ '%-12s' | format(d.name) }} {{ '%-18s' | format('character special' if d.char_device else 'NOT A CHAR DEVICE') }} mode {{ d.mode }}  {{ d.owner | default(d.uid, true) }}:{{ d.group | default('(gid unresolved)', true) }} gid {{ d.gid }}
+{% endfor %}
+{% else %}
+ dev         /dev/dri DOES NOT EXIST — no DRM device node on this node at all
+{% endif %}
+{# GID is printed, never asserted: `render` is a systemd dynamic allocation, so a rebuild may
+   legitimately move it and Plex's container init joins whichever group owns the device. See
+   defaults/main.yml (node_health_graphics_expected_driver) for the full reasoning. #}
+{% set guc = gfx.uc.get(node_health_graphics_expected_driver) %}
+ GuC/HuC     {{ ('enable_guc=' ~ guc.enable_guc.value) if (guc and guc.enable_guc.value is not none) else (('NOT READ — ' ~ (guc.enable_guc.reason | default('no reason recorded', true))) if guc else ('no /sys/module/' ~ node_health_graphics_expected_driver ~ '/parameters')) | regex_replace('\s+', ' ') | trim | wordwrap(64) | indent(14) }}
+             {{ ('kernel cmdline: ' ~ (gfx.cmdline_params | join(' '))) if gfx.cmdline_params else 'no i915/xe parameter on the kernel cmdline' }}
+ modules     {% for m, loaded in gfx.modules.items() %}{{ m }}={{ loaded | string | lower }}{{ ', ' if not loop.last }}{% endfor %} (/proc/modules — corroboration only; the bound-driver symlink above decides)
+ vainfo      {{ node_health_facts.tools.vainfo | default('NOT installed on this host — not a defect: Plex ships its own libva/media driver inside its container', true) }}
+
 --------------------------------------------------------------------------------
  PRESSURE STALL (/proc/pressure — % of wall time stalled)
 --------------------------------------------------------------------------------

platform/intel-gpu-plugin/README.md

@@ -0,0 +1,225 @@
+# platform/intel-gpu-plugin
+
+**Intel Device Plugin for Kubernetes (GPU)** — the standalone GPU plugin DaemonSet from
+[`intel/intel-device-plugins-for-kubernetes`](https://github.com/intel/intel-device-plugins-for-kubernetes),
+pinned at **v0.37.0**. It makes every node advertise **`gpu.intel.com/i915`** so a workload can
+*request* a GPU instead of hoping a `hostPath` is enough.
+
+- **Serves:** hardware transcoding for Plex (`hardware.gpu: intel-quicksync`), and any later
+  consumer of an Intel iGPU.
+- **Tier:** its **own** Flux Kustomization
+  ([`clusters/base/intel-gpu-plugin.yaml`](../../clusters/base/intel-gpu-plugin.yaml),
+  `dependsOn: cilium`, `wait: true`) — **not** a line in `platform/kustomization.yaml`. See
+  *Placement* below.
+- **Namespace:** `intel-gpu-plugin` — a PLATFORM namespace, labelled
+  `homelab.keithjackson.dev/app-namespace: "false"`, so the app-scoped Kyverno policies
+  (default-deny generation, auth-mode, egress-class, host-namespaces) do not apply.
+- **No RBAC, no ServiceAccount, no CRDs, no webhook, no API-server access.** A device plugin talks
+  to the *local kubelet* over a unix socket.
+
+## What this component owns
+
+| File | Purpose |
+|---|---|
+| `namespace.yaml` | `intel-gpu-plugin` ns, explicitly `app-namespace=false` |
+| `intel-gpu-plugin.yaml` | the DaemonSet, **vendored** (machine-generated) from upstream v0.37.0 |
+| `kustomization.yaml` | namespace scoping + the pinned image (`images:` transformer) |
+
+## The problem this fixes
+
+Plex was given a `/dev/dri` hostPath (merged on `main` @ `cdf3e14`). The device is **visible** in
+the container and **cannot be opened**. Measured in the running pod on `k3s-prod-2`:
+
+```
+uid 0,    O_RDWR -> EPERM "Operation not permitted"
+uid 1000, O_RDWR -> EPERM "Operation not permitted"
+```
+
+`EPERM` from the **device cgroup**, not `EACCES` from file permissions — proven three ways:
+
+1. root on the **host** opens the same device with `rc=0`;
+2. the node is unified `cgroup2fs`;
+3. the container's runc `config.json` reads
+   `linux.resources.devices: [{'allow': False, 'access': 'rwm'}]` and `linux.devices: None` —
+   one deny-all rule, no allow entry for major 226, and the runtime was never asked to grant a
+   device.
+
+A hostPath is a **filesystem** operation. It makes the device *node* appear and adds no cgroup
+allow rule, because Kubernetes cannot infer one from a volume.
+
+**A device plugin is the mechanism that closes exactly this gap.** Verified in the v0.37.0 source
+(`cmd/gpu_plugin/gpu_plugin.go`, `createDeviceSpecsFromDrmFiles`), the `Allocate` response carries
+per DRM node:
+
+```go
+specs = append(specs, pluginapi.DeviceSpec{
+    HostPath: devPath, ContainerPath: devPath, Permissions: "rw",
+})
+```
+
+and it is that `DeviceSpec` which makes kubelet populate `linux.devices` **and** the matching
+cgroup allow rule. It is the only **non-privileged** way to get there. `privileged: true` would
+also work and is not available: `platform/kyverno/policies/restrict-host-namespaces.yaml` blocks it
+under Enforce in app namespaces, and `plex` is labelled `app-namespace=true` (measured
+2026-10-01). Correctly so.
+
+## Placement — why its own Kustomization
+
+The cheap option was one line in the shared `platform` tier. Neither that file nor a new
+`platform/<name>/` directory is a sensitive path in `infra/gitea/approval-policy.json`, so it would
+have needed no human reviewer *on the policy's terms*. It is still the wrong place, and the repo
+already argued the case when Cilium was split out (read the comment in
+`platform/kustomization.yaml`):
+
+* `platform` is `wait: true` over eleven-plus components, so its Ready condition cannot distinguish
+  "the GPU plugin is wedged" from "longhorn is slow".
+* The only emergency valve would be `flux suspend kustomization platform` — freezing storage, TLS,
+  DNS, ingress, monitoring, Keycloak and KubeVirt in order to stop touching a device plugin.
+* The tier's 10m/5m interval/timeout are called out there as **wrong for a 3-node DaemonSet
+  rollout**. This is a 3-node DaemonSet rollout.
+
+The consequence is worse than average here because of what sits downstream: `platform` gates
+`platform-config`, which gates `apps-wave-0` (`wait: true`), which gates `apps-wave-1`. A plugin
+that never went healthy inside that tier would stall **every app** on "dependency not ready" — so a
+component added to fix a *silent no-op* (software transcoding) would have acquired the ability to
+cause a *real outage*.
+
+The usual objection — that `clusters/**` needs a human approval — costs nothing: Gitea enforces
+`required_approvals: 1` with `block_admin_merge_override: true` on **every** PR here regardless of
+path, and authors cannot self-approve. The approval was required either way.
+
+### `wait: true` is a health signal, not a gate
+
+This departs from the `headlamp`/`crafty` leaf template, deliberately. Those run `wait: false`
+because a leaf UI can be legitimately not-Ready for a long time during a cold start while it waits
+on Keycloak/DNS/TLS, making its health useless as a signal. This DaemonSet has no such dependency —
+it reads sysfs and opens a socket, so on healthy nodes it is Ready in seconds and **not-Ready
+genuinely means broken**. `wait: true` makes `flux get kustomization intel-gpu-plugin` a one-line
+answer to "did it come up on all three nodes", which is precisely the question that must be
+answered before the consumer PR may merge.
+
+Nothing in the graph `dependsOn` this, so `wait: true` gates nothing. **Think hard before adding
+`dependsOn: [intel-gpu-plugin]`** anywhere — for the app waves that would hand a device plugin the
+power to stall every app, which is the thing this placement exists to prevent.
+
+## Deliberate non-choices
+
+**Standalone DaemonSet, not the operator.** The operator adds a CRD (`GpuDevicePlugin`), a
+controller-manager Deployment, cert-manager-backed admission webhooks and a second namespace —
+three more things that can be unhealthy — to buy declarative management of a DaemonSet that is
+already declared in git. For three identical nodes and one consumer that is strictly more blast
+radius for no gain.
+
+**It lists devices; it never opens them** — worth being certain about, since *opening* is exactly
+what fails today. Verified in v0.37.0 source: the scan path uses only `os.ReadDir`, `os.ReadFile`,
+`os.Stat` and `os.Readlink` over `/sys/class/drm` and `/dev/dri`. There is no `os.Open`, no
+`OpenFile`, no `ioctl` and no raw syscall against a DRM node anywhere in
+`cmd/gpu_plugin/gpu_plugin.go`. Listing and `stat()` demonstrably work today — that is how the Plex
+probe found the device before failing to open it — so the plugin needs no cgroup grant of its own
+and no privilege. Accordingly `/dev/dri` and `/sys/class/drm` are mounted **readOnly**.
+
+* **`--health-management` stays OFF** (upstream default). It is the one flag that *would* need to
+  open a device: it starts a Level-Zero sidecar (`setupLevelZeroService` returns early unless the
+  flag is set). Enabling it would reintroduce the exact `EPERM` this component exists to fix.
+* `/dev/dri/by-path` being absent is non-fatal — `newDevicePlugin` warns and sets
+  `bypathFound = false`.
+
+**`--shared-dev-num` left at the upstream default of 1** (i.e. not set). The flag multiplies each
+card into N schedulable device IDs, letting N containers share one GPU. Not raised, because:
+
+* Plex is the only consumer today and wants one device;
+* the count is a **lie the scheduler believes** — there is no GPU-side partitioning or quota behind
+  it, so N containers get the same `renderD128` and contend for the same two media engines
+  (`vcs0`/`vecs0`). Raising it converts "Unschedulable", which is loud and obvious, into
+  "transcodes are mysteriously slow", which is neither;
+* it is a one-line, zero-migration change on the day a second consumer exists — and *then* the
+  right number will be known rather than guessed.
+
+**No NFD overlay.** All three nodes are identical (HP EliteDesk 800 G5 Mini, i7-9700T, UHD Graphics
+630, PCI `8086:3e98`, `i915` bound), so a nodeSelector on a GPU feature label would gate on
+something true everywhere. If a GPU-less node ever joins, add upstream's `nfd` overlay *then*
+rather than relying on the plugin's behaviour on such a node.
+
+**No `supplementalGroups`, anywhere.** The `pms-docker` image's own cont-init hook stats each
+`/dev/dri` char device and adds the `plex` user to its group — measured live as
+`Groups: 44 100 992 1000` on the running Plex process, where GID 992 is the host `render` group.
+That mechanism is image-side and works identically on a plugin-injected device.
+
+## Resource name
+
+`gpu.intel.com/i915`. Not a guess: `scan()` calls `devTree.AddDevice(devProps.driver(), ...)`, so
+the suffix is the **kernel driver name**. These nodes have `i915` bound (not `xe`), so the resource
+is `i915`. A future kernel moving CoffeeLake to `xe` would silently rename it to `gpu.intel.com/xe`
+and a consumer's request would stop matching — worth knowing before that kernel arrives.
+
+## Versioning
+
+Pinned to **v0.37.0** via the `images:` transformer in `kustomization.yaml`; the DaemonSet itself is
+vendored as `kubectl kustomize` output of the upstream base (see its header for the exact re-vendor
+command and both sha256s). It is *generated* rather than a byte copy because upstream mixes YAML
+sequence-indentation styles within the one document and `ci/yamllint.yaml` sets
+`indent-sequences: consistent` — a verbatim copy is a hard lint error that blocks the gate
+(measured: it did). `kubectl kustomize` normalises the formatting, and the result was proven to be
+a formatting-only change: `yq -P 'sort_keys(..)'` of the generated file and of the raw upstream file
+diff clean. This is also the re-vendor idiom `platform/external-snapshotter` documents.
+
+Vendored rather than referenced as a remote kustomize base for the same reason as
+`platform/external-snapshotter`: a remote base is `git fetch`ed on **every** reconcile, those
+fetches intermittently exceeded kustomize-controller's 27s timeout, and transient GitHub slowness
+then rendered as "kustomize build failed". A local file makes the build offline and deterministic —
+which also means the CI gate's `kustomize build` proves the manifests rather than proving GitHub
+was up.
+
+Renovate tracks it through the inline-marker idiom (`# renovate: datasource=github-releases
+depName=...` immediately above the version literal), because `renovate.json` sets
+`"docker": {"enabled": false}` and `"kubernetes": {"enabled": false}` — an image reference in a
+plain manifest is tracked by nothing. **Caveat:** upstream *release* tags carry a `v` (`v0.37.0`)
+while *image* tags do not (`0.37.0`). Whether Renovate normalises that for this dep is unverified;
+if it ever proposes `newTag: v0.38.0`, the fix is an `extractVersion` packageRule in
+`renovate.json`, not a hand-edit.
+
+## Ordering — do not reorder the stack
+
+The renderer change (`catalog/render.py` emitting `gpu.intel.com/i915: 1` for
+`hardware.gpu: intel-quicksync` instead of the hostPath) **must not land until this plugin is
+merged AND confirmed advertising the resource on all three nodes.** If the renderer requests a
+resource no node advertises, the Plex pod is **Unschedulable**; `apps-wave-0` is `wait: true` and
+`apps-wave-1` `dependsOn` it, so that stalls the whole app tier — turning today's silent no-op into
+a real outage.
+
+## Post-merge verification
+
+Nothing about this component's runtime behaviour can be verified from a PR: it does not exist until
+the operator merges and Flux reconciles. Afterwards:
+
+```bash
+D=./infra/devcontainer/dev.sh
+
+# 1. Flux applied it, and (wait:true) the DaemonSet went healthy.
+$D run bash -c 'flux get kustomization intel-gpu-plugin'
+$D run bash -c 'kubectl --context k3s-prod -n intel-gpu-plugin get ds intel-gpu-plugin'
+#    want: DESIRED=3 CURRENT=3 READY=3 AVAILABLE=3
+
+# 2. The resource is advertised on ALL THREE nodes. This is the gate on the renderer PR.
+$D run bash -c 'kubectl --context k3s-prod get nodes -o json \
+  | jq -r ".items[] | \"\(.metadata.name) i915=\(.status.allocatable[\"gpu.intel.com/i915\"] // \"ABSENT\")\""'
+#    want: three lines, each i915=1   (before this change: ABSENT on all three)
+
+# 3. If a node shows ABSENT, read that node's plugin logs.
+$D run bash -c 'kubectl --context k3s-prod -n intel-gpu-plugin logs -l app=intel-gpu-plugin --tail=50 --prefix'
+```
+
+### The decisive test
+
+Resource-advertised is necessary but **not** sufficient — it proves the plugin registered, not that
+the cgroup allow rule reaches Plex. The decisive test is re-running the `open()` probe **as uid 1000
+through `s6-setuidgid plex`** inside the Plex pod, *after* the renderer PR has switched Plex to
+request the resource:
+
+```bash
+$D run bash -c 'kubectl --context k3s-prod -n plex exec deploy/plex -- \
+  s6-setuidgid plex python3 -c "import os; os.close(os.open(\"/dev/dri/renderD128\", os.O_RDWR)); print(\"rc=0 OK\")"'
+```
+
+**Passing as uid 0 is weaker evidence and must not be accepted as proof** — Plex transcodes as uid
+1000, and the two differ in group membership, which is the half the image's cont-init hook supplies.

platform/intel-gpu-plugin/intel-gpu-plugin.yaml

@@ -0,0 +1,117 @@
+---
+# VENDORED, MACHINE-GENERATED, from intel/intel-device-plugins-for-kubernetes @ v0.37.0:
+#   deployments/gpu_plugin/base
+#
+# RE-VENDOR on a version bump (and bump `newTag` in kustomization.yaml to match):
+#   kubectl kustomize \
+#     "https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/gpu_plugin/base?ref=vX.Y.Z" \
+#     > platform/intel-gpu-plugin/intel-gpu-plugin.yaml
+#   # then re-add this header, and READ THE DIFF — upstream has changed the securityContext
+#   # and the volume set between minor releases before.
+#
+# Generated 2026-10-01 at v0.37.0; sha256 of the body below (header excluded):
+#   80ebd070f983f3116f4004f719d9c872220997ec766b4ab7e7290585ac8ef842
+# The raw upstream source file reviewed to produce it
+# (deployments/gpu_plugin/base/intel-gpu-plugin.yaml) was sha256
+#   c8773918c2370e711d4ba84120a9971a290bf75ecf81dc520782bcccd7ef24b6
+# and the two were proven EQUIVALENT, not merely similar: `yq -P 'sort_keys(..)'` of each diffs
+# clean, so passing it through kustomize changed formatting only and no content.
+#
+# WHY GENERATED RATHER THAN A BYTE COPY of the upstream file: upstream mixes YAML sequence
+# indentation styles within the one document (`containers:` non-indented, `env:` indented), and
+# ci/yamllint.yaml sets `indentation: indent-sequences: consistent`, so a verbatim copy is a hard
+# lint ERROR and blocks the merge gate — measured, this file failed exactly that way first time
+# round. `kubectl kustomize` normalises the output, which is also precisely the re-vendor idiom
+# platform/external-snapshotter documents for the same reason.
+#
+# WHY VENDORED AT ALL rather than a remote kustomize base (`...gpu_plugin/base?ref=v0.37.0` left in
+# `resources:`): the same reason platform/external-snapshotter is vendored. A remote base is
+# `git fetch`ed by Flux's kustomize-controller on EVERY reconcile, those fetches intermittently
+# exceeded the controller's 27s timeout, and a transient GitHub slowness then rendered as
+# "kustomize build failed" on the whole tier. A local file makes the build OFFLINE and
+# deterministic — which also means `kustomize build` in the CI gate proves the real thing instead
+# of proving GitHub was up. (platform/kubevirt still carries the remote-base pattern and its latent
+# fragility; not this component's problem.)
+#
+# DO NOT HAND-EDIT THE BODY. Every local change belongs in kustomization.yaml as a transformer or
+# patch, so the next re-vendor stays a mechanical regenerate-and-diff instead of a three-way merge.
+# The image tag below is re-pinned (and fully qualified) by the `images:` transformer there. The
+# rationale for each thing deliberately NOT changed — no privileged, no supplementalGroups, no
+# -shared-dev-num, no NFD overlay, no health-management — is in README.md.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: intel-gpu-plugin
+  name: intel-gpu-plugin
+spec:
+  selector:
+    matchLabels:
+      app: intel-gpu-plugin
+  template:
+    metadata:
+      labels:
+        app: intel-gpu-plugin
+    spec:
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: HOST_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        image: intel/intel-gpu-plugin:0.37.0
+        imagePullPolicy: IfNotPresent
+        name: intel-gpu-plugin
+        resources:
+          limits:
+            cpu: 100m
+            memory: 115Mi
+          requests:
+            cpu: 40m
+            memory: 45Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          seLinuxOptions:
+            type: container_device_plugin_t
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /dev/dri
+          name: devfs
+          readOnly: true
+        - mountPath: /sys/class/drm
+          name: sysfsdrm
+          readOnly: true
+        - mountPath: /var/lib/kubelet/device-plugins
+          name: kubeletsockets
+        - mountPath: /var/run/cdi
+          name: cdipath
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      volumes:
+      - hostPath:
+          path: /dev/dri
+        name: devfs
+      - hostPath:
+          path: /sys/class/drm
+        name: sysfsdrm
+      - hostPath:
+          path: /var/lib/kubelet/device-plugins
+        name: kubeletsockets
+      - hostPath:
+          path: /var/run/cdi
+          type: DirectoryOrCreate
+        name: cdipath
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate

platform/intel-gpu-plugin/kustomization.yaml

@@ -0,0 +1,99 @@
+---
+# Intel GPU device plugin for Kubernetes — makes the nodes advertise `gpu.intel.com/i915`.
+#
+# WHAT THIS FIXES, and why nothing cheaper works. Plex already has a `/dev/dri` hostPath mount
+# (merged on main @ cdf3e14). The device node is VISIBLE in the container and cannot be OPENED.
+# Measured in the running pod on k3s-prod-2:
+#     uid 0,    O_RDWR -> EPERM "Operation not permitted"
+#     uid 1000, O_RDWR -> EPERM "Operation not permitted"
+# EPERM from the DEVICE CGROUP, not EACCES from file permissions — proven three ways: root on the
+# HOST opens the same device rc=0; the node is unified cgroup2fs; and the container's runc
+# config.json carries one deny-all rule and nothing else:
+#     linux.resources.devices: [{'allow': False, 'access': 'rwm'}]
+#     linux.devices: None
+# No allow entry for major 226, and the runtime was never asked to grant a device. A hostPath is a
+# FILESYSTEM operation: it makes the device node appear, and adds no cgroup allow rule, because
+# Kubernetes cannot infer one from a volume.
+#
+# A device plugin is the mechanism that closes exactly that gap. Verified by reading the v0.37.0
+# source (cmd/gpu_plugin/gpu_plugin.go, createDeviceSpecsFromDrmFiles): the Allocate response
+# carries `pluginapi.DeviceSpec{HostPath, ContainerPath, Permissions: "rw"}` per DRM node, and it
+# is that DeviceSpec which makes kubelet populate `linux.devices` AND the matching cgroup allow
+# rule. It is the only NON-PRIVILEGED way to get there. `privileged: true` would also work and is
+# not available: platform/kyverno/policies/restrict-host-namespaces.yaml blocks it under Enforce in
+# app namespaces, and `plex` is labelled app-namespace=true (measured 2026-10-01). Correctly so.
+#
+# STANDALONE DAEMONSET, NOT THE OPERATOR. Upstream ships both. The operator adds a CRD
+# (GpuDevicePlugin), a controller-manager Deployment, cert-manager-backed admission webhooks and a
+# second namespace — i.e. three more things that can be unhealthy — to buy declarative management
+# of a DaemonSet we are declaring in git anyway. For three identical nodes and one consumer that is
+# strictly more blast radius for no gain. The DaemonSet has no CRDs, no controller, no webhook, and
+# no API-server access at all: a device plugin talks to the LOCAL kubelet over a unix socket, which
+# is why there is no ServiceAccount or RBAC here (and therefore no `**/rbac.yaml` sensitive path).
+#
+# LISTING, NOT OPENING — the thing worth being sure about, since opening is precisely what fails
+# today. Verified in v0.37.0 source: the scan path uses only os.ReadDir / os.ReadFile / os.Stat /
+# os.Readlink over /sys/class/drm and /dev/dri. There is no os.Open, no OpenFile, no ioctl and no
+# syscall of any kind against a DRM node anywhere in cmd/gpu_plugin/gpu_plugin.go. Listing and
+# stat() demonstrably work today (that is how the Plex probe found the device before failing to
+# open it), so the plugin needs no cgroup grant of its own and no privilege.
+#   * `--health-management` is left OFF (upstream default). It is the one flag that WOULD need to
+#     open a device: it starts a Level-Zero sidecar (setupLevelZeroService returns early unless the
+#     flag is set). Turning it on would reintroduce the exact EPERM this component exists to fix.
+#   * /dev/dri and /sys/class/drm are mounted readOnly, consistent with list-and-stat only.
+#   * /dev/dri/by-path absence is non-fatal (newDevicePlugin warns and sets bypathFound=false).
+#
+# RESOURCE NAME: `gpu.intel.com/i915`. Not a guess — `scan()` does
+# `devTree.AddDevice(devProps.driver(), ...)`, so the suffix is the KERNEL DRIVER name. These nodes
+# have i915 bound (not xe) on UHD Graphics 630 / PCI 8086:3e98, so the resource is i915. A future
+# kernel moving CoffeeLake to xe would silently rename it to `gpu.intel.com/xe` and the consumer's
+# request would stop matching — worth knowing before that kernel arrives.
+#
+# `--shared-dev-num` LEFT AT THE UPSTREAM DEFAULT OF 1 (so: not set). The flag multiplies each
+# card into N schedulable device IDs (`for i := 0; i < sharedDevNum; i++`), letting N containers
+# share one GPU. Deliberately not raised:
+#   * Plex is the only consumer and wants one device.
+#   * The count is a LIE the scheduler believes. There is no GPU-side partitioning or quota behind
+#     it — N containers get the same renderD128 and contend for the same two media engines
+#     (vcs0/vecs0). Raising it converts "Unschedulable", which is loud and obvious, into
+#     "transcodes are mysteriously slow", which is neither.
+#   * It is a one-line, zero-migration change on the day a second consumer actually exists, and at
+#     that point the right number will be known rather than guessed.
+# No NFD overlay either: all three nodes are identical (established cluster fact), so a nodeSelector
+# on a GPU feature label would gate on something that is true everywhere. If a GPU-less node ever
+# joins, add upstream's nfd overlay THEN rather than relying on the plugin's behaviour there.
+#
+# NO supplementalGroups, anywhere, on purpose. The pms-docker image's own cont-init hook stats each
+# /dev/dri char device and adds the `plex` user to its group — measured live as `Groups: 44 100 992
+# 1000` on the running Plex process, where GID 992 is the host `render` group. That mechanism is
+# image-side and works identically on a plugin-injected device, so adding a supplementalGroup here
+# or on the consumer would duplicate a thing that already works.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: intel-gpu-plugin
+resources:
+  - namespace.yaml
+  - intel-gpu-plugin.yaml
+images:
+  # PINNED, never `latest`. Upstream release v0.37.0 (2026-09-16); verified present on Docker Hub
+  # 2026-10-01, manifest digest sha256:7001200071db10587fc31184eedda4f5cf291efba737adf85a8c80539f7fa5e1
+  # (single amd64 manifest, not an index). Recorded here rather than pinned by digest because this
+  # repo pins upstream platform components by VERSION TAG (platform/kubevirt, external-snapshotter);
+  # digest pinning is reserved for the conformance runner, where it is a security boundary.
+  # `newName` fully qualifies the registry: upstream writes the bare `intel/intel-gpu-plugin`, which
+  # relies on containerd's docker.io default rather than saying so.
+  #
+  # RENOVATE: the inline-marker idiom (customManagers in renovate.json), not the docker datasource —
+  # renovate.json sets `"docker": {"enabled": false}` and `"kubernetes": {"enabled": false}`, so an
+  # image reference in a plain manifest is tracked by NOTHING. The marker must sit on the line
+  # IMMEDIATELY above the version literal for the regex to match, which is why it is down here on
+  # `newTag` and not up in this header.
+  # CAVEAT, stated because it will bite someone: the GitHub release tags carry a `v`
+  # (`v0.37.0`) and the IMAGE tags do not (`0.37.0` — confirmed against the Docker Hub tag list).
+  # Whether Renovate normalises that for this dep is UNVERIFIED here (no Renovate run available).
+  # If it ever proposes `newTag: v0.38.0`, the fix is an `extractVersion` packageRule in
+  # renovate.json, not a hand-edit of this line.
+  - name: intel/intel-gpu-plugin
+    newName: docker.io/intel/intel-gpu-plugin
+    # renovate: datasource=github-releases depName=intel/intel-device-plugins-for-kubernetes
+    newTag: "0.37.0"

platform/intel-gpu-plugin/namespace.yaml

@@ -0,0 +1,34 @@
+---
+# intel-gpu-plugin — namespace for the Intel GPU device plugin DaemonSet.
+#
+# PLATFORM INFRA, NOT AN APPLICATION NAMESPACE, and that distinction is load-bearing here rather
+# than cosmetic. Every Kyverno policy that could object to this DaemonSet is scoped by a
+# `namespaceSelector: matchLabels: homelab.keithjackson.dev/app-namespace: "true"` — verified by
+# reading the policies, not assumed:
+#   * platform/kyverno/policies/restrict-host-namespaces.yaml  (hostNetwork/PID/IPC + privileged)
+#   * platform/kyverno/policies/require-egress-class-label.yaml
+#   * platform/kyverno/policies/require-auth-mode-label.yaml
+#   * platform/kyverno/policies/require-default-deny-baseline.yaml (a *generate* rule, so an
+#     app-namespace also gets a default-deny NetworkPolicy minted into it)
+# Labelling this namespace "false" therefore puts it out of scope for all four. That is the same
+# mechanism restrict-host-namespaces' own header describes: "platform components that do [need host
+# access] live in platform namespaces that do NOT carry the app-namespace label".
+#
+# Measured on prod 2026-10-01: `kyverno` and `vpn-egress` are the two namespaces that carry the
+# label explicitly as "false"; the rest of the platform namespaces leave it unset. Both are out of
+# scope identically — an unset label does not match `"true"`. We set it EXPLICITLY, following
+# vpn-egress, because "someone chose this" and "nobody thought about it" should not look the same
+# in git, and because this component's correctness depends on the answer.
+#
+# NOTE this plugin does NOT actually need any of the things those policies forbid: it sets no
+# hostNetwork/hostPID/hostIPC and is not privileged (see intel-gpu-plugin.yaml). The namespace
+# placement is not a workaround for a policy — there is nothing to work around. It is here because
+# a node-level platform DaemonSet is not an application, and because the app-namespace fabric
+# (default-deny NetworkPolicy, egress-class, auth-mode) describes obligations that are meaningless
+# for a process that talks only to the local kubelet over a unix socket and never uses the network.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: intel-gpu-plugin
+  labels:
+    homelab.keithjackson.dev/app-namespace: "false"

platform/monitoring/alerts.yaml

@@ -839,6 +839,14 @@ spec:
         # that has never applied anything — or a GitRepository with no artifact yet — publishes the
         # series with no `revision` label. Without the matcher those group together under the empty
         # string and compare EQUAL, i.e. a never-applied Kustomization would read as converged.
+        #
+        # A THIRD THING THAT MUST NOT REGRESS — THE SEVERITY LABEL. These alerts are delivered
+        # because of the labels they do NOT carry as much as the ones they do. Do not "tidy" this
+        # pair by giving FluxRolloutInProgress a `severity`, and never `severity: info`: the chart's
+        # InfoInhibitor rule plus its `severity="info"` inhibit_rule suppress an info alert whenever
+        # it is the only thing firing in its namespace, which is every healthy deploy. Measured on
+        # prod 2026-09-30 — the alert fired correctly for 12 Kustomizations, resolved correctly, and
+        # was inhibited in between. The full reasoning is on the `notification: deploy` label below.
         # ===================================================================================
         - alert: FluxRolloutInProgress
           expr: |
@@ -857,7 +865,31 @@ spec:
           # `flux reconcile`, a change touching one Kustomization), and a deploy notification that
           # only sometimes arrives is worse than no notification at all.
           labels:
-            severity: info    # -> the email-deploy Alertmanager route; subject "[FLUX] applying/applied"
+            # ROUTED BY `notification`, AND DELIBERATELY CARRYING NO `severity` (fixed 2026-09-30).
+            # This alert originally used `severity: info`. That was WRONG, and wrong in the silent
+            # direction — measured on prod within minutes of the first real deploy.
+            # kube-prometheus-stack ships `general.rules` -> InfoInhibitor:
+            #     group by (namespace) (ALERTS{severity="info"} == 1)
+            #       unless on (namespace) group by (namespace)
+            #         (ALERTS{alertname!="InfoInhibitor", alertstate="firing", severity=~"warning|critical"} == 1)
+            # which fires for any namespace that has a firing info alert AND NOTHING ELSE WRONG in
+            # it; paired with the chart's inhibit_rule
+            #     source_matchers: [alertname="InfoInhibitor"]
+            #     target_matchers: [severity="info"]
+            #     equal: [namespace]
+            # the effect is that an info alert is suppressed EXACTLY WHEN it is the only thing
+            # happening in its namespace. For a deploy notification that is the only case that
+            # matters: on a healthy rollout of a healthy cluster, flux-system has no warning or
+            # critical alert, so InfoInhibitor fired and this alert never reached the receiver.
+            # Verified live: InfoInhibitor went active 2026-09-30T04:58:48Z, the same moment this
+            # alert began firing for 12 Kustomizations.
+            # Upstream's intent is explicit — `severity: info` means "only worth showing next to a
+            # real problem". A deploy notice is not that, so it must not claim that severity.
+            # `notification` is a label no inhibit_rule in the chart targets (all three match on
+            # severity values), and omitting `severity` also stops InfoInhibitor firing for
+            # flux-system at all. Route: platform/monitoring/kube-prometheus-stack.yaml matches
+            # `notification = "deploy"` -> the email-deploy receiver, subject "[FLUX] applying|applied".
+            notification: deploy
           annotations:
             summary: "Flux is applying a new revision — {{ $labels.name }} is behind main"
             description: "Kustomization {{ $labels.namespace }}/{{ $labels.name }} has applied {{ $labels.revision }}, which is not the revision the flux-system GitRepository has fetched from main. THIS MAIL MEANS A GIT CHANGE STARTED ROLLING OUT; the RESOLVED mail for this same alert means every Kustomization reached the new revision and the rollout is complete."

platform/monitoring/kube-prometheus-stack.yaml

@@ -139,11 +139,19 @@ spec:
             #
             # A DEDICATED ROUTE, because the default receiver is `email-warning`: without it a
             # routine deploy arrives subject-lined "[WARNING]" and becomes indistinguishable from a
-            # real platform fault, which is how alert mail stops being read at all. `severity: info`
-            # is used by NO other rule in this repo (grepped 2026-09-29: zero matches), so this
-            # route claims exactly the deploy notices and nothing else. Keep it that way — giving a
-            # genuine fault severity: info would silently route it here and re-label it a deploy.
-            - matchers: ['severity = "info"']
+            # real platform fault, which is how alert mail stops being read at all.
+            #
+            # MATCHES `notification`, NOT `severity` (corrected 2026-09-30). This route first read
+            # `severity = "info"`, which routed correctly and then had the mail SUPPRESSED before it
+            # was sent: the chart's own InfoInhibitor rule fires for any namespace holding a firing
+            # info alert with no warning/critical beside it, and its inhibit_rule
+            # (source alertname="InfoInhibitor" -> target severity="info", equal [namespace]) drops
+            # exactly that alert. A healthy deploy in a healthy flux-system is precisely that case,
+            # so the notification was inhibited on every occasion it was wanted. Measured live on
+            # prod 2026-09-30: InfoInhibitor went active at 04:58:48Z as the alert began firing.
+            # Routing on a non-severity label sidesteps all three of the chart's inhibit_rules,
+            # which match on severity values only. See the label's own comment in alerts.yaml.
+            - matchers: ['notification = "deploy"']
               receiver: email-deploy
               # One mail per ROLLOUT, not per Kustomization: the alert fires once per lagging
               # Kustomization (16 possible), and grouping on alertname alone collapses them into a

platform/monitoring/tests/flux-rollout.test.yaml

@@ -89,7 +89,7 @@ tests:
         alertname: FluxRolloutInProgress
         exp_alerts:
           - exp_labels:
-              severity: info
+              notification: deploy
               job: kube-state-metrics
               namespace: flux-system
               name: platform
@@ -148,7 +148,7 @@ tests:
         alertname: FluxRolloutInProgress
         exp_alerts:
           - exp_labels:
-              severity: info
+              notification: deploy
               job: kube-state-metrics
               namespace: flux-system
               name: platform
@@ -229,7 +229,7 @@ tests:
         alertname: FluxRolloutInProgress
         exp_alerts:
           - exp_labels:
-              severity: info
+              notification: deploy
               job: kube-state-metrics
               namespace: flux-system
               name: conformance

platform/oauth2-proxy/README.md

@@ -88,3 +88,72 @@ This proves the R-IDP-02 property (unauthenticated cannot reach the upstream) on
 3. Log in as a **non-platform-admins** user -> expect **403** (authenticated but not authorized).
 4. Confirm the session cookie is `.home.keithjackson.dev`-scoped (single sign-on across the other
    protected UIs).
+
+## Observability: what this component can and cannot see (measured 2026-09-30)
+
+`servicemonitor.yaml` scrapes **this namespace's** deployment and nothing else. On prod that is
+two targets, `job="oauth2-proxy"`, both `up=1`. The complete set of oauth2-proxy-specific series
+in the TSDB is:
+
+    count by (__name__) ({__name__=~"oauth2_proxy.*"})
+      oauth2_proxy_requests_in_flight => 2
+
+**The per-app gate sidecars are not scraped, and there is nothing to scrape.** Six apps run an
+`oauth2-proxy` container of their own — crafty, jackett, pihole, qbittorrent, radarr, sonarr —
+each with `--http-address=0.0.0.0:4180` and **no `--metrics-address`**, so oauth2-proxy's metrics
+listener never starts. They declare only a `http:4180` container port, no `metrics` port exists on
+their Services, and `kubectl get servicemonitors,podmonitors -A` selects none of them. Any alert
+written against `oauth2_proxy_requests_total{namespace="jackett"}` would be structurally inert.
+
+This matters because the gate's characteristic failure is invisible by construction: an
+unauthenticated **XHR** gets a `302` to Keycloak rather than a `401`, the browser follows it
+against a live SSO session with no prompt, and the call resolves **200 carrying an HTML page**
+where the app expected JSON. Nothing in the request path reports an error. On 2026-09-30 jackett
+spent a burst of requests in exactly that state, re-established a session, and lost it 26 seconds
+into a 5-minute session, while `jackett-down` sat at 1 replica available the whole time.
+
+`auth-visibility-alerts.yaml` therefore alerts on **the gap**, not on the incident:
+`OAuth2ProxyGateUnobserved` compares `kube_pod_container_info{container="oauth2-proxy"}` against
+`up{container="oauth2-proxy"}` — two already-scraped families — and names every namespace running
+a gate that nothing observes. It fires for six namespaces today and clears per-namespace as each
+one gains a scrape. `runbooks/oauth2-proxy-auth-failures.md` is the by-hand substitute meanwhile,
+and it reads the sidecar logs out of Loki (Alloy collects them; 72h retention).
+
+**To close the gap for an app**, three things are needed together, on that app's own manifest:
+`--metrics-address=0.0.0.0:44180` on the sidecar, a `metrics` containerPort plus the matching
+Service port, and a ServiceMonitor selecting it. Only then does the auth-churn rule — sustained
+`code="302"` as a share of `oauth2_proxy_requests_total`, held across a window longer than the
+session lifetime — become writable rather than decorative.
+
+### Update 2026-10-01: the listener exists; the scrape path does not
+
+PR #28 added `--metrics-address=0.0.0.0:44180` and a named `metrics` container port to the five
+fleet sidecars (jackett, radarr, sonarr, qbittorrent, pihole). `podmonitor.yaml` in this directory
+discovers them by the pod label `homelab.keithjackson.dev/auth-mode: forward-auth` — a **PodMonitor**
+rather than a ServiceMonitor because no Service exposes 44180 and adding one would mean changing the
+renderer.
+
+The endpoint is real: port-forwarded straight to the jackett pod, `/metrics` returns **http=200,
+10208 bytes**, exporting `oauth2_proxy_requests_in_flight`, `oauth2_proxy_requests_total` (with
+`code` labels) and `oauth2_proxy_response_duration_seconds`. That is verified by payload, not by the
+flag appearing in `args` — a distinction worth keeping, because the two claims came apart here.
+
+**The scrape is nevertheless dropped.** From a pod in ns `monitoring`, both `:44180` and `:4180` on
+the jackett pod time out; the shared proxy's `:44180` answers. The only difference is that ns
+`oauth2-proxy` has **no NetworkPolicy at all**, while each app namespace has a `default-deny` whose
+`policyTypes` include `Ingress` with no `ingress` rules (deny-all) plus a single CNP admitting
+`fromEntities: [ingress]` to port 4180 — the Gateway's Envoy and nothing else. Prometheus matches no
+clause.
+
+So the targets read `up == 0` and the TSDB is still empty for those namespaces. This is the same bug
+and the same fingerprint as `platform/longhorn/networkpolicy-metrics.yaml` documents for Longhorn,
+and the same narrow fix applies: ns `monitoring` → the gate pod label → TCP 44180 only. It cannot
+ship from this directory, because `oauth2-proxy` and `apps-wave-0` are **siblings** in the Flux graph
+(both `dependsOn: [platform-config]`), so a NetworkPolicy naming ns `jackett` would apply on today's
+cluster and fail on a cold rebuild. It belongs with each app's own rendered policy.
+
+Consequence for `auth-visibility-alerts.yaml`: its `unless` clause requires `up == 1`, not merely a
+present `up` series. A discovered-but-failing target publishes `up` with value 0, so the bare form
+would have cleared `OAuth2ProxyGateUnobserved` the moment the PodMonitor landed — reporting the gates
+as observed while nothing was being scraped. The unit test pins that scenario; deleting the `== 1`
+fails it and nothing else.

platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml

@@ -0,0 +1,177 @@
+---
+# promtool unit tests for OAuth2ProxyGateUnobserved (platform/oauth2-proxy/auth-visibility-alerts.yaml).
+#
+# WHY. The rule is an `unless` between two aggregations over DIFFERENT metric families, and the
+# failure modes of that shape are both silent: a label that does not survive `count by` makes the
+# right-hand side match nothing (the alert fires for the scraped namespace too, i.e. permanently),
+# and a right-hand side that over-matches makes it fire for nobody (i.e. inert). Neither is
+# visible by reading the expression, and neither is visible on a live cluster where the answer
+# happens to look plausible. So all four states are pinned: fires for an unscraped namespace,
+# stays quiet for a scraped one, honours `for: 1h`, and STOPS once a scrape appears.
+#
+# HOW TO RUN. The one repo-wide harness, by the companion convention:
+#   ./infra/devcontainer/dev.sh run bash -c 'platform/monitoring/tests/run.sh platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml'
+# promtool is pinned in infra/devcontainer/versions.env; never run it off the host PATH.
+#
+# THE SERIES BELOW ARE REAL SHAPES read off live prod Prometheus on 2026-09-30 before being
+# copied here — kube_pod_container_info as kube-state-metrics emits it (job="kube-state-metrics",
+# its own instance/pod labels, image/image_id/container_id) and `up` as the oauth2-proxy
+# ServiceMonitor emits it (endpoint="metrics", port 44180). The point of using the real shapes is
+# that `count by (namespace)` has to discard all of them; a test with tidy two-label series would
+# pass while the real expression collapsed differently.
+#
+# ASSERTIONS USE `ALERTS` RATHER THAN exp_alerts: the description annotation is long prose and
+# comparing it verbatim would couple this file to every wording change while proving nothing the
+# ALERTS series does not already prove.
+rule_files:
+  - auth-visibility-alerts.rules.yaml
+evaluation_interval: 1m
+
+tests:
+  # ---------------------------------------------------------------------------------------------
+  # 1. THE STATE PROD IS ACTUALLY IN (2026-09-30): jackett runs a gate sidecar with no scrape;
+  #    the shared oauth2-proxy namespace runs two that ARE scraped. Asserts the alert fires for
+  #    jackett and NOT for oauth2-proxy, and that `for: 1h` is honoured (pending at 59m).
+  - interval: 1m
+    name: an unscraped gate namespace fires after 1h; the scraped one never does
+    input_series:
+      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="jackett",pod="jackett-c5bfcbfb8-c75f9",service="kube-prometheus-stack-kube-state-metrics",uid="5b57d48f-e7d3-4050-9cd1-16aaaa052996"}'
+        values: '1x130'
+      # The shared instance: kube-state-metrics sees both replicas...
+      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://aa11bb22",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-cqz27",service="kube-prometheus-stack-kube-state-metrics",uid="1111aaaa-0000-0000-0000-000000000001"}'
+        values: '1x130'
+      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://cc33dd44",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-bscxg",service="kube-prometheus-stack-kube-state-metrics",uid="1111aaaa-0000-0000-0000-000000000002"}'
+        values: '1x130'
+      # ...and both replicas are genuinely scraped, so `up` carries container="oauth2-proxy" there.
+      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.1.189:44180",job="oauth2-proxy",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-cqz27",service="oauth2-proxy"}'
+        values: '1x130'
+      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.2.47:44180",job="oauth2-proxy",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-bscxg",service="oauth2-proxy"}'
+        values: '1x130'
+    promql_expr_test:
+      # Not yet firing at 59m — `for: 1h` has not elapsed. Guards against someone "simplifying"
+      # the hold away and turning every rollout into a page.
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing"}'
+        eval_time: 59m
+        exp_samples: []
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="pending",namespace="jackett"}'
+        eval_time: 59m
+        exp_samples:
+          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="pending",namespace="jackett",severity="warning"}'
+            value: 1
+      # Firing for jackett past the hold...
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing"}'
+        eval_time: 65m
+        exp_samples:
+          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett",severity="warning"}'
+            value: 1
+      # ...and never for the namespace that IS scraped. This is the assertion that catches an
+      # `unless` whose right-hand side stopped matching: without it, a broken rule that fires for
+      # every gate namespace would pass the test above.
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",namespace="oauth2-proxy"}'
+        eval_time: 65m
+        exp_samples: []
+      # The count must be 1 per namespace-with-a-gate, not 1 overall: a `count by (namespace)`
+      # that lost its `by` would collapse to a single unlabelled series.
+      - expr: 'count by (namespace) (kube_pod_container_info{container="oauth2-proxy"}) unless count by (namespace) (up{container="oauth2-proxy"})'
+        eval_time: 65m
+        exp_samples:
+          - labels: '{namespace="jackett"}'
+            value: 1
+
+  # ---------------------------------------------------------------------------------------------
+  # 2. THE FIX LANDING. jackett's sidecar gains --metrics-address and a ServiceMonitor 70m in, so
+  #    `up` starts carrying container="oauth2-proxy" for namespace="jackett" under a DIFFERENT job
+  #    name than the shared proxy's. The alert must stop. This is what makes the rule a gap
+  #    tracker rather than a permanent nag, and it pins that the match is on `container`, not on
+  #    a hardcoded job="oauth2-proxy" (which would keep firing forever after a correct fix).
+  - interval: 1m
+    name: the alert clears once the sidecar is actually scraped
+    input_series:
+      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="jackett",pod="jackett-c5bfcbfb8-c75f9",service="kube-prometheus-stack-kube-state-metrics",uid="5b57d48f-e7d3-4050-9cd1-16aaaa052996"}'
+        values: '1x130'
+      # Absent for the first 70 minutes, then present. `_x` is "no sample"; the scrape did not
+      # exist yet.
+      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.1.213:44180",job="jackett-auth",namespace="jackett",pod="jackett-c5bfcbfb8-c75f9",service="jackett-auth-metrics"}'
+        values: '_x70 1x60'
+    promql_expr_test:
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett"}'
+        eval_time: 65m
+        exp_samples:
+          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett",severity="warning"}'
+            value: 1
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",namespace="jackett"}'
+        eval_time: 80m
+        exp_samples: []
+
+  # ---------------------------------------------------------------------------------------------
+  # 3. THE REGRESSION GUARD FOR `== 1` — the scenario that exposed a real bug on 2026-10-01.
+  #
+  #    A target is DISCOVERED but every scrape FAILS: `up` exists with value 0. This is not
+  #    hypothetical — it is the exact state podmonitor.yaml creates today, because the app
+  #    namespaces' default-deny drops the scrape (measured; see podmonitor.yaml).
+  #
+  #    With the original `unless count by (namespace) (up{container="oauth2-proxy"})`, the `up`
+  #    SERIES EXISTS, so `count by (namespace)` returns 1, `unless` removes the namespace, and the
+  #    alert GOES QUIET while nothing is being scraped at all — a green signal standing in for an
+  #    absent one, which is the precise disease this whole rule was written to cure. Deleting the
+  #    `== 1` makes THIS test fail and every other test in this file still pass, which is the
+  #    property that makes it worth having.
+  - interval: 1m
+    name: a discovered-but-FAILING target does not count as observed (up==0 must still fire)
+    input_series:
+      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="jackett",pod="jackett-6877fcd9f5-ht9d9",service="kube-prometheus-stack-kube-state-metrics",uid="5b57d48f-e7d3-4050-9cd1-16aaaa052996"}'
+        values: '1x130'
+      # The PodMonitor target exists and never succeeds: NetworkPolicy drops the packets, so the
+      # scrape times out and `up` is published as 0 rather than going absent.
+      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.0.189:44180",job="oauth2-proxy-sidecars",namespace="jackett",pod="jackett-6877fcd9f5-ht9d9"}'
+        values: '0x130'
+    promql_expr_test:
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett"}'
+        eval_time: 65m
+        exp_samples:
+          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett",severity="warning"}'
+            value: 1
+      # And the inner clause itself yields NOTHING while the scrape is failing — which is the
+      # mechanism the alert depends on, asserted directly so a future reader can see why the
+      # `== 1` matters without re-deriving it. (Scenario 4 below covers the other direction:
+      # a succeeding scrape really does clear the alert, so `== 1` has not made the rule
+      # impossible to satisfy.)
+      - expr: 'count by (namespace) (up{container="oauth2-proxy"} == 1)'
+        eval_time: 65m
+        exp_samples: []
+
+  # ---------------------------------------------------------------------------------------------
+  # 4. A FLAPPING target: up goes 1 -> 0 and stays there. The alert must come BACK, not latch
+  #    cleared. Covers the ordering bug where a once-observed namespace is permanently excluded.
+  - interval: 1m
+    name: a target that starts working and then fails re-fires
+    input_series:
+      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="radarr",pod="radarr-7d55cbffb9-42hmh",service="kube-prometheus-stack-kube-state-metrics",uid="3333cccc-0000-0000-0000-000000000001"}'
+        values: '1x250'
+      # Healthy for the first 60m, then every scrape fails for the rest.
+      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.1.77:44180",job="oauth2-proxy-sidecars",namespace="radarr",pod="radarr-7d55cbffb9-42hmh"}'
+        values: '1x60 0x180'
+    promql_expr_test:
+      # Quiet while it was working.
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",namespace="radarr"}'
+        eval_time: 55m
+        exp_samples: []
+      # Firing again an hour after it broke.
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="radarr"}'
+        eval_time: 125m
+        exp_samples:
+          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="radarr",severity="warning"}'
+            value: 1
+
+  # ---------------------------------------------------------------------------------------------
+  # 5. NO GATE, NO ALERT. A namespace with no oauth2-proxy container must never appear, or the
+  #    rule would page for every app on the platform the day someone widened the matcher.
+  - interval: 1m
+    name: namespaces without a gate sidecar are never reported
+    input_series:
+      - series: 'kube_pod_container_info{container="plex",container_id="containerd://feed0001",endpoint="http",image="ghcr.io/example/plex:1",image_id="ghcr.io/example/plex@sha256:feed0001",image_spec="ghcr.io/example/plex:1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="plex",pod="plex-0",service="kube-prometheus-stack-kube-state-metrics",uid="2222bbbb-0000-0000-0000-000000000001"}'
+        values: '1x130'
+    promql_expr_test:
+      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved"}'
+        eval_time: 65m
+        exp_samples: []

platform/oauth2-proxy/auth-visibility-alerts.yaml

@@ -0,0 +1,137 @@
+---
+# oauth2-proxy AUTH-VISIBILITY alerting (R-OBS-04).
+#
+# ============================================================================================
+# WHY THIS FILE EXISTS — THE 2026-09-30 JACKETT FAILURE THAT NOTHING OBSERVED
+# ============================================================================================
+# Jackett's UI stopped working. Reloading never fixed it and the user was never re-prompted to
+# log in. The sidecar's own access log had the whole story:
+#
+#   [oauthproxy.go:1026] No valid authentication in request. Initiating login.
+#   GET "/api/v2.0/indexers/yts/config?_=..."          HTTP/1.1 ... 302 400
+#   GET "/api/v2.0/indexers/thepiratebay/config?_=..." HTTP/1.1 ... 302 409
+#   ... ten of these at 15:30:53, then a fresh [AuthSuccess] at 15:31:06,
+#   then "No valid authentication in request" again 26 SECONDS LATER, inside the same session.
+#
+# NOTHING ANYWHERE ELSE RECORDED IT. oauth2-proxy answers an unauthenticated XHR with a 302 to
+# Keycloak rather than a 401; the browser follows it silently, the live SSO session satisfies it
+# without a prompt, and the XHR resolves 200 carrying a 44 KB HTML dashboard where the app's
+# JavaScript wanted JSON. From the browser, from the user, and from any uptime check, the service
+# is healthy. `jackett-down` (apps/wave-0/jackett.yaml) watches
+# kube_deployment_status_replicas_available, which was 1 throughout and would have been 1 if the
+# gate had rejected every request ever made.
+#
+# ============================================================================================
+# WHY THIS FILE DOES NOT CONTAIN THE OBVIOUS ALERT — MEASURED, prod, 2026-09-30
+# ============================================================================================
+# The alert you would want is "a gated app is serving a sustained rate of auth redirects to a
+# user who is already authenticated". It is NOT WRITABLE against this cluster's TSDB, and the
+# reason is this file's real subject:
+#
+#   THE PER-APP GATE SIDECARS EXPORT NO METRICS AT ALL.
+#
+#   $ promtool query instant :9090 'count by (__name__) ({__name__=~"oauth2_proxy.*"})'
+#     oauth2_proxy_requests_in_flight => 2          <- the SHARED proxy's two replicas. That is ALL.
+#
+#   PARTLY SUPERSEDED 2026-10-01, and the correction matters more than the original. PR #28 gave
+#   the five FLEET sidecars `--metrics-address=0.0.0.0:44180` and a named `metrics` port, and the
+#   endpoint genuinely serves (verified by payload: http=200, 10208 bytes, all three
+#   `oauth2_proxy_*` families present — not merely by the flag being in `args`). podmonitor.yaml
+#   in this directory now discovers them. But the scrape is still DROPPED by the app namespaces'
+#   default-deny ingress, so the TSDB remains empty for those namespaces; and crafty still has no
+#   metrics listener at all. The rule below is unchanged in purpose and still has work to do.
+#
+#   Every sidecar (crafty, jackett, pihole, qbittorrent, radarr, sonarr) ran with
+#   `--http-address=0.0.0.0:4180` and NO `--metrics-address`, so oauth2-proxy's metrics listener
+#   was never started; the container declared only port `http:4180`, there was no `metrics` port on
+#   any of their Services, and there was no ServiceMonitor or PodMonitor for any of them anywhere
+#   in the cluster (measured: `kubectl get servicemonitors,podmonitors -A`). Six gates, zero
+#   telemetry. An alert written against `oauth2_proxy_requests_total{namespace="jackett"}` would
+#   have been PERMANENTLY INERT — present, evaluating, silent, and indistinguishable from health.
+#   This repo has shipped that mistake before (VeleroNoRecentBackup, the Longhorn rules), which is
+#   why `conformance/tests/l0-static/alert-metrics-have-scrape-targets` exists, and why the
+#   temptation was refused here rather than ratcheted into a register.
+#
+#   The SHARED proxy IS scraped (job="oauth2-proxy", 2 targets, up=1) but it is not in the data
+#   path for any sidecar-gated app — each app gates itself — and its current pods export no
+#   `oauth2_proxy_requests_total` at all, having proxied nothing since they started. An alert on
+#   the shared instance would not have fired on this incident either. Saying so is the point.
+#
+# So this file alerts on the CONDITION THAT MADE THE INCIDENT INVISIBLE, not on the incident.
+# That condition is measurable today, from two already-scraped metric families, and it is the
+# precondition for ever being able to alert on the incident itself. It self-clears the moment a
+# sidecar gains a metrics endpoint and a scrape — at which point the auth-churn rule becomes
+# writable and belongs here beside it. See runbooks/oauth2-proxy-auth-failures.md, which is the
+# by-hand equivalent an operator uses in the meantime.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: oauth2-proxy-auth-visibility
+  namespace: oauth2-proxy
+  labels:
+    release: kube-prometheus-stack     # kube-prometheus-stack's ruleSelector
+    app.kubernetes.io/name: oauth2-proxy
+spec:
+  groups:
+    - name: oauth2-proxy.auth-visibility
+      rules:
+        - alert: OAuth2ProxyGateUnobserved
+          # A namespace runs an oauth2-proxy gate container, and Prometheus holds no scrape
+          # target for that container in that namespace.
+          #
+          # BOTH SIDES ARE ALREADY-SCRAPED FAMILIES, deliberately:
+          #   * kube_pod_container_info  — kube-state-metrics, the declared owner in
+          #     alert-metrics-have-scrape-targets. It sees the container whether or not the
+          #     container exports anything, which is exactly the property needed here.
+          #   * up                       — UNIVERSAL. `up` carries the `container` label for any
+          #     ServiceMonitor/PodMonitor target, so this works for a future per-app scrape
+          #     without knowing what job name it will pick.
+          # Nothing in this expression touches `oauth2_proxy_*`, so the rule cannot itself be
+          # inert for the reason it is reporting.
+          #
+          # MEASURED ON PROD 2026-09-30 — this expression returned six namespaces:
+          #   {namespace="crafty"} {namespace="jackett"} {namespace="pihole"}
+          #   {namespace="qbittorrent"} {namespace="radarr"} {namespace="sonarr"}
+          # and excluded namespace="oauth2-proxy", whose two replicas ARE scraped. It fires on
+          # the state the cluster is in; it is not a rule waiting for a hypothetical.
+          #
+          # `== 1` IS LOAD-BEARING, AND IT IS A BUG FIX (2026-10-01). This clause was originally
+          # a bare `up{container="oauth2-proxy"}`, which was WRONG in the one way that matters
+          # here: a target that is DISCOVERED BUT FAILING still publishes `up`, with value 0. So
+          # the series exists, `count by (namespace)` returns 1, `unless` removes the namespace,
+          # and the alert GOES QUIET while nothing whatsoever is being scraped. That is this
+          # branch's own disease — a green signal standing in for an absent one — reproduced
+          # inside the alert written to cure it. It was not hypothetical: podmonitor.yaml lands
+          # five targets whose scrape is DROPPED by NetworkPolicy (measured; see that file), so
+          # the bare form would have silently cleared on the very next commit. `== 1` means
+          # "observed" requires a scrape that actually succeeds.
+          #
+          # A failing target is therefore reported by BOTH this rule and kube-prometheus-stack's
+          # generic TargetDown. That duplication is deliberate: TargetDown says "a target is
+          # down", this says "an authentication gate is unobserved", and during an incident those
+          # are different sentences. The overlap is the cheap half of the trade.
+          expr: |
+            count by (namespace) (kube_pod_container_info{container="oauth2-proxy"})
+              unless
+            count by (namespace) (up{container="oauth2-proxy"} == 1)
+          # 1h, not 5m: a rollout briefly has a running sidecar whose target has not been
+          # discovered yet, and this condition is a standing architectural gap, never an
+          # emergency. Long enough that no deploy can produce it.
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            summary: "oauth2-proxy gate in {{ $labels.namespace }} exports no metrics"
+            description: >-
+              The {{ $labels.namespace }} namespace runs an oauth2-proxy gate container and
+              Prometheus has NO scrape target for it, so every authentication decision that gate
+              makes — every redirect to Keycloak, every 403 on a group check, every session that
+              silently fails to hold — is invisible to metrics and to alerting. This is not a
+              theoretical gap: on 2026-09-30 the jackett gate answered a burst of the app's own
+              XHR calls with 302s to Keycloak, re-established a session, and lost it again 26
+              seconds later, and the only record anywhere was a line in the sidecar's stdout.
+              No alert could have fired, because there is nothing to alert on. Until a scrape
+              exists the runbook's LogQL queries are the substitute; the fix is
+              `--metrics-address=0.0.0.0:44180` on the sidecar plus a `metrics` port and a
+              ServiceMonitor, after which the auth-churn rule becomes writable.
+            runbook_url: "https://${BASE_DOMAIN}/runbooks/oauth2-proxy-auth-failures"

platform/oauth2-proxy/kustomization.yaml

@@ -14,3 +14,12 @@ resources:
   - oauth2-proxy-credentials.sops.yaml
   - httproute.yaml       # forward-auth sign-in/callback ingress (was referenced in-place by platform-config)
   - servicemonitor.yaml  # Prometheus scrape (was referenced in-place by platform-config)
+  # Alerting on whether the gates are OBSERVABLE at all. Travels with the servicemonitor above
+  # for the same reason it does: monitoring.coreos.com CRDs are registered by the `platform`
+  # tier, which is Ready before this tier's platform-config dependency completes.
+  - auth-visibility-alerts.yaml
+  # Discovers the PER-APP gate sidecars' :44180 metrics endpoints by pod label (PR #28 added the
+  # listener; no Service exposes it, so this is a PodMonitor, not a ServiceMonitor). The scrape is
+  # still denied by the app namespaces' default-deny ingress — read podmonitor.yaml's header
+  # before concluding the targets should be up.
+  - podmonitor.yaml

platform/oauth2-proxy/podmonitor.yaml

@@ -0,0 +1,109 @@
+---
+# PodMonitor for the PER-APP oauth2-proxy gate sidecars (R-OBS-04).
+#
+# ============================================================================================
+# WHY A PodMonitor AND NOT A ServiceMonitor
+# ============================================================================================
+# PR #28 gave all five fleet sidecars `--metrics-address=0.0.0.0:44180` and a named `metrics`
+# container port. It did NOT add a Service port, and measured on prod 2026-10-01 none of the five
+# Services exposes 44180:
+#
+#   jackett      jackett:     [('port-9117', 9117), ('authproxy', 4180)]
+#   radarr       radarr:      [('port-7878', 7878), ('authproxy', 4180)]
+#   sonarr       sonarr:      [('port-8989', 8989), ('authproxy', 4180)]
+#   qbittorrent  qbittorrent: [('port-8080', 8080), ('authproxy', 4180)]
+#   pihole       pihole:      [('port-80',     80), ('authproxy', 4180)]
+#
+# A ServiceMonitor selects Service PORTS, so it would have nothing to select. A PodMonitor targets
+# pod ports directly and bypasses Services entirely — so this needs no change to the Services, and
+# therefore no change to `catalog/render.py` or the `apps/wave-*` renders and goldens.
+#
+# SELECTOR: the `homelab.keithjackson.dev/auth-mode: forward-auth` POD LABEL, not a namespace list.
+# Measured 2026-10-01: that label is present on exactly the five fleet gate pods (jackett, radarr,
+# sonarr, qbittorrent, pihole) and absent from crafty's, which is a separately-managed deployment
+# (platform/crafty) that still has no `--metrics-address` at all. So the selector describes the
+# PROPERTY being scraped rather than enumerating today's apps, and a sixth forward-auth app is
+# picked up with no edit here. `namespaceSelector.any: true` because those five namespaces are
+# created by the apps-wave-0/1 tiers, which this tier does not and must not depend on.
+#
+# ============================================================================================
+# READ THIS BEFORE BELIEVING THE TARGETS ARE UP: THE SCRAPE IS CURRENTLY DENIED BY NETWORKPOLICY
+# ============================================================================================
+# This PodMonitor is NECESSARY AND NOT SUFFICIENT. The endpoint serves, and the path from ns
+# monitoring is blocked. Both halves measured on prod 2026-10-01, read-only:
+#
+#   THE ENDPOINT IS HEALTHY — port-forward straight to the jackett pod's 44180:
+#     GET /metrics -> http=200 bytes=10208
+#     # TYPE oauth2_proxy_requests_in_flight gauge
+#     # TYPE oauth2_proxy_requests_total counter
+#     # TYPE oauth2_proxy_response_duration_seconds histogram
+#     oauth2_proxy_requests_total{code="200"} 3
+#     oauth2_proxy_requests_total{code="302"} 2
+#
+#   THE SCRAPE PATH IS DROPPED — from a pod in ns monitoring (grafana, which sits beside
+#   Prometheus and so occupies the same policy position) to the jackett pod IP:
+#     http://10.0.0.189:44180/metrics   -> wget: download timed out
+#     http://10.0.0.189:4180/ping       -> wget: download timed out     (even the "allowed" port)
+#     http://<shared-proxy>:44180/metrics -> 200, oauth2_proxy_* present (control: ns oauth2-proxy
+#                                            has NO NetworkPolicy, which is the only reason the
+#                                            shared instance has ever been scrapable)
+#
+#   WHY: ns jackett's `default-deny` NetworkPolicy is `podSelector: {}` with
+#   `policyTypes: [Ingress, Egress]` and an `egress` rule ONLY — no `ingress` key, which in
+#   NetworkPolicy semantics is deny-all-ingress. The only ingress allowance is the CNP
+#   `jackett-allow-gateway-ingress`: `fromEntities: [ingress]` to port 4180 — the Gateway's Envoy
+#   and nothing else. Prometheus matches no clause and its packets are dropped, not refused.
+#
+# SO THE TARGETS BELOW WILL READ `up == 0` UNTIL AN INGRESS ALLOWANCE LANDS. That is a known,
+# measured, documented state — NOT a regression introduced here, and NOT something to silence.
+# It is deliberately left visible: `OAuth2ProxyGateUnobserved` in auth-visibility-alerts.yaml
+# requires `up == 1`, precisely so a discovered-but-failing target does not read as "observed"
+# (see that file's header — this exact trap is why the `== 1` is there).
+#
+# THIS IS THE SAME BUG LONGHORN HAD, and the platform has a documented fix idiom for it:
+# platform/longhorn/networkpolicy-metrics.yaml, whose header records the identical fingerprint
+# measured on 2026-09-13 — ServiceMonitor in place, 3 targets DISCOVERED, `up` 0/0/0, TSDB empty,
+# `lastError: context deadline exceeded` (a drop, not a refusal). The fix there is a narrow
+# additive `networking.k8s.io/v1` NetworkPolicy: ns monitoring -> the pod label, that one port.
+#
+# WHY THAT POLICY IS NOT IN THIS DIRECTORY. It would have to be five objects, one per app
+# namespace, and those namespaces are created by apps-wave-0/1. Measured Flux graph, 2026-10-01:
+#   oauth2-proxy  dependsOn=['platform-config']        <- this tier
+#   apps-wave-0   dependsOn=['platform-config']        <- SIBLING, no ordering between them
+#   apps-wave-1   dependsOn=['apps-wave-0']
+# They are siblings, so on a cold bootstrap there is no guarantee ns jackett exists when this tier
+# applies, and a NetworkPolicy naming a missing namespace fails to apply. It would work on today's
+# running cluster and break on a rebuild — the "works now, broken on restore" shape this repo
+# keeps paying for. Longhorn's precedent is safe only because longhorn-system is created by the
+# same tier that ships its policy. The allowance therefore belongs WITH THE APPS, rendered next to
+# the rest of each app's network policy — i.e. in the catalog/renderer, which is in flight on
+# another branch. Tracked in the handoff; it is one narrow rule per app, not a redesign.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: oauth2-proxy-sidecars
+  namespace: oauth2-proxy
+  labels:
+    release: kube-prometheus-stack     # kube-prometheus-stack's podMonitorSelector
+    app.kubernetes.io/name: oauth2-proxy
+    homelab.keithjackson.dev/purpose: metrics-scrape
+spec:
+  namespaceSelector:
+    # The gate sidecars live in the app namespaces, not this one. `any` rather than a fixed list so
+    # a new forward-auth app needs no edit here — the pod label below is what scopes this.
+    any: true
+  selector:
+    matchLabels:
+      # The PROPERTY, not the app list. Present on exactly the five fleet gate pods (measured
+      # 2026-10-01); absent from crafty, which has no metrics listener yet.
+      homelab.keithjackson.dev/auth-mode: forward-auth
+  podMetricsEndpoints:
+    # By PORT NAME (`metrics`, 44180), never by number: the name is what PR #28 declared on the
+    # container, and a numeric port here would silently stop matching if the port were renumbered.
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      # 10s, matching the drop signature above: a DENIED path does not refuse, it hangs until the
+      # scrape times out. Leaving the default would make each failing target hold a scrape slot for
+      # the full interval.
+      scrapeTimeout: 10s

runbooks/README.md

@@ -51,6 +51,36 @@ rule groups still evaluate against nothing and **cannot fire**:
 | Flux (`gotk_*`) | `FluxReconciliationFailure`, `FluxResourceSuspended`, `ImageUpdateAutomationFailing`, `ImagePolicyStalled`, `UpdateHealthGateFailed` |
 | Hubble (`hubble_drop_total`) | `HubbleDeniedFlowSpike` |
 | etcd (`kubeEtcd` disabled for k3s, deliberately) | no etcd fsync alerting exists at all |
+| per-app oauth2-proxy sidecars (`oauth2_proxy_*`) — updated 2026-10-01 | no authentication alerting exists at all for crafty, jackett, pihole, qbittorrent, radarr, sonarr |
+
+The oauth2-proxy row is the newest, and it has **moved twice in two days** — worth reading in order,
+because each step looked like the end of it.
+
+**2026-09-30 — nothing to scrape.** `count by (__name__) ({__name__=~"oauth2_proxy.*"})` returned
+exactly one family, `oauth2_proxy_requests_in_flight`, from the two replicas of the **shared**
+forward-auth instance. All six per-app gate sidecars ran without `--metrics-address`, so their metrics
+listener never started, and no ServiceMonitor or PodMonitor selected them.
+
+**2026-10-01 — the listener exists, and the scrape is still blocked.** PR #28 added
+`--metrics-address=0.0.0.0:44180` and a named `metrics` port to the five fleet sidecars, and the
+endpoint genuinely serves (port-forwarded: `http=200`, 10208 bytes, all three `oauth2_proxy_*`
+families, `oauth2_proxy_requests_total{code="302"}` present). `platform/oauth2-proxy/podmonitor.yaml`
+now discovers them by pod label. **But the app namespaces' `default-deny` NetworkPolicy drops the
+scrape**: from a pod in ns `monitoring`, both `:44180` and `:4180` on the jackett pod time out, while
+the shared proxy's `:44180` answers — the only difference being that ns `oauth2-proxy` has no
+NetworkPolicy at all. So the targets are discovered and `up == 0`, and the TSDB is still empty for
+those namespaces. **This is the same bug Longhorn had**, with the same fingerprint; the fix idiom is
+`platform/longhorn/networkpolicy-metrics.yaml` (ns monitoring → one pod label → one port), and for
+these apps it has to be rendered alongside each app's own policy rather than shipped from the
+oauth2-proxy tier, which is a *sibling* of `apps-wave-0` in the Flux graph and so cannot rely on those
+namespaces existing yet.
+
+`OAuth2ProxyGateUnobserved` (`platform/oauth2-proxy/auth-visibility-alerts.yaml`) alerts on the gap
+itself from `kube_pod_container_info` and `up` — two families that *are* scraped. Note its `up == 1`:
+a discovered-but-failing target publishes `up` with value **0**, so the earlier bare form would have
+gone quiet the moment the PodMonitor landed, while nothing was being scraped. That near-miss is the
+whole reason to distrust a cleared alert here. Crafty remains outside all of this — no metrics
+listener at all. See `oauth2-proxy-auth-failures`.
 
 Each affected runbook says so on its own page and gives the by-hand equivalent. See `targets-down`.
 
@@ -108,6 +138,14 @@ Each affected runbook says so on its own page and gives the by-hand equivalent.
 - **`keycloak-db`** — Keycloak's CloudNativePG database is unavailable.
 - **`keycloak-client-secrets`** — the client/URL reconciler failing or silent. The 2026-09-06 lockout that
   every green signal missed.
+- **`oauth2-proxy-auth-failures`** — a gated app "is up but does not work". Linked from
+  `OAuth2ProxyGateUnobserved`. The gate answers an unauthenticated **XHR** with a 302 to Keycloak
+  rather than a 401, the browser follows it silently against a live SSO session, and the call
+  resolves **200 carrying HTML** where the app's JavaScript wanted JSON — so the user is never
+  re-prompted and every uptime check stays green. Tells the gate's refusal from the group check
+  from the app's own login, from the sidecar log and from Loki, with the 2026-09-30 jackett trace
+  as the worked example. **Read the caveat**: the per-app gate sidecars export no metrics at all,
+  so these logs (72h of them) are the only record this class of failure leaves.
 
 ### Operator procedures (no alert links here)
 - **`plex-claim`** — claiming the Plex Media Server. **Claimed 2026-09-29**; the page now only
@@ -115,6 +153,22 @@ Each affected runbook says so on its own page and gives the by-hand equivalent.
   plex.tv/claim — so it is a timed sequence, scripted as `infra/plex/claim-ship.sh`:
   token → `sops` → PR → gate → merge → **forced** `flux reconcile` → pod roll (~2m45s measured).
   Includes why a changed Secret does not restart anything, and why an expired token is harmless.
+- **`plex-quicksync`** — hardware transcoding on the host iGPU. GitOps puts `/dev/dri` in the pod
+  (`hardware.gpu: intel-quicksync`); **a human still has to tick Settings → Transcoder → "Use hardware
+  acceleration when available"**, and no environment variable can do it (the image supports no
+  `PLEX_PREFERENCE*`, verified). The page names the real preference key (`HardwareAcceleratedCodecs`,
+  read out of the binary), says which widely-cited key **does not exist**, and is explicit that the
+  **only** proof is `Transcode (hw)` on the Plex dashboard — Plex renders no ServiceMonitor, so no
+  alert could ever fire. Also: the device-cgroup failure to expect first and its device-plugin
+  fallback, and why a `:latest` pull can silently break it.
+- **`oauth2-proxy-session`** — a gated UI loads but its saves quietly do nothing, and you are never
+  re-prompted to log in. Tells the three stacked auth layers apart from the sidecar's own access log
+  (the four lines that matter, including the upstream `-` that means the proxy answered a request the
+  app never saw). Covers the cookie collision between per-app sidecars sharing the default
+  `_oauth2_proxy` name on `.home.keithjackson.dev` — which deletes sessions belonging to apps in
+  *other* namespaces and is only legible with two logs side by side — the 5-minute access-token
+  session with no `--cookie-refresh`, and why an XHR answered with a 302 instead of a 401 loses a
+  POST body silently.
 - **`crafty-admin-credential`** — you signed into Keycloak and Crafty *still* asks for a password.
   Expected: Crafty 4 has no OIDC, so its own local login sits behind the oauth2-proxy gate as a second
   factor. The page tells the two layers apart from the sidecar log, and explains that the bootstrap

runbooks/kustomization.yaml

@@ -85,11 +85,29 @@ configMapGenerator:
       - node-io=node-io.md
       - node-memory=node-memory.md
       - node-memory-swap=node-memory-swap.md
+      # Linked from OAuth2ProxyGateUnobserved, and the diagnostic procedure for any gated app
+      # that "is up but does not work" — the gate answers an unauthenticated XHR with a 302,
+      # not a 401, so the failure arrives as an HTTP 200 carrying the wrong body.
+      - oauth2-proxy-auth-failures=oauth2-proxy-auth-failures.md
+      # Operator procedure, not alert-linked. Reading the per-app oauth2-proxy sidecar log to tell
+      # Keycloak, the sidecar and the app's own login apart. Cookie collision between two sidecars
+      # presents as random session loss in ONE app's log and is only legible with both side by side,
+      # so the page has to be reachable at 3am without going looking for it in git.
+      # SIBLING PAGE to oauth2-proxy-auth-failures above, and they are deliberately both here: this
+      # one is the SESSION-LIFECYCLE procedure (why a session died), that one is the DIAGNOSTIC
+      # TRIAGE (which of the three layers refused, and how to tell a 302-to-200 from a real success).
+      - oauth2-proxy-session=oauth2-proxy-session.md
       # Operator procedure, not an alert page (no alert links to it — the
       # alert-runbooks-resolvable check treats that as advisory). It is served
       # because the plex.tv claim window is short and is not a thing to go looking
       # for in git: /runbooks/plex-claim has to resolve the moment it is needed.
       - plex-claim=plex-claim.md
+      # Operator procedure, not an alert page, and NO alert is possible for it: plex renders no
+      # metricsEndpoint, so there is no ServiceMonitor, no transcode-mode metric and nothing a rule
+      # could evaluate. Served because the feature it completes is INERT until a human ticks a box in
+      # the Plex UI — the one step GitOps cannot reach — and a page nobody can find is how a shipped
+      # feature stays switched off for months.
+      - plex-quicksync=plex-quicksync.md
       - pvc-capacity=pvc-capacity.md
       - targets-down=targets-down.md
       - update-rollback=update-rollback.md

runbooks/oauth2-proxy-auth-failures.md

@@ -0,0 +1,244 @@
+# oauth2-proxy auth failures — telling the three layers apart
+
+Linked from `OAuth2ProxyGateUnobserved`, and the page to open whenever a gated app "is up but
+does not work": a UI that loads and then shows nothing, a page that never finishes, a list that
+stays empty, a save that silently does not save. The alert is about the *visibility* gap; this
+page is the diagnostic procedure, and it works whether or not the alert ever fired.
+
+## What this means here
+
+Every app behind Keycloak on this platform has **three** places a request can be refused, and
+from a browser they look identical — because two of the three answer with **HTTP 200**.
+
+| Layer | What refuses | What the browser sees |
+|---|---|---|
+| 1. The gate (oauth2-proxy sidecar) | no valid session cookie | **302 to Keycloak**, then usually **200** |
+| 2. The gate's group check | session valid, wrong group | 403, or a redirect loop |
+| 3. The app itself | the app's own login / API key | the app's own 401 or login page |
+
+**Layer 1 is the one that hides.** oauth2-proxy answers an unauthenticated request with a
+redirect, not a 401 — correct for a browser navigation, disastrous for an **XHR**. The app's
+JavaScript asks for JSON, the gate answers `302`, the browser follows it silently, the live
+Keycloak SSO session satisfies it with no prompt, and the XHR resolves **200 with an HTML
+dashboard in the body**. `fetch()` sees a success. The user sees an empty page. Reloading does
+nothing, and nobody is ever asked to log in, because from the browser's point of view nothing
+failed.
+
+The worked example, jackett on 2026-09-30, straight from the sidecar log:
+
+    [oauthproxy.go:1026] No valid authentication in request. Initiating login.
+    GET "/api/v2.0/indexers/yts/config?_=..."          HTTP/1.1 ... 302 400
+    GET "/api/v2.0/indexers/thepiratebay/config?_=..." HTTP/1.1 ... 302 409
+    ... ten of these at 15:30:53
+    [AuthSuccess] Authenticated via OAuth2: Session{email:... created:2026-09-30 15:31:06
+                  expires:2026-09-30 15:36:06 refresh_token:true groups:[platform-admins]}
+    GET "/UI/Dashboard"        200 44379
+    GET "/api/v2.0/indexers?_=..." 200 1501229
+    ... and 26 seconds later, inside that same 5-minute session:
+    [oauthproxy.go:1026] No valid authentication in request. Initiating login.
+
+A session created at 15:31:06, valid until 15:36:06, not honoured at 15:31:32. That is the
+signature of the session being **re-established over and over instead of held** — a cookie that
+another proxy is overwriting, or a cookie that never comes back at all — not of Keycloak being
+down and not of the app being broken.
+
+**Read this before you trust any silence (updated 2026-10-01).** The five fleet gate sidecars now
+*do* run a metrics listener on `:44180`, and `platform/oauth2-proxy/podmonitor.yaml` discovers
+them — but **the scrape is dropped by the app namespaces' `default-deny` NetworkPolicy**, so those
+targets read `up == 0` and the TSDB holds nothing for them. Crafty has no listener at all. Only
+the shared `oauth2-proxy` namespace instance is genuinely scraped, and it is not in the data path
+for any of these apps. So there is still nothing to graph and nothing that could have alerted.
+**The logs below remain the only record that this class of failure happened.** Check which state
+you are in before you trust a quiet dashboard — "First commands" step 1 tells you in one query.
+
+## First commands
+
+`kubectl` does not exist on the workstation; everything runs in the pinned container, and always
+with an explicit context.
+
+**1. Which namespaces have an unobserved gate** (this is what the alert measures):
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n monitoring exec prometheus-kube-prometheus-stack-prometheus-0 \
+     -c prometheus -- /bin/promtool query instant http://localhost:9090 \
+     "count by (namespace) (kube_pod_container_info{container=\"oauth2-proxy\"}) unless count by (namespace) (up{container=\"oauth2-proxy\"} == 1)"'
+```
+
+**1b. If a namespace is listed, is it "no target" or "target failing"?** The two need different
+fixes and the alert deliberately does not distinguish them:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n monitoring exec prometheus-kube-prometheus-stack-prometheus-0 \
+     -c prometheus -- /bin/promtool query instant http://localhost:9090 \
+     "up{container=\"oauth2-proxy\"}"'
+```
+
+No series for the namespace → nothing discovers it (no PodMonitor match; check the pod carries
+`homelab.keithjackson.dev/auth-mode=forward-auth` and a named `metrics` port). A series with
+value **0** → discovered but every scrape fails; get the reason from the target's `lastError`:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n monitoring port-forward svc/kube-prometheus-stack-prometheus 19090:9090 & sleep 5; \
+   curl -s localhost:19090/api/v1/targets | python3 -c "
+import json,sys
+for t in json.load(sys.stdin)[\"data\"][\"activeTargets\"]:
+    if t[\"labels\"].get(\"container\")==\"oauth2-proxy\":
+        print(t[\"labels\"].get(\"namespace\"), t[\"health\"], t.get(\"lastError\",\"\")[:90])"'
+```
+
+`context deadline exceeded` is a **drop**, not a refusal — that is NetworkPolicy, not a dead
+endpoint. Confirm the endpoint itself is fine by port-forwarding straight to the pod, which
+bypasses policy entirely:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n jackett port-forward \
+     $(kubectl --context k3s-prod -n jackett get pod -l app.kubernetes.io/name=jackett -o name | head -1) \
+     14418:44180 & sleep 5; curl -s -o /dev/null -w "http=%{http_code} bytes=%{size_download}\n" \
+     http://127.0.0.1:14418/metrics'
+```
+
+`http=200` with a few KB means the listener is healthy and the path is what is broken. As of
+2026-10-01 that is the state of all five fleet apps, and the fix is an ingress allowance from ns
+`monitoring` to port 44180 — see `platform/longhorn/networkpolicy-metrics.yaml` for the idiom.
+
+**2. The sidecar log — but you must grep it.** The gate's stdout is ~11,500 lines a day per app
+and almost all of it is the kubelet health probe (`GET "/ping" ... 200`). A plain
+`kubectl logs --tail=100` shows you nothing but `/ping` and looks perfectly healthy. Filter:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n jackett logs deploy/jackett -c oauth2-proxy --since=2h \
+     | grep -v "/ping" | grep -E "No valid authentication|AuthSuccess|403|401|callback"'
+```
+
+Substitute the namespace: `crafty`, `jackett`, `pihole`, `qbittorrent`, `radarr`, `sonarr`.
+
+**3. The same question across every gate at once, from Loki.** Alloy collects all of these, so
+you can ask about the whole fleet and about a window that has already scrolled out of
+`kubectl logs`. Port-forward Loki and query it:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n monitoring port-forward svc/loki 13100:3100 & sleep 5; \
+   curl -s -G --data-urlencode "query=sum by (namespace) (count_over_time({container=\"oauth2-proxy\"} |= \"No valid authentication in request\" [24h]))" \
+     http://127.0.0.1:13100/loki/api/v1/query | python3 -m json.tool | grep -E "namespace|value" '
+```
+
+and the denominator, which is what makes the number mean something:
+
+```
+   ... --data-urlencode "query=sum by (namespace) (count_over_time({container=\"oauth2-proxy\"} |= \"AuthSuccess\" [24h]))" ...
+```
+
+**How to read the ratio.** A healthy sign-in is roughly **one or two** `No valid authentication`
+lines per `AuthSuccess` — the first request of a session is unauthenticated by definition. A
+ratio in the tens means sessions are not surviving. Measured on prod 2026-09-30 (24h):
+
+| namespace | `No valid authentication` | `AuthSuccess` | ratio |
+|---|---|---|---|
+| crafty | 81 | 1 | 81:1 |
+| radarr | 67 | 1 | 67:1 |
+| jackett | 39 | 4 | ~10:1 |
+| pihole, qbittorrent, sonarr | 0 | 0 | (unused in the window) |
+
+All three of those are the failure, not the baseline.
+
+**The cleanest worked case of all — radarr, measured 2026-10-01.** After the cookie fix landed,
+jackett dropped to **0** `No valid authentication` lines in 13,612 (from 39), and `removing
+session` / `cookie signature not valid` went to **0 on both apps** (radarr had 8 and 8 before).
+The eviction fingerprint is gone. But radarr still logged **206** rejections — and sampling 209 of
+them, **every single one was the same request**:
+
+    GET "/signalr/messages/negotiate" ... 302    <- browser UA, over and over
+
+That is one left-open browser tab. Its cookie is gone, its SignalR socket keeps trying to
+reconnect, and every attempt is silently 302'd to Keycloak. Nobody is at the keyboard, so nobody
+is ever re-prompted; the tab just sits there looking like a working page that has stopped
+updating. **This is the entire failure class in one line**: a non-navigation request answered with
+a redirect, forever, with no error surfaced anywhere.
+
+So when you read a rejection count, **always group by request path before concluding anything**:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n radarr logs deploy/radarr -c oauth2-proxy --since=24h \
+     | grep -v "/ping" | grep "No valid authentication" -A 1 \
+     | grep -oE "\"[^\"]+\" HTTP" | sort | uniq -c | sort -rn | head'
+```
+
+One path dominating means an orphaned client, not a broken gate — annoying but harmless, and it
+clears when the tab is closed. Many different paths from one session means the session itself is
+not surviving, which is the real defect. A per-app `--api-route` is the structural fix for the
+first shape: it makes those paths answer **401** instead of 302, so the client fails loudly
+instead of looping. Note that a count of **0 `AuthSuccess`** alongside a high rejection count
+means nobody has signed in since the last restart — so `--cookie-refresh` has never run, and no
+session-lifecycle conclusion can be drawn from that window at all.
+
+**4. Did the gate admit the user, or did the app refuse them?** Look for the `[AuthSuccess]` line
+and what follows it. If `[AuthSuccess]` appears and the very next thing is the *app's own* login
+page or the app's 401, the gate did its job and layer 3 is your problem — that is the Crafty
+shape, and `runbooks/crafty-admin-credential.md` covers it.
+
+**5. Cookie names, when sessions do not survive.** Two oauth2-proxy instances emitting the
+**same** cookie name on the **same** domain hold different cookie-secrets, so each overwrites the
+other's cookie and the loser logs exactly the `No valid authentication` line above. List what
+each gate is configured to emit (arguments only — never print a Secret's data):
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'for ns in crafty jackett pihole qbittorrent radarr sonarr; do printf "%-12s " "$ns"; \
+     kubectl --context k3s-prod -n $ns get deploy $ns \
+       -o jsonpath="{range .spec.template.spec.containers[?(@.name==\"oauth2-proxy\")]}{.args}{end}" \
+     | tr "," "\n" | grep -E "cookie-name|cookie-domain" | tr "\n" " "; echo; done'
+```
+
+A gate with no `--cookie-name` is using oauth2-proxy's default `_oauth2_proxy`. If two of them
+share that default **and** a zone-wide `--cookie-domain=.home.keithjackson.dev`, they are
+evicting each other, and the symptom is intermittent and depends on which app you visited last.
+
+## Likely causes, most likely first
+
+1. **Cookie collision between two gates on one domain.** Default cookie names plus a zone-wide
+   cookie domain. Signature: high `No valid authentication`-to-`AuthSuccess` ratio, sessions
+   dying seconds after being created, symptom depends on browsing order, and *nothing* wrong at
+   Keycloak. Fix is a unique `--cookie-name` and a host-scoped `--cookie-domain` per gate — see
+   `decisions/2026-09-29-crafty-oauth2-proxy-signin.md`, which found and fixed exactly this.
+2. **The cookie is too large to come back.** With `--pass-access-token` the session carries the
+   access token; a fat Keycloak token pushes the cookie past 4 KB and it gets split into
+   `_oauth2_proxy_0`, `_oauth2_proxy_1`… If a chunk is dropped the session is unreadable and the
+   gate logs the same line. Look for numbered cookie names in the log, and for a session that
+   never survives a single navigation.
+3. **The XHR path is not the navigation path.** The app's API is under a prefix the gate handles
+   differently, or the app issues cross-origin XHRs the cookie does not accompany. Signature:
+   `/UI/...` returns 200 and `/api/...` returns 302 in the *same second*, which is precisely the
+   jackett trace above.
+4. **Session lifetime shorter than the app's polling interval,** with no refresh. The jackett
+   session above was a 5-minute session. If `refresh_token:true` is present and sessions still
+   die early, the refresh is failing, not missing.
+5. **Group check, not authentication.** `--allowed-group` is a floor: a valid Keycloak session in
+   the wrong group is refused *after* a successful login, which reads as a loop. Look for 403 and
+   for the `groups:[...]` list on the `[AuthSuccess]` line.
+6. **The app, not the gate.** `[AuthSuccess]` followed by the app's own login page or 401.
+
+## When to escalate or stop
+
+- **Stop and hand over if the fix is a manifest change.** Everything above is read-only
+  diagnosis. Changing a `--cookie-name`, adding `--metrics-address`, or editing an alert goes
+  through a branch and a PR; `kubectl edit` on a gate is drift that the next Flux reconcile
+  reverts, and in the meantime you have an unreproducible cluster.
+- **Stop if you find yourself about to print a Secret.** The cookie-secret and client-secret live
+  in `oauth2-proxy-credentials` and the per-app equivalents. Compare *key names* and whether two
+  Secrets differ; never dump `.data` or `.stringData`. A password has been leaked into a
+  transcript here exactly this way.
+- **Escalate to the platform owner if Keycloak itself is the suspect** — check `KeycloakDown` and
+  `KeycloakDatabaseDown` first (`runbooks/keycloak-down.md`, `runbooks/keycloak-db.md`). In the
+  2026-09-30 incident Keycloak was healthy throughout and blaming it would have cost days.
+- **The honest limit of this page.** With no metrics from the sidecars there is no history beyond
+  Loki's **72h** retention (`platform/monitoring/loki.yaml`, `retention_period: 72h`). If the
+  report is older than three days, the evidence is gone and the only remaining move is to
+  reproduce it. That is the cost of the gap `OAuth2ProxyGateUnobserved` is tracking, and it is
+  the reason to close it rather than silence it.

runbooks/oauth2-proxy-session.md

@@ -0,0 +1,195 @@
+# oauth2-proxy-session — telling the auth layers apart from the sidecar log
+
+**Not alert-driven.** This is an operator procedure for the per-app oauth2-proxy sidecars
+(`jackett`, `pihole`, `qbittorrent`, `radarr`, `sonarr`, `crafty`) and the shared forward-auth
+proxy in namespace `oauth2-proxy`. Use it when a gated UI behaves as though you are signed in and
+signed out at the same time: the page loads, but its buttons and saves quietly do nothing.
+
+> The sidecar log contains **email addresses, session UUIDs and full callback URLs including an
+> OAuth `code`**. Read it, do not paste it. (Repo rule §6: secrets never enter a transcript.)
+
+## What this means here
+
+Every gated app on `home.keithjackson.dev` has **at least two** independent authentications
+stacked on it, and the symptom rarely names the one that refused:
+
+1. **Keycloak** — the `homelab` realm issues the token. Failures here are visible: you see a login
+   form, or an error page from Keycloak's own hostname.
+2. **The oauth2-proxy sidecar** — validates the session cookie and the group claim
+   (`media-users` / `platform-admins`). **This is the layer that fails silently**, and the rest of
+   this page is about reading it.
+3. **The app's own login**, where it has one — Crafty and Radarr/Sonarr each present a local login
+   *behind* the Keycloak gate. See `runbooks/crafty-admin-credential.md` for that layer.
+
+The sidecar's access log distinguishes all three, if you know the four lines to look for. Get it:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n jackett logs \
+      -l app.kubernetes.io/name=jackett -c oauth2-proxy --tail=-1'
+
+Substitute the namespace and label for the app you are chasing. Always pass `--context k3s-prod`:
+`dev.sh` merges both clusters and picks a current-context **per invocation**, and that default has
+already changed once. Do not run two `dev.sh` invocations at the same time.
+
+### The four lines, and what each one means
+
+| Log line | What actually happened | Which layer |
+|---|---|---|
+| `[AuthSuccess] Authenticated via OAuth2: Session{... groups:[...]}` | Keycloak issued a token and the proxy accepted it. **The IdP is fine.** | Keycloak ✔ |
+| `[oauthproxy.go:1026] No valid authentication in request. Initiating login.` | **No session cookie was presented at all.** Either you never had one, or something *deleted* it — see below. | sidecar |
+| `[stored_session.go:94] Error loading cookied session: cookie signature not valid, removing session` | A cookie **of the right name** arrived that this proxy **could not decrypt**. Almost always a *different* proxy's cookie. Note `removing session`: this proxy then **deletes** it. | sidecar, cookie collision |
+| an upstream path of `-` instead of `/` in an access-log line | the **proxy answered the request itself**; it never reached the app. Compare `POST - "/api/..."` against `POST / "/api/..."`. | sidecar |
+
+That last column is the one people miss. The upstream field sits between the method and the quoted
+path. `-` means the proxy handled it (a redirect, a 401, its own `/oauth2/*` endpoints); `/` means
+it was proxied to the app. Count them:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n jackett logs \
+      -l app.kubernetes.io/name=jackett -c oauth2-proxy --tail=-1 \
+      | grep -v "\"/ping\"" | grep -E " (POST|PUT|DELETE) " | cut -c1-140'
+
+If your writes show `-`, the app never saw them. On 2026-09-30, **4 of the 6 POSTs** in jackett's
+log had `-`.
+
+## First commands
+
+**1. Did Keycloak ever succeed?** No `AuthSuccess` at all means this is not a cookie problem —
+go to `runbooks/keycloak-down.md` or `runbooks/keycloak-client-secrets.md`.
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n <ns> logs \
+      -l app.kubernetes.io/name=<app> -c oauth2-proxy --tail=-1 | grep -c AuthSuccess'
+
+**2. How long does a session last?** Subtract `created` from `expires` on an `AuthSuccess` line.
+If it is exactly **5m**, the session's lifetime is Keycloak's access-token lifetime and the proxy
+is not refreshing it:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n <ns> logs \
+      -l app.kubernetes.io/name=<app> -c oauth2-proxy --tail=-1 \
+      | grep AuthSuccess | tail -1 | grep -o "created:[^ ]* [^ ]*\|expires:[^ ]* [^ ]*"'
+
+The same line says `refresh_token:true` — the proxy is *holding* a refresh token. It will only
+spend it if `--cookie-refresh` is set; the default is `0`, meaning disabled.
+
+**3. Is this a cookie collision?** Two checks, and you want both.
+
+*Names and scopes* — every instance that emits the default name `_oauth2_proxy` on the shared
+`.home.keithjackson.dev` domain is in the collision set with every other one:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get pods -A -o json \
+      | grep -E "cookie-name|cookie-domain"'
+
+An instance with **no** `--cookie-name` argument is using the default. That is the failure — the
+flag being absent, not wrong.
+
+*The smoking gun* — search **every** proxy, not just the one you are debugging, for the removal
+line, and put the timestamps next to each other:
+
+    infra/devcontainer/dev.sh run bash -c 'for ns in jackett pihole qbittorrent radarr sonarr crafty; do \
+      echo "== $ns"; kubectl --context k3s-prod -n $ns logs -l app.kubernetes.io/name=$ns \
+      -c oauth2-proxy --tail=-1 2>/dev/null | grep stored_session; done'
+
+**4. Do the cookie-secrets actually differ?** They should — but confirm the collision is between
+instances that genuinely cannot read each other. Compare **hashes**, never values:
+
+    infra/devcontainer/dev.sh run bash -c 'for ns in jackett pihole qbittorrent radarr sonarr crafty; do \
+      printf "%s " "$ns"; kubectl --context k3s-prod -n $ns get secret $ns-oauth2-proxy \
+      -o jsonpath="{.data.cookie-secret}" | sha256sum | cut -c1-12; done'
+
+Never print `.data` or `.stringData` itself. A password has been leaked into a transcript here
+before.
+
+### Do not rank apps by their failure ratio
+
+Counting `No valid authentication` against `AuthSuccess` per app looks like a health metric and is
+not one. **It ranks chattiness, not breakage.** An app whose UI polls or holds a WebSocket
+manufactures one failure line per retry, so a single dead cookie in one idle browser tab becomes
+scores of log lines, while an app nobody is using scores zero no matter how broken it is. Measured
+2026-09-30: Radarr showed **156** unauthenticated requests against **1** `AuthSuccess`, but the top
+paths were `/api/v3/command` (62), `/api/v3/queue/details` (46) and `/signalr/messages/negotiate`
+(43) — polling, not 156 user-visible failures.
+
+**Bound the window to the current pod, not to wall-clock.** A ratio measured over 24 h across a pod
+that restarted 13 h ago is mostly about the *previous* configuration, and that is exactly how a
+deployed fix gets written off as ineffective. The pod's own log has an exact boundary — its process
+start — so read it there:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n <ns> get pod \
+      -l app.kubernetes.io/name=<app> -o jsonpath="{.items[*].status.startTime}{\"\n\"}"'
+
+then count within that same pod's log. On 2026-09-30 this took Crafty from an apparent 81:1 over 24 h
+to **3 failures in its fixed pod's entire 13 h 52 m life**, two of them `/ws` reconnects.
+
+## Likely causes, most likely first
+
+1. **Cookie collision between two per-app proxies.** Several instances emit the default
+   `_oauth2_proxy` on `.home.keithjackson.dev` with different `cookie-secret`s. Whoever
+   authenticates last overwrites the rest; the losers log `cookie signature not valid, removing
+   session` and — because `removing session` clears the cookie on *their* `--cookie-domain`, which
+   is the whole zone — **delete a session belonging to an app in another namespace**. The victim
+   then logs `No valid authentication` with no cookie at all, seconds after a clean login, while
+   its own `expires` is still minutes away. **Read both logs side by side; one app's log alone
+   makes this look like random session loss.** Fix: a unique `--cookie-name` and a host-scoped
+   `--cookie-domain` on the per-app sidecar. The shared forward-auth proxy keeps the zone-wide
+   scope — cross-host auth is its job.
+2. **Session pinned to a 5-minute access token with no refresh.** `--cookie-refresh` unset means
+   the proxy never spends the refresh token it holds, so the session expires with the access token.
+   Set it to about half the measured token lifetime.
+3. **An XHR got a 302 instead of a 401.** The app's JavaScript follows the redirect invisibly, the
+   live Keycloak SSO session satisfies it without a prompt, and the XHR resolves with **HTML where
+   it expected JSON**. Nothing is logged as an error and the operator is never re-prompted — this
+   is what "it just doesn't save, and reloading doesn't help" looks like. Worse on a write: a
+   browser following a 302 re-issues a POST as a GET and **drops the body**, so the change is
+   discarded silently. oauth2-proxy's built-in AJAX guard only fires on an exact
+   `Accept: application/json`, which most app JavaScript does not send; make it deterministic with
+   `--api-route=<regex>` (**singular** — `--api-routes` does not exist and the proxy refuses to
+   start). Keep browser-navigable paths *out* of the regex so a plain page load still signs you in.
+4. **Group membership.** An `AuthSuccess` followed by `403` is the `--allowed-group` gate, not a
+   session problem. Check the `groups:[...]` field on the `AuthSuccess` line.
+5. **The app's own second login.** Crafty, Radarr and Sonarr each have one. `AuthSuccess` followed
+   by the app's own `/login` page is layer 3, not layer 2.
+
+## When to escalate or stop
+
+- **Stop before you `kubectl edit`/`patch`/`apply` anything.** Flux reconciles `main`; a
+  hand-applied flag is drift that the next reconcile reverts or, worse, adopts. Argument changes
+  go through a branch and a PR. Reading logs is always safe; changing the Deployment is not an
+  agent action.
+- **Do not hand-edit `apps/**/<app>.yaml` — those files are GENERATED.** The five per-app sidecars
+  (jackett, pihole, qbittorrent, radarr, sonarr) get their argument list from one shared function,
+  `_oauth2_sidecar_container()` in `catalog/render.py`; `catalog/<app>.yaml` declares only
+  `auth.mode` and `auth.groups`, and the `auth` schema is `additionalProperties: false`, so there
+  is **no per-app override hook**. A hand-edit to the rendered file is caught by the merge gate's
+  `render-golden` stage ("GOLDEN DRIFT ... does not match the canonical render") and would be
+  erased by the next render anyway. Change the renderer, then regenerate:
+
+      python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation
+
+  **Budget for the blast radius**: because the function is shared, any change to it rewrites the
+  sidecar block in **all five** rendered apps, and the gate requires all five be regenerated in
+  the same commit. There is no way to change one app's sidecar arguments alone. Measured
+  2026-09-30. Coordinate before starting — this is a single change, not five.
+- **Crafty is the exception** and is not rendered from the catalog: its sidecar is hand-written in
+  `platform/crafty/deployment.yaml`, so it can be, and was, fixed independently.
+- **You cannot currently alert on any of this**, and that is worth knowing before you promise
+  monitoring. Measured 2026-09-30: no per-app sidecar sets `--metrics-address` and none declares a
+  `metrics` container port, so oauth2-proxy's metrics listener never starts — the only oauth2-proxy
+  series in Prometheus come from the shared proxy in namespace `oauth2-proxy`, which does set both.
+  Until the renderer emits them for the per-app sidecars, every diagnosis on this page is
+  log-reading only.
+- **Validate an argument change before you open the PR**, not after. The pinned image will tell
+  you what it understood, without touching the cluster:
+
+      podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help
+
+  Running it with your full proposed argument list prints a `Cookie settings:` line and an
+  `API route - Path:` line back at you, and rejects mutually exclusive flags outright
+  (`--session-cookie-minimal` with `--cookie-refresh`, for one). A flag typo here is a CrashLoop
+  on a service you have just told someone is fixed.
+- **Do not rotate a `cookie-secret` to "clear" a bad session.** It signs every live session out of
+  that app and does nothing about the collision, which is caused by the *names* matching, not the
+  secrets differing.
+- **Escalate to Keycloak** (`runbooks/keycloak-down.md`, `runbooks/keycloak-client-secrets.md`) if
+  there is no `AuthSuccess` in any proxy's log — that is an IdP or client-secret problem and no
+  cookie flag will help.
+- **Hand off rather than guess** if you see `cookie signature not valid` on a proxy that already
+  has a unique `--cookie-name`. That combination should not occur; it means something is writing
+  that name which is not in this inventory. Record what you measured.

runbooks/plex-quicksync.md

@@ -0,0 +1,422 @@
+# plex-quicksync — making Plex actually use Intel Quick Sync, and proving it did
+
+**Alert:** none, and none is possible — see "How to verify" below. This is an **operator procedure**,
+not an incident page. It is indexed here because the feature it completes is **inert until a human
+clicks one button**, and a switched-off feature that looks shipped is exactly what this page exists
+to prevent.
+**Files:** `catalog/plex.yaml` (`hardware.gpu: intel-quicksync`), `apps/wave-0/plex.yaml` (the
+rendered golden — the `resources.limits: gpu.intel.com/i915: 1` block), `catalog/render.py` (the
+mechanism), `schemas/renderer-contract.md` `### hardware` (the contract),
+`platform/intel-gpu-plugin/` (the DaemonSet that advertises the resource).
+**Related:** `runbooks/plex-claim.md` — the other Plex procedure with a manual last mile.
+
+## STOP HERE IF PLEX IS `Pending` / `Unschedulable`
+
+That is this feature's **one serious failure mode** and it **stalls the whole app tier**
+(`apps-wave-0` is `wait: true`; `apps-wave-1` `dependsOn` it). Since 2026-10-01 Plex requests the
+extended resource `gpu.intel.com/i915: 1`, which is a **hard scheduling constraint** — if the
+`intel-gpu-plugin` DaemonSet is not advertising it on at least one schedulable node, Plex does not
+start at all. Go to **"Plex will not schedule"** below; it is the first troubleshooting section on
+purpose.
+
+**The escape hatch, in one sentence:** delete the two `hardware:` lines from `catalog/plex.yaml`,
+re-render, and merge — Plex returns to software transcoding and the tier unblocks. Exact commands
+are in **"Plex will not schedule" → remedy B**.
+
+## What this means here
+
+Plex transcodes in **software** by default. On the UHD Graphics 630 iGPU of an i7-9700T, Quick Sync
+moves that work off the CPU — which matters on a three-node cluster where the CPU is also running
+everything else. Two independent things have to be true before that happens, and **only one of them
+is in git**:
+
+| Half | Who does it | State |
+|---|---|---|
+| The render device is inside the container **and openable** | **GitOps.** `hardware.gpu: intel-quicksync` in `catalog/plex.yaml` renders `resources.limits: {gpu.intel.com/i915: 1}` on the app container. The `intel-gpu-plugin` DaemonSet (`platform/intel-gpu-plugin/`) injects the device **and the matching device-cgroup allow rule**. Flux applies it. | Done by merging that change. |
+| Plex is *told* to use it | **The operator, by hand, in the Plex web UI.** | **NOT done.** `HardwareAcceleratedCodecs` was **absent** from `Preferences.xml` when this page was written — the toggle has never been on. |
+
+**There is no GitOps path to the second half, and that is verified rather than assumed.** A bounded
+grep across the running image's entire init surface — `/etc/cont-init.d`, `/etc/services.d`,
+`/etc/fix-attrs.d`, `/plex-common.sh`, `/usr/local/bin` — found **zero** `PLEX_PREFERENCE*` support.
+Every `setPref` the image performs is one of six hardcoded keys (`MachineIdentifier`,
+`ProcessedMachineIdentifier`, `PlexOnlineToken`, `customConnections`, `allowedNetworks`,
+`TranscoderTempDirectory`), none of them the transcode toggle. No environment variable, no ConfigMap
+and no `postBuild` substitution can set it. So do not go looking for the clever way to automate this;
+there isn't one, and the search has already been done.
+
+### Why a resource request and not a `/dev/dri` hostPath
+
+Because the hostPath was **tried and measured not to work**, and knowing that is the only thing that
+will stop someone "simplifying" it back. Merged 2026-09-30, measured in the live pod on k3s-prod-2 on
+**2026-10-01**: the device node was plainly **visible** in the container
+(`crw-rw---- root video2 226,128 renderD128`) and **could not be opened** —
+
+| attempt | result |
+|---|---|
+| `uid 0`, `O_RDWR` | **`EPERM`** "Operation not permitted" |
+| `uid 1000`, `O_RDWR` | **`EPERM`** "Operation not permitted" |
+
+`EPERM` is the **device cgroup** refusing, not `EACCES` from file permissions. The two look identical
+from the outside and are not the same problem. Proven three ways: root on the **host** opened the
+same device `rc=0`; the node is unified `cgroup2fs`; and the container's runc `config.json` read
+`linux.resources.devices: [{allow: False, access: 'rwm'}]` with `linux.devices: None` — **one
+deny-all rule, no allow entry for major 226**. A hostPath is a *filesystem* operation: it binds the
+device node in and adds **no cgroup allow rule**, because Kubernetes cannot infer one from a volume.
+
+Two non-workarounds, so nobody spends an evening on them: **`uid 0` failed too**, so running as root
+does not bypass the cgroup gate; and **`privileged: true` is not available** — Kyverno
+`restrict-host-namespaces` is **Enforce** on app namespaces and rejects it.
+
+The device plugin is what produces the allow rule — in the v0.37.0 source,
+`createDeviceSpecsFromDrmFiles` emits `DeviceSpec{…, Permissions: "rw"}`, which the kubelet turns
+into both the device injection and the matching cgroup allow.
+
+**The failure mode that trade bought, stated plainly.** Before, a missing device meant the pod ran
+and **silently software-transcoded** — invisible, but harmless to everything else. Now, a missing
+*resource* means Plex is **Unschedulable** and the app tier stalls. That is strictly worse in blast
+radius and it is **unavoidable**: an extended resource is a hard scheduling constraint, and it is
+also the only thing that produces the cgroup allow rule. There is no graceful-degradation variant.
+The one improvement is that the loss is no longer silent.
+
+What this does **not** do, so nobody reads more into a green gate than is there: it does not enable
+hardware transcoding (that is the manual toggle below), a green gate only proves the **manifest
+renders**, and it writes no `nodeSelector` — the resource itself constrains Plex to nodes the plugin
+advertises, which is the scheduler's doing and not a selector in this repo.
+
+## Running the commands in this runbook
+
+`kubectl` does not work on the operator workstation; the toolchain lives only in the pinned dev
+container. Every command below runs as:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get nodes'
+
+Always pass `--context k3s-prod` explicitly. `dev.sh` merges both clusters into one kubeconfig and
+picks a current-context **per invocation**, and that default has already changed once. Do not run two
+`dev.sh` invocations at once.
+
+**Never `cat` `Preferences.xml`.** It contains `PlexOnlineToken`, a plex.tv account credential. Every
+read of that file below greps a single named key — do not widen one.
+
+## First commands
+
+**0. Is Plex running at all?** Ask this first now — the current mechanism can stop it starting.
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex -o wide'
+```
+
+`Pending` ⇒ go straight to **"Plex will not schedule"**. Running ⇒ carry on.
+
+**1. Do the nodes advertise the resource, and is the plugin Ready?** This replaces the old "is the
+hostPath mounted" check, and it is now the first thing that can be wrong.
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod get nodes -o json \
+   | jq -r ".items[] | \"\(.metadata.name) i915=\(.status.allocatable[\"gpu.intel.com/i915\"] // \"ABSENT\")\""'
+```
+
+Expect `i915=1` on each of the three prod nodes (measured 2026-10-01: all three). Any `ABSENT` ⇒ that
+node cannot run Plex. Then the DaemonSet behind it:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n intel-gpu-plugin get daemonset intel-gpu-plugin'
+```
+
+Expect **3 desired / 3 ready / 3 available** (measured 2026-10-01). Anything less is the cause, not a
+symptom — fix it there, in `platform/intel-gpu-plugin/`, not here.
+
+**2. Did Flux apply the current manifest?** `gpu.intel.com/i915` on the app container means the
+GitOps half landed:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \
+     -o jsonpath="{.items[0].spec.containers[0].resources.limits}"; echo'
+```
+
+A `/dev/dri` **volumeMount** instead ⇒ the running pod predates the 2026-10-01 merge and is on the
+mechanism that does not work. Compare
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n flux-system get kustomization apps-wave-0 \
+     -o jsonpath="{.status.lastAppliedRevision}"; echo'
+```
+
+against the merge commit before blaming anything else.
+
+**3. Is the device visible inside the container, and can Plex *open* it?** This is the decisive test
+of the whole mechanism, and **it must be run as uid 1000**, because that is what Plex runs as.
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex exec deploy/plex -- ls -l /dev/dri'
+```
+
+Expect `renderD128`, mode `crw-rw----`. The plugin injects it; there is no longer any
+`DirectoryOrCreate` empty-directory case — if the resource were missing the pod would not be running
+to exec into. Then the open, **as the app user**:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex exec deploy/plex -- \
+     s6-setuidgid plex sh -c "exec 3<>/dev/dri/renderD128 && echo RW-OPEN-OK || echo RW-OPEN-DENIED"'
+```
+
+`RW-OPEN-OK` **as uid 1000** is the pass condition. **Passing as `uid 0` is weaker evidence and must
+not be accepted as proof** — Plex runs as 1000, and the whole point of the 2026-10-01 change was a
+cgroup rule that uid 0 could not buy its way past either. `RW-OPEN-DENIED` with the device plainly
+visible is the **device-cgroup** failure — i.e. the plugin's allow rule did not arrive; see "Likely
+causes" #2.
+
+**4. Is the toggle on?** Named key only — never the whole file:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex exec deploy/plex -- \
+     grep -o "HardwareAcceleratedCodecs=\"[01]\"" \
+     "/config/Library/Application Support/Plex Media Server/Preferences.xml" || echo ABSENT'
+```
+
+`ABSENT` ⇒ never enabled (the state this page was written in). `="0"` ⇒ explicitly off. `="1"` ⇒ on,
+and the remaining question is whether it *works* — go to "How to verify".
+
+## The one manual step
+
+Plex web UI → **Settings → Transcoder → "Use hardware acceleration when available"** → tick → Save.
+(It sits under *Show Advanced*.) A transcode already in flight is unaffected; the next one picks it up.
+
+The underlying preference key is **`HardwareAcceleratedCodecs`**. That name was confirmed by
+extracting strings from the 23 MB `Plex Media Server` binary in the running container — **not** from
+recall and not from a forum post. Real neighbouring keys from the same binary, so you can recognise
+them and not invent them:
+
+| Key | What it is |
+|---|---|
+| `HardwareAcceleratedCodecs` | the toggle this page is about |
+| `HardwareAcceleratedEncoders` | hardware **encode**, as distinct from decode |
+| `HardwareDevicePath` | how Plex is pointed at a *specific* render node — relevant only if a host ever exposes more than one |
+| `TranscoderToneMapping`, `TranscoderToneMappingAgorithm` | HDR→SDR tone mapping. **That misspelling of "Algorithm" is upstream's, in the binary. It is not a typo in this page — do not "fix" it** here or anywhere else; the corrected spelling is not a key Plex reads. |
+| `TranscoderHEVCEncoding` | HEVC output |
+
+**`HardwareAcceleratedCodecsEnableHEVC` does not exist.** It is widely cited and it is not in the
+binary. Recorded explicitly so nobody spends an evening hunting for it or, worse, writes it into
+`Preferences.xml` and concludes hardware transcoding is broken.
+
+Editing `Preferences.xml` by hand is **not** the procedure. It is app state on the replicated Longhorn
+tier, it is in no git commit, and the UI writes the key correctly. Use the UI.
+
+## How to verify — and why there is no alert
+
+Start a playback that **forces a transcode** (a client that cannot direct-play the source, or pick a
+lower quality explicitly), then read Plex's own **Dashboard → Activity**:
+
+- **`Transcode (hw)`** on the session ⇒ hardware transcoding. **This is the proof.**
+- plain **`Transcode`** ⇒ software. The feature is not working, whatever the manifest says.
+
+**That dashboard is the *only* proof available, and nothing else will ever tell you.**
+`catalog/plex.yaml` declares no `metricsEndpoint`, so Plex renders **no ServiceMonitor at all** —
+there is no scrape, no transcode-mode metric, and therefore no alert that could fire. Do not open work
+to "alert on software-transcode fallback": with no metric behind it such a rule would be permanently
+inert, and this repo has already been burned at scale by rules that evaluated against nothing while
+looking like coverage (`runbooks/README.md`, "Alerts you should not trust the silence of"). If that
+gap is ever worth closing, the honest order is a real exporter first and a rule second.
+
+Corroborating, not a substitute — the server log on the config PVC:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex exec deploy/plex -- \
+     sh -c "grep -i -m20 -e hardware -e vaapi -e hwdevice \
+     \"/config/Library/Application Support/Plex Media Server/Logs/Plex Media Server.log\""'
+```
+
+Treat those strings as advisory: they are Plex's, not this repo's, and they move between versions.
+Absence of a match is **not** evidence of software transcoding; a VA-API device open is good evidence
+of hardware. The dashboard remains the verdict.
+
+## Likely causes, most likely first
+
+When the dashboard still says plain `Transcode`:
+
+1. **The toggle is off.** Overwhelmingly the most likely answer, and the reason this page exists.
+   First-command #4 returns `ABSENT` or `="0"`. Remedy: the manual step above. Check this **before**
+   investigating anything technical.
+
+2. **The device cannot be `open()`ed — the plugin's cgroup allow rule did not arrive.** Symptom:
+   first-command #3 shows `renderD128` present and correctly owned, and the open returns
+   `RW-OPEN-DENIED` / `Operation not permitted`. This is `EPERM` from the **device cgroup**, not
+   `EACCES` from file permissions. **This is no longer a design risk — it is the measured reason the
+   design is what it is** (2026-10-01; see "Why a resource request and not a `/dev/dri` hostPath").
+   If it recurs *under the current mechanism*, the plugin is the suspect, not the manifest: confirm
+   the pod really did consume the resource
+
+   ```
+   infra/devcontainer/dev.sh run bash -c \
+     'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \
+        -o jsonpath="{.items[0].spec.containers[0].resources}"; echo'
+   ```
+
+   and then check the plugin pod on Plex's node (first-command #1). **Do not reach for
+   `privileged: true`** — Kyverno `restrict-host-namespaces` is Enforce and rejects it, and uid 0
+   failed the same open in 2026-09-30's measurement anyway. **Do not re-add a `/dev/dri` hostPath**;
+   that is the mechanism this replaced, and it was measured not to work.
+
+3. **The pod landed on a node whose GPU went away.** Under the current mechanism this does **not**
+   present as an empty `/dev/dri` — the node simply stops advertising `gpu.intel.com/i915` and Plex
+   either never schedules there or gets evicted off. So this cause now shows up as **`Pending`**, not
+   as silent software transcoding: see "Plex will not schedule". All three prod nodes carried
+   `renderD128` with `i915` bound when measured (2026-09-30) and all three advertised the resource on
+   2026-10-01, so this means a node changed — a rebuild, a BIOS reset, or a driver regression.
+   **Fix the node.**
+
+4. **The image changed under you.** See "The floating-image caveat". A `:latest` pull can remove the
+   mechanism this feature leans on with no commit in this repo.
+
+5. **The session was never transcoding.** Direct Play / Direct Stream never touches the transcoder, so
+   `Transcode (hw)` cannot appear. Confirm the session actually says `Transcode` before treating it as
+   a hardware failure.
+
+6. **The source needs a codec Quick Sync on this generation cannot handle** — e.g. AV1 decode, which
+   Coffee Lake's UHD 630 has no engine for. Plex falls back to software *per stream*. Symptom: some
+   sessions show `(hw)` and others do not. Not a fault.
+
+## Plex will not schedule
+
+**Symptom:** the Plex pod sits `Pending`, and `apps-wave-0` does not go Ready, which holds
+`apps-wave-1` behind it. This is the cost of the 2026-10-01 mechanism and the section you want at
+2am.
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex describe pod -l app.kubernetes.io/name=plex | tail -20'
+```
+
+`0/3 nodes are available: 3 Insufficient gpu.intel.com/i915` (or similar) confirms it: **nothing is
+advertising the resource Plex requires**. Plex's schedulability is coupled to the `intel-gpu-plugin`
+DaemonSet, deliberately and unavoidably — the plugin is also the only thing that produces the
+device-cgroup allow rule, so there is no variant of this feature that degrades gracefully.
+
+**Remedy A — fix the plugin (preferred).** Run first-command #1. If the DaemonSet is not 3/3, or the
+nodes show `i915=ABSENT`, the fault is in `platform/intel-gpu-plugin/` or on the node, and that is
+where to fix it. Check its Flux Kustomization reconciled:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n flux-system get kustomization intel-gpu-plugin'
+```
+
+A plugin that is merely **not yet reconciled** resolves itself; one that is failing does not.
+
+**Remedy B — the escape hatch: give up hardware transcoding and unblock the tier.** One field, no
+code change. In a branch (never on `main`, never `kubectl edit` — Flux reconciles `main` and would
+revert you):
+
+1. delete these two lines from `catalog/plex.yaml`:
+
+   ```yaml
+   hardware:
+     gpu: intel-quicksync
+   ```
+
+2. re-render the golden:
+
+   ```
+   python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation
+   ```
+
+   The only change must be `apps/wave-0/plex.yaml`, losing its `resources` block.
+3. gate, PR, merge. Plex schedules anywhere and **software-transcodes**; the app tier unblocks.
+
+Re-adding the field later is the same edit in reverse. Nothing about the Plex-side toggle or the
+`/config` PVC is affected either way — do **not** delete the PVC as part of this.
+
+## The floating-image caveat
+
+Everything this page states about the image's behaviour — the `cont-init.d` hook that adds the app
+user to the device's group, the absence of `PLEX_PREFERENCE*`, the six `setPref` keys, the
+`HardwareAcceleratedCodecs` string — was read from digest **`sha256:e0ab27395614…`**
+(`plexmediaserver 1.43.4.10903`).
+
+The deployment runs **`plexinc/pms-docker:latest` with `imagePullPolicy: Always`**. A silent pull can
+therefore change the behaviour this feature depends on **with no commit in this repo**, while the
+manifest still looks identical. Symptom to watch for: hardware transcoding that worked and quietly
+stopped, with no git change to point at.
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \
+     -o jsonpath="{.items[0].status.containerStatuses[0].imageID}"; echo'
+```
+
+If that digest is not the one above, **re-verify the hook before trusting this page**:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex exec deploy/plex -- ls /etc/cont-init.d/'
+```
+
+`45-plex-hw-transcode-and-connected-tuner` still present ⇒ the group mechanism is still the one
+described here. Gone or renamed ⇒ stop and re-measure. In particular the "no `supplementalGroups`
+needed" conclusion in `catalog/plex.yaml` and `schemas/renderer-contract.md` is only true while that
+hook exists.
+
+## When to escalate or stop
+
+- **Do not add `supplementalGroups` or a `podSecurityContext` to `catalog/plex.yaml`.** It would be
+  **inert**, and it would mislead the next person. Measured in-container 2026-09-30: the image's
+  `cont-init.d` hook stats each `/dev/dri` char device and `usermod -a -G`s the `plex` user into its
+  group *inside* the container, and the service then execs via `s6-setuidgid plex`, which **replaces**
+  the process group list from `/etc/group` rather than appending — `Plex Media Server` runs with
+  `Groups: 100 1000`, root's gid `0` (which PID 1 does hold) gone. Anything Kubernetes supplies is
+  discarded before Plex starts. `render.py --selftest` asserts that absence deliberately.
+- **Do not reach for `privileged: true`.** Kyverno `restrict-host-namespaces` is Enforce and will
+  reject it — and it would not help: `uid 0` failed the same `open()` in the 2026-09-30 measurement,
+  because root does not bypass the cgroup device gate.
+- **Do not re-add a `/dev/dri` hostPath, and do not "simplify" the resource request into one.** That
+  was the original mechanism and it was measured non-functional on 2026-10-01 (device-cgroup `EPERM`
+  with the device plainly visible). The device plugin's `DeviceSpec{… Permissions: "rw"}` is what
+  grants access; a volume cannot.
+- **Do not `kubectl edit`/`patch` the Deployment to add or change the device mount.** Flux reconciles
+  `main`; a hand-applied change is drift that the next reconcile reverts or, worse, adopts. Mutating
+  `kubectl` and `flux reconcile` are **operator** actions, and the durable change is a git change
+  through a branch, a PR and a green gate.
+- **Do not edit `Preferences.xml` on the PVC**, never `cat` it, and never `sops -d` anything into a
+  terminal. It holds `PlexOnlineToken`.
+- **Do not delete the `plex-config` PVC to "start clean".** It is the library database, metadata and
+  watch state (`policy: daily-offcluster`). The transcode toggle is one preference, not a reason to
+  lose the DB.
+- **Stop and get the operator** if the device cannot be opened from the pod as uid 1000 (cause #2
+  confirmed) *under the current mechanism*. The next step is a platform change in
+  `platform/intel-gpu-plugin/`, with its own PR, not more poking at this workload.
+
+## Known gaps, deliberately not fixed here
+
+Recorded so they are findable, and so nobody assumes they were overlooked:
+
+- **Transcode scratch is on the wrong medium, and it is invisible.** The image sets
+  `TranscoderTempDirectory=/transcode`, which is **not a volume** — it is the container's writable
+  overlay, i.e. the node's `/data`. That is NVMe on prod-2 but **SATA on prod-1 and prod-3**, so
+  transcode scratch performance changes silently whenever the pod is rescheduled. Both candidate fixes
+  need a **schema** change and are therefore out of scope here: pin Plex to prod-2 (needs a
+  `nodeSelector` field the schema does not have), or give `/transcode` a `medium: Memory` emptyDir and
+  transcode in RAM (~31 GiB per node; needs an emptyDir field the schema does not have). The RAM option
+  is the better one, because it removes the node asymmetry outright instead of working around it.
+- **GuC/HuC firmware is not loaded** (`i915.enable_guc=0`). The cost is low-power H.264 **encode**
+  quality only — decode, and the feature on this page, are unaffected. Changing it needs a kernel
+  parameter and a **serialized rolling reboot**, which is an operator action (CLAUDE.md §5), and this
+  cluster has a documented history of that going badly (a prod node hung on a warm reboot on
+  2026-09-03). Not worth coupling to this change.
+
+---
+Source of truth: `runbooks/plex-quicksync.md`. Index: `/runbooks/`
+The node measurements (all three prod nodes) and the in-container image measurements cited here were
+taken **2026-09-30** against `plexinc/pms-docker` digest `sha256:e0ab27395614…`
+(`plexmediaserver 1.43.4.10903`). The `EPERM` open measurements, the `Groups: 44 100 992 1000`
+reading and the `gpu.intel.com/i915: 1` allocatable/DaemonSet readings were taken **2026-10-01**.
+Hardware transcoding has **still not been observed working**: the open test has not been re-run under
+the device-plugin mechanism (it cannot be, until that merges and Flux reconciles), and the Plex-side
+toggle is still unset.

schemas/README.md

@@ -15,7 +15,7 @@ docs/13 exactly. Do not diverge from it here — escalate spec problems, don't f
 | `servicedefinition.schema.json` | Formal JSON Schema (draft 2020-12) for a ServiceDefinition. Enforces enums, required fields, patterns, and cross-field invariants. |
 | `renderer-contract.md` | Authoritative field → rendered-resource mapping, with requirement IDs. The contract other workstreams consume. |
 
-**Schema version:** 1.0.0 (in the schema `description`). Treat any change as an interface change:
+**Schema version:** 1.8.0 (in the schema `description`, which carries the full per-version changelog — this line had said `1.0.0` since 1.1.0 and is now synced). Treat any change as an interface change:
 propose on the task board, update dependent renderers/tests in the same change, never silently break a
 consumer (coordination-protocol §2).
 
@@ -35,6 +35,9 @@ consumer (coordination-protocol §2).
 - `ingressRoutes[].host` must match `<label>.home.keithjackson.dev`.
 - `storage[].class ∈ {longhorn, bulk}`; `exposure ∈ {internal, public}`; `egress ∈ {internet, vpn}`;
   `auth.mode ∈ {oidc, forward-auth, native, none}`.
+- `hardware.gpu ∈ {intel-quicksync}` — a host **capability class**, never a device path. The enum
+  is the contract: the mechanism that satisfies it lives only in `catalog/render.py` (see
+  `renderer-contract.md` §`hardware`).
 - `additionalProperties: false` everywhere — unknown keys are rejected (catches typos in review/CI).
 
 ## Validate examples against the schema — L1, no cluster (R-TEST-02)

schemas/renderer-contract.md

@@ -23,6 +23,7 @@ platform must render, and cites the requirement each mapping satisfies. Changing
 | `dependencies` | `[]` → reaches **only DNS** (default-deny) | R-NET-01 |
 | `inClusterClients` | `[]` → no direct east-west client; only the Gateway may reach an exposed port | R-NET-01/02 |
 | `auth.mode` | `forward-auth` for any exposed UI | R-IDP-02 |
+| `auth.apiRoutes` | `[]` → the sidecar renders **no** `--api-route`; every unauthenticated request is 302'd to Keycloak | R-IDP-02 |
 | `update.channel` | `stable` | R-UPD-01 |
 | `update.autoRollbackOnFailedHealth` | `true` | R-UPD-03 |
 | `observability.metrics` | `true` | R-OBS-01 |
@@ -160,6 +161,123 @@ platform must render, and cites the requirement each mapping satisfies. Changing
 - **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication),
   **R-STATE-04** (bulk tier). (docs/04)
 
+### `hardware` (`gpu`) — host device capability
+- **The field is INTENT, never MECHANISM.** `hardware.gpu` is an **enum of capability classes**
+  (today: `intel-quicksync`), not a device path, not a hostPath, not a resource request. The
+  ServiceDefinition says *what class of host device this workload needs*; how the platform hands it
+  over is `catalog/render.py`'s business and lives nowhere else. **Do not "simplify" this into a raw
+  `/dev/...` path.** The payoff is no longer hypothetical: on **2026-10-01** the mechanism was
+  swapped outright — from a `/dev/dri` hostPath to the Intel GPU device plugin's extended resource —
+  as a **`catalog/render.py` change plus a re-render, with ZERO catalog edits** and no re-review of
+  per-service device wiring. `catalog/plex.yaml`'s `hardware:` block came through byte-identical.
+  Extend the enum for a genuinely different capability class (e.g. `nvidia-nvenc`); never to name a
+  second route to the same one.
+- `gpu: intel-quicksync` → on the **app container only** (never a sidecar), **exactly one thing**:
+
+      resources:
+        limits:
+          gpu.intel.com/i915: 1
+
+  **That is the whole mapping.** No volume, no volumeMount, no `securityContext`, no `nodeSelector`,
+  no privileged anything.
+  - **`limits`, not `requests`, and that is not a style choice.** An extended resource must be
+    expressed in `limits`; Kubernetes mirrors `requests` from `limits` automatically, and a
+    requests-only extended resource is rejected as invalid.
+  - The resource is advertised by the **`intel-gpu-plugin` DaemonSet** (`platform/intel-gpu-plugin/`,
+    its own Flux Kustomization at `clusters/base/intel-gpu-plugin.yaml`, pinned
+    `intel/intel-gpu-plugin:0.37.0`). **Measured 2026-10-01:** all three prod nodes report
+    `.status.allocatable["gpu.intel.com/i915"] == 1`; DaemonSet 3 desired / 3 ready / 3 available.
+  - This is the **only** app-container `resources` render path in the file, and it is deliberately
+    narrow. **Do not generalise it into a CPU/memory resources feature** while passing through — that
+    is a separate field with a separate justification, and `render.py --selftest` asserts the emitted
+    dict *exactly* so the attempt fails loudly.
+- **WHY THE hostPath MECHANISM WAS ABANDONED — read this before "simplifying" it back.** The first
+  implementation (merged 2026-09-30) rendered a `dri` hostPath volume at `/dev/dri` plus a read-write
+  mount. It was **measured not to work** in the live Plex pod on k3s-prod-2 on **2026-10-01**. The
+  device node was **visible** in the container — `crw-rw---- root video2 226,128 renderD128` — and
+  **could not be opened**:
+
+  | attempt | result |
+  |---|---|
+  | `uid 0`, `O_RDWR` | **`EPERM`** "Operation not permitted" |
+  | `uid 1000`, `O_RDWR` | **`EPERM`** "Operation not permitted" |
+
+  `EPERM` is the **device cgroup** refusing, not `EACCES` from file permissions — the two look
+  identical from the outside and are not the same problem. Proven three ways: root on the **host**
+  opens the same device `rc=0`; the node is unified `cgroup2fs`; and the container's runc
+  `config.json` reads `linux.resources.devices: [{allow: False, access: 'rwm'}]` with
+  `linux.devices: None` — **one deny-all rule, no allow entry for major 226**. A hostPath is a
+  *filesystem* operation: it binds the device node in but adds **no cgroup allow rule**, because
+  Kubernetes cannot infer one from a volume. **Note that `uid 0` failed too** — root does not bypass
+  the cgroup device gate, so "run it as root" is not a workaround. Neither is `privileged: true`:
+  Kyverno `restrict-host-namespaces` is **Enforce** on app namespaces and rejects it.
+  The device plugin is what produces the allow rule — confirmed in the v0.37.0 source,
+  `createDeviceSpecsFromDrmFiles` emits `DeviceSpec{…, Permissions: "rw"}`, which the kubelet turns
+  into both the device injection and the matching cgroup allow.
+- **THE FAILURE MODE THIS TRADED INTO IS WORSE IN BLAST RADIUS, AND IT IS UNAVOIDABLE.** Stated
+  plainly rather than sold:
+
+  | | device/resource unavailable ⇒ |
+  |---|---|
+  | **Before** (hostPath, `DirectoryOrCreate`) | kubelet mounts an empty dir; Plex runs and **silently software-transcodes**. Invisible, but harmless to everything else. |
+  | **Now** (extended resource) | the pod is **Unschedulable**. `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so this **stalls the app tier**. |
+
+  There is **no graceful-degradation variant**: an extended resource is a hard scheduling
+  constraint, and it is also the only thing that produces the cgroup allow rule. The workload's
+  schedulability is now **coupled to the `intel-gpu-plugin` DaemonSet** — a plugin that is broken,
+  removed, or merely not yet reconciled takes the app tier with it. That is the price of hardware
+  transcoding working at all.
+  - **Escape hatch, one field and no code change:** delete the `hardware:` block from the
+    ServiceDefinition (`catalog/plex.yaml`) and re-render. The workload returns to software
+    transcoding and the tier unblocks. `runbooks/plex-quicksync.md` carries it as a first-class
+    procedure, because it is what someone will need at 2am.
+  - The old mechanism's cost is **gone**, and that is the one improvement: a missing device is no
+    longer silent. There is still no metric and no alert (media apps here render no
+    `metricsEndpoint`, so no ServiceMonitor exists), but "Plex is Unschedulable" is loud, whereas
+    "Plex is quietly software-transcoding" never was.
+- **`podSecurityContext.supplementalGroups` is deliberately NOT rendered, and adding it would be a
+  regression rather than a hardening.** This survives the mechanism swap unchanged, and it was
+  re-measured against the plugin's behaviour. `renderD128` is `root:render` `0660`, so the obvious
+  move is to push the host `render` GID in — and it does not work. **Measured in the running
+  container on prod-2 (2026-09-30, re-measured 2026-10-01):** the image's own
+  `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` hook `find`s every char device under
+  `/dev/dri`, `stat -c '%g'` each one's GID, `groupadd`s a matching group *inside the container* when
+  none exists, and `usermod -a -G` the app user into it — every start, nothing hardcoded. It does
+  exactly the same thing to a **plugin-injected** device as to a bind-mounted one, which is why this
+  swap needed no group work. `cont-init.d` completes before `services.d`, so the account database is
+  already correct when the service execs. The service then starts via `exec s6-setuidgid plex …`, and
+  `s6-setuidgid` **replaces** the process group list from `/etc/group` rather than appending to it —
+  measured live, `Plex Media Server` ran with `Groups: 44 100 992 1000`, where **992 is the host
+  `render` GID**, while root's gid `0` (which PID 1, `s6-svscan`, does hold) was **gone**. A
+  Kubernetes `supplementalGroups` therefore reaches `s6-svscan` and is discarded before the app ever
+  starts. It would be inert *and* it would mislead whoever debugs this next. The renderer asserts its
+  own absence in `render.py --selftest`.
+  - A corollary worth having: because the hook re-`stat`s the device on every start, the host GID
+    **drifting** (a node rebuild renumbering the *systemd dynamic* `render` group) is self-healing.
+    No GID is pinned anywhere in this repo.
+  - **Scope of that measurement, stated so nobody over-reads it.** "The group is the image's job" was
+    established for an image that resolves the device group itself. A FUTURE consumer that runs
+    non-root and has no equivalent init hook would still need the group solved somehow — and the
+    answer for that case is a **renderer** change (per-image handling), not a new ServiceDefinition
+    field. The enum is what keeps that option open, and the 2026-10-01 swap is the proof it works.
+    The first such consumer must re-measure rather than assume this bullet covers it.
+- **Deliberately NOT rendered** from this field: no `privileged: true` (Kyverno
+  `restrict-host-namespaces` is Enforce on app namespaces and would reject it, and it would not have
+  helped anyway — see the `EPERM` table); no `nodeSelector` (placement is a separate concern with its
+  own justification — this field is about the device, not about where the pod lands; the extended
+  resource does constrain placement to nodes the plugin advertises, but that is the **scheduler's**
+  doing, not a selector this renderer writes); no CPU/memory requests or limits (see above — the
+  `resources` block carries the extended resource and nothing else).
+- **Historical, kept so a reader recognises it rather than reintroduces it:**
+  `type: DirectoryOrCreate` on the retired hostPath was load-bearing *for that mechanism* — it is
+  what made a GPU-less node degrade to software transcoding instead of failing the mount and
+  stalling the app tier. That rationale died with the hostPath; it is **not** an argument for bringing it back, since
+  the mechanism it protected never granted access in the first place.
+- Absent ⇒ **nothing at all** is rendered; every service that does not opt in stays byte-identical
+  (no `resources` key, no volume, no mount, no `securityContext`).
+  Ignored for `kind: vm` — a VM receives devices through KubeVirt's own `domain.devices`.
+- **Requirements:** **R-PLAT-02** (the PaaS provides the host capability; the app declares need).
+
 ### `ingressRoutes[]` (`host` / `port`) — HTTPRoute + Certificate + DNS
 - Each entry renders an **HTTPRoute** attaching `host` → the service `port` on the shared Gateway.
 - TLS: the Gateway terminates with the cert-manager **wildcard `*.home.keithjackson.dev`** certificate
@@ -202,7 +320,7 @@ platform must render, and cites the requirement each mapping satisfies. Changing
   undeclared), **R-GITOPS-02** (the allowance is authored in the one reviewable object rather than
   surviving as a live-applied cluster object). Auditable via Hubble (R-SEC-03, R-OBS-04).
 
-### `auth` (`mode` / `groups[]`) — identity integration (Keycloak / oauth2-proxy)
+### `auth` (`mode` / `groups[]` / `apiRoutes[]`) — identity integration (Keycloak / oauth2-proxy)
 - `oidc` → register the app as a **Keycloak OIDC client** (declarative realm/client config, GitOps;
   client secret SOPS-encrypted). App does its own login redirect.
 - `forward-auth` → protect the service's `ingressRoutes` at the **Gateway** with **oauth2-proxy →
@@ -213,6 +331,20 @@ platform must render, and cites the requirement each mapping satisfies. Changing
 - `groups[]` → the Keycloak groups permitted access, enforced by the OIDC client mappers /
   oauth2-proxy `allowed_groups`. **Schema requires ≥1 group for `oidc`/`forward-auth`** — access is
   "in an authorized group," not merely "logged in."
+- `apiRoutes[]` (**`forward-auth` only**) → one repeatable **`--api-route=<regex>`** per entry on
+  the per-app oauth2-proxy sidecar, making matching paths answer **401** instead of a **302** to
+  Keycloak. Default `[]` renders **no flag at all**, which is the common case. It exists because
+  oauth2-proxy's built-in AJAX guard fires only on an exact `Accept: application/json`, which
+  these apps' JavaScript does not consistently send: the XHR is 302'd, the browser follows it
+  invisibly, the live SSO session satisfies it silently, and the call resolves 200 carrying the
+  app's **HTML** where the JavaScript expected JSON — and a POST following a 302 is re-issued as
+  a GET with its **body dropped**. It is **per-service and must never become a renderer
+  constant**: the path shape belongs to the app (radarr `/signalr/`, jackett `/api/`, pihole
+  none), and a wrong constant does not merely fail to help — a path a **browser navigates to**,
+  answered 401, shows a logged-out operator a bare error page and never offers a sign-in. Each
+  entry must be **anchored at `^/`** and continue with literal path characters; `^/`, `^/.*`,
+  `.*`, an unanchored `/api/` and any alternation are **rejected** by the schema and by the
+  renderer (alternation is refused rather than supported because the flag is repeatable).
 - Keycloak, oauth2-proxy, and app→Keycloak / proxy→Keycloak / Keycloak→DB flows are ordinary
   workloads under default-deny with explicit allows (ties back to `dependencies`).
 - **Requirements:** **R-IDP-02** (protect OIDC-capable and bare UIs), **R-IDP-06** (group-based
@@ -271,6 +403,12 @@ platform must render, and cites the requirement each mapping satisfies. Changing
 ## Cross-field invariants the renderer / L0-L1 tests rely on
 1. **kind ↔ source:** container⇒`image`, vm⇒`vmTemplate` (schema-enforced).
 2. **auth ↔ groups:** `oidc`/`forward-auth` ⇒ non-empty `groups` (schema-enforced; R-IDP-06).
+2a. **auth ↔ apiRoutes:** a non-empty `auth.apiRoutes` ⇒ `auth.mode: forward-auth`
+   (schema-enforced). It renders onto the per-app oauth2-proxy sidecar, and no sidecar exists for
+   `oidc`/`native`/`none`, so declaring it elsewhere would be a **silent no-op** — the operator
+   believes the app's XHRs now answer 401 and nothing changed. Every entry must be anchored at
+   `^/` (schema-enforced **and** enforced in `catalog/render.py`, so a breadth mistake is caught by
+   `render-golden` even on a run where the schema stage did not execute).
 3. **default-deny is unconditional:** every namespace gets the deny baseline regardless of
    `dependencies`; `dependencies` only *adds* allows (R-NET-01/02). The same holds for
    `inClusterClients`: it adds one namespace+port ingress allow and never relaxes the baseline.
@@ -286,6 +424,18 @@ platform must render, and cites the requirement each mapping satisfies. Changing
    `external-dns.alpha.kubernetes.io/controller: none`, and no `public` service's route carries it.
    Derived from `exposure` alone; enforced on every gate run by
    `conformance/tests/l0-static/internal-services-publish-no-public-dns` (R-DNS-03).
+8. **`hardware` is inert until opted into, and its mechanism is renderer-local — now DEMONSTRATED,
+   not merely claimed.** An SD with no `hardware` key renders no `resources` block, no volume and no
+   volumeMount; an SD *with* it gains a `gpu.intel.com/i915: 1` **limit on the app container only**
+   (never a sidecar) and no `securityContext` either. All directions are proven by
+   `render.py --selftest`, and each assertion is mutation-tested. The resource name
+   `gpu.intel.com/i915` appears in `catalog/render.py` and nowhere else in the catalog path, so the
+   capability's implementation can be replaced without touching a single ServiceDefinition — which
+   is exactly what happened on **2026-10-01**, when the mechanism moved from a `/dev/dri` hostPath
+   (measured non-functional: device-cgroup `EPERM`) to the device plugin's extended resource with
+   **zero** catalog edits. The cost of that mechanism is recorded in §`hardware`: the workload is now
+   **Unschedulable** when the resource is absent, which stalls the app tier, and the escape hatch is
+   to delete the `hardware:` block and re-render.
 
 ## Requirement traceability (contract-level)
 Primary **R-GITOPS-02** (single reviewable object wires every requirement). Field-level: R-NET-01/02

schemas/servicedefinition.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
   "title": "ServiceDefinition",
-  "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
+  "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked proLine truncated
   "type": "object",
   "additionalProperties": false,
   "required": ["name", "source"],
@@ -139,6 +139,19 @@
         "supplementalGroups": { "type": "array", "items": { "type": "integer" }, "description": "Additional GIDs added to the first process in each container." }
       }
     },
+    "hardware": {
+      "type": "object",
+      "description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume and a matching mount on the app container, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "gpu": {
+          "type": "string",
+          "description": "GPU capability class this service needs. `intel-quicksync` = the host's Intel Quick Sync Video render device (the iGPU's /dev/dri/renderD* node) for hardware video transcode/decode. Enum, not a free-form device path, so the satisfying mechanism stays swappable (see the parent description). Extend the enum when a genuinely different capability class appears (e.g. nvidia-nvenc), never to name a different path to the same one.",
+          "enum": ["intel-quicksync"]
+        }
+      }
+    },
     "exposure": {
       "type": "string",
       "description": "internal = LAN-only (internal DNS + Gateway on LAN); public = additionally edge port-forwarded 80/443 (docs/09; R-DNS/R-NET).",
@@ -311,6 +324,16 @@
           "items": { "type": "string", "minLength": 1 },
           "uniqueItems": true,
           "default": []
+        },
+        "apiRoutes": {
+          "type": "array",
+          "description": "forward-auth ONLY: path regexes this app serves as a MACHINE API, which the oauth2-proxy sidecar must answer with 401 instead of a 302 to Keycloak (one --api-route per entry; the flag is repeatable and singular -- --api-routes does not exist in v7.8.1 and the proxy refuses to start with it). WHY THIS IS PER-APP AND NOT A RENDERER CONSTANT: oauth2-proxy's built-in AJAX guard fires only on an EXACT `Accept: application/json`, which these apps' JavaScript does not consistently send, so an unauthenticated XHR gets a 302 that the browser follows invisibly, the live Keycloak SSO session satisfies silently, and the call resolves 200 carrying the app's HTML where the JavaScript expected JSON -- no prompt, no error, and on a write it is worse, because a browser following a 302 re-issues a POST as a GET and DROPS THE BODY. Measured on jackett 2026-09-30: 33 x 302 against 2 x 401 on /api/ paths, with 4 of 6 POSTs answered by the proxy and never reaching the app (access-log upstream field `-`). Measured on radarr 2026-10-01, 23h after the cookie fix deployed: 232 x 302, ALL of them POST /signalr/messages/negotiate from the operator's own browser -- a left-open tab whose cookie the rename orphaned, polling forever, silently redirected every time and NEVER re-prompting. But the path shape is a property of the APP, not of the pattern: radarr's broken path is /signalr/ and jackett's is /api/, and pihole serves no /api/ at all, so a renderer-wide constant would apply one app's path shape to five. Declare only paths this app actually serves as JSON/RPC and that a browser never NAVIGATES to. BREADTH IS THE DANGER: too broad and a plain page load answers 401 instead of signing the operator in, i.e. the service looks permanently broken to a logged-out user and no sign-in is ever offered. The item pattern therefore REQUIRES each entry to be anchored at `^/` and to continue with literal path characters, which rejects `^/`, `^/.*`, `.*` and any unanchored or alternated regex; alternation is refused rather than supported because the flag is repeatable, so one regex per route costs nothing and keeps each entry readable. N/A for oidc/native/none (no sidecar is rendered).",
+          "items": {
+            "type": "string",
+            "pattern": "^\\^/[A-Za-z0-9._~/-]+(\\.\\*|\\$)?$"
+          },
+          "uniqueItems": true,
+          "default": []
         }
       }
     },
@@ -458,6 +481,26 @@
         }
       }
     },
+    {
+      "$comment": "auth.apiRoutes only has a consumer under forward-auth: it renders --api-route onto the per-app oauth2-proxy sidecar, and no sidecar is rendered for oidc/native/none. Declaring it elsewhere would be a SILENT NO-OP -- the operator believes the app's XHRs now answer 401 and nothing changed -- so it is rejected at the schema instead (R-IDP-02).",
+      "if": {
+        "required": ["auth"],
+        "properties": {
+          "auth": {
+            "required": ["apiRoutes"],
+            "properties": { "apiRoutes": { "minItems": 1 } }
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "auth": {
+            "required": ["mode"],
+            "properties": { "mode": { "const": "forward-auth" } }
+          }
+        }
+      }
+    },
     {
       "$comment": "public exposure must not be unauthenticated: an explicit auth.mode 'none' is disallowed on public services (docs/13 v1.1, docs/15, R-IDP-02). Absent auth defaults to forward-auth at render time, so it is permitted here.",
       "if": {