docs(auth): record what PR #30's CI actually ran, not what its tick said

Read the commit's /statuses rather than the check mark:

  gate / gate (pull_request)         state=success  desc='Successful in 1m41s'
  gate / conformance (pull_request)  state=success  desc='Has been skipped'

The second is CLAUDE.md 2b's trap in the wild -- a SKIPPED Gitea job reports
state=success, so that green tick certifies nothing. It agrees exactly with
the local rc 3: the offline L0/L1 subset passed, L2/L3 never ran. Anyone
reading the PR's check mark alone would conclude the opposite.

Also re-measured branch protection instead of trusting the table in CLAUDE.md:
GET /repos/.../branches/main -> protected=true, user_can_push=false.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>

catalog/render.py

@@ -133,24 +133,6 @@ OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1"
 OAUTH2_PROXY_PORT = 4180
 KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
 
-# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------
-# The ServiceDefinition declares the CAPABILITY (schemas/servicedefinition.schema.json
-# `hardware.gpu`); everything below is the MECHANISM that currently satisfies it and is
-# deliberately confined to this file. Swapping to the Intel GPU device plugin
-# (`gpu.intel.com/i915` resource requests instead of a hostPath) is a change here and
-# nowhere else -- no catalog entry moves.
-DRI_VOLUME_NAME = "dri"
-DRI_DEVICE_DIR = "/dev/dri"
-# GID of the host `render` group, which owns /dev/dri/renderD128 (mode 0660, root:render).
-# MEASURED 2026-09-30 on ALL THREE prod nodes (HP EliteDesk 800 G5 Mini, i7-9700T, UHD
-# Graphics 630, i915 bound -- not xe): GID 992, group name `render`, identical on each.
-# This is a *systemd dynamic* allocation (systemd-udev creates `render` out of the dynamic
-# system range at first boot), NOT a Debian/LSB static GID, so it is NOT guaranteed stable:
-# a node rebuild, or a reinstall that creates system groups in a different order, can move
-# it. If hardware transcode stops working after a node rebuild, re-measure
-# `getent group render` on every node before assuming anything else.
-INTEL_RENDER_GID = 992
-
 
 # =============================================================================
 # Defaults (renderer-contract.md "Default posture") — materialize BEFORE mapping.
@@ -501,30 +483,6 @@ def r_deployment(s: dict) -> dict:
                 "persistentVolumeClaim": {"claimName": f"{s['name']}-{vol['name']}"},
             }
         )
-    # hardware capability (schema-optional): `hardware.gpu: intel-quicksync` means this
-    # workload needs the host iGPU's render node for hardware video transcode. The SD says
-    # only that; the wiring below is the platform's current mechanism for satisfying it.
-    #
-    # `type: DirectoryOrCreate` is DELIBERATE and load-bearing -- do NOT "tighten" it to
-    # `Directory`. A node that comes back without the render device (rebuild, BIOS change,
-    # driver regression, hardware swap) must degrade the workload to SOFTWARE transcoding,
-    # not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` dependsOn it, so a
-    # failed hostPath mount stalls the entire app tier for the sake of a feature that is not
-    # load-bearing. `DirectoryOrCreate` turns that failure mode into "transcodes in
-    # software" instead of "the app tier does not reconcile". /dev is tmpfs, so the empty
-    # directory the kubelet creates on such a node leaves nothing durable behind.
-    #
-    # The mount is deliberately NOT readOnly: VA-API opens the render node read-write.
-    hw_gpu = (s.get("hardware") or {}).get("gpu")
-    if hw_gpu == "intel-quicksync":
-        volume_mounts.append({"name": DRI_VOLUME_NAME, "mountPath": DRI_DEVICE_DIR})
-        volumes.append(
-            {
-                "name": DRI_VOLUME_NAME,
-                "hostPath": {"path": DRI_DEVICE_DIR, "type": "DirectoryOrCreate"},
-            }
-        )
-
     if volume_mounts:
         container["volumeMounts"] = volume_mounts
 
@@ -547,19 +505,8 @@ def r_deployment(s: dict) -> dict:
     # pod spec. Its primary use is `fsGroup`, which makes a mounted Longhorn PVC group-writable by a
     # non-root image (e.g. semaphore runs as UID 1001 and cannot otherwise write its config PVC).
     # Absent => the key is never set, so apps that declare none render byte-identical.
-    #
+    if s.get("podSecurityContext"):
-    # `hardware.gpu: intel-quicksync` additionally needs the host `render` GID in
+        pod_spec["securityContext"] = s["podSecurityContext"]
-    # supplementalGroups, because /dev/dri/renderD128 is root:render 0660 and the container
-    # does not run as root. It is MERGED into whatever the SD already declares rather than
-    # replacing it -- an SD that sets its own supplementalGroups keeps every one of them.
-    psc = dict(s.get("podSecurityContext") or {})
-    if hw_gpu == "intel-quicksync":
-        groups = list(psc.get("supplementalGroups") or [])
-        if INTEL_RENDER_GID not in groups:
-            groups.append(INTEL_RENDER_GID)
-        psc["supplementalGroups"] = groups
-    if psc:
-        pod_spec["securityContext"] = psc
 
     if volumes:
         pod_spec["volumes"] = volumes
@@ -2493,85 +2440,6 @@ def _selftest() -> int:
         "an SD without podSecurityContext must not gain a pod securityContext",
     )
 
-    # 6b) hardware.gpu: intel-quicksync -> /dev/dri hostPath volume + RW mount + the host
-    #     `render` GID. The SD declares the CAPABILITY; these assertions pin the MECHANISM
-    #     that currently satisfies it, so a change to it is a deliberate, visible edit.
-    hw_docs = render(
-        {
-            "name": "hw",
-            "source": {"image": "a:1"},
-            "hardware": {"gpu": "intel-quicksync"},
-        }
-    )
-    hw_pod = next(
-        d[1] for d in hw_docs if d[1]["kind"] == "Deployment"
-    )["spec"]["template"]["spec"]
-    check(
-        {
-            "name": "dri",
-            "hostPath": {"path": "/dev/dri", "type": "DirectoryOrCreate"},
-        }
-        in hw_pod.get("volumes", []),
-        "hardware.gpu=intel-quicksync must render a /dev/dri hostPath volume "
-        "with type DirectoryOrCreate (a GPU-less node must degrade, not fail the mount)",
-    )
-    hw_mounts = hw_pod["containers"][0].get("volumeMounts", [])
-    check(
-        {"name": "dri", "mountPath": "/dev/dri"} in hw_mounts,
-        "hardware.gpu=intel-quicksync must mount /dev/dri READ-WRITE on the app container",
-    )
-    check(
-        hw_pod.get("securityContext", {}).get("supplementalGroups")
-        == [INTEL_RENDER_GID],
-        "hardware.gpu=intel-quicksync must add the host render GID to supplementalGroups",
-    )
-    # Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce)
-    # rejects `privileged: true` in an app namespace, and placement is a separate concern
-    # with its own justification -- this field is about the DEVICE only.
-    check(
-        "privileged" not in hw_pod.get("securityContext", {})
-        and "securityContext" not in hw_pod["containers"][0]
-        and "nodeSelector" not in hw_pod,
-        "hardware.gpu must emit no privileged securityContext and no nodeSelector",
-    )
-    # The regression that matters: absence renders NOTHING. Mirrors the podSecurityContext
-    # negative above -- an SD that does not opt in must be byte-identical to before.
-    nohw_pod = next(
-        d[1] for d in render({"name": "nohw", "source": {"image": "a:1"}})
-        if d[1]["kind"] == "Deployment"
-    )["spec"]["template"]["spec"]
-    check(
-        "volumes" not in nohw_pod
-        and "volumeMounts" not in nohw_pod["containers"][0]
-        and "securityContext" not in nohw_pod,
-        "an SD without `hardware` must gain no dri volume, no mount and no securityContext",
-    )
-    # Merge, not clobber: an SD that declares its own supplementalGroups keeps them AND
-    # gains the render GID, and its other podSecurityContext keys survive untouched.
-    hwm_pod = next(
-        d[1]
-        for d in render(
-            {
-                "name": "hwm",
-                "source": {"image": "a:1"},
-                "hardware": {"gpu": "intel-quicksync"},
-                "podSecurityContext": {
-                    "fsGroup": 1001,
-                    "supplementalGroups": [44],
-                },
-            }
-        )
-        if d[1]["kind"] == "Deployment"
-    )["spec"]["template"]["spec"]
-    check(
-        hwm_pod["securityContext"] == {
-            "fsGroup": 1001,
-            "supplementalGroups": [44, INTEL_RENDER_GID],
-        },
-        "hardware.gpu must MERGE the render GID into the SD's own supplementalGroups "
-        "(both GIDs, fsGroup preserved), never clobber them",
-    )
-
     # 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that
     #     fronts the Gateway path (fail-closed), while oidc/native/none apps are unchanged.
     fa_docs = render(

decisions/2026-09-30-jackett-oauth2-proxy-session.md

@@ -0,0 +1,273 @@
+---
+date: 2026-09-30
+branch: fix/jackett-oauth2-proxy-session
+title: Jackett — radarr's oauth2-proxy deletes jackett's session cookie; the fix cannot be made per-app
+---
+
+## What changed
+
+Reported as *"GET requests work, but most PUT/POST requests to add indexers fail. Reloading the page
+never fixes it, and I am never re-prompted for login if it is a login issue."*
+
+**This branch lands the diagnosis and a runbook. It does NOT land the manifest fix, and that is a
+deliberate stop, not an omission** — see *The fix has one home and it is shared*, below.
+
+Three mechanisms, all measured. **Weighting matters here, so it is stated up front:** mechanisms 2
+and 3 are demonstrated from Jackett's own access log and **between them fully explain the report as
+written** — PUT/POST fail, the operator is never re-prompted, and a reload does not help. Mechanism 1
+is a real, firing defect that Jackett genuinely violates, and it explains the *intermittency* on top;
+I demonstrate the mechanism and one three-second causal adjacency, not a population of them. Fix all
+three; do not rest the case on the first.
+
+**1. Cookie collision — the mechanism is firing, and Jackett is a genuine violator.** Six oauth2-proxy
+instances (seven pods) emit oauth2-proxy's **default** cookie name `_oauth2_proxy` scoped to
+`.home.keithjackson.dev` while holding seven **different** cookie-secrets. The ping-pong is in two
+pods' own logs, interleaved to the second:
+
+    15:23:17  radarr   [AuthSuccess]                                   -> radarr writes _oauth2_proxy
+    15:23:44  jackett  "cookie signature not valid, removing session"  -> re-login #1
+    15:23:58  radarr   "cookie signature not valid, removing session"  -> radarr DELETES jackett's
+    15:27:17  jackett  [AuthSuccess]                                   -> re-login #2
+    15:27:59  radarr   "cookie signature not valid, removing session"
+    15:31:06  jackett  [AuthSuccess]                                   -> re-login #3
+    15:31:29  radarr   "cookie signature not valid, removing session"  <- deletes it again
+    15:31:32  jackett  "No valid authentication in request"            <- 3 SECONDS later, no cookie
+
+`removing session` is oauth2-proxy calling ClearSession, which expires the cookie on **its own**
+`--cookie-domain` — the whole zone. So jackett's session is not merely overwritten, it is **deleted
+by a process in another namespace**, 26 s after a clean login whose own `expires` was five minutes
+out. That is the answer to the open question in the brief ("26 s is well inside the 5-minute window,
+so plain expiry does not explain it"): it was never expiry, and it was never Jackett.
+
+It also explains the half of the report that looked like user error. **Reloading genuinely does not
+help**, because the operator's Radarr tab re-clears the cookie seconds after each new login — and
+Radarr is not a passive tab. Its UI polls: of 156 unauthenticated requests in its log against
+**1** AuthSuccess, the top paths are `/api/v3/command` (62), `/api/v3/queue/details` (46) and
+`/signalr/messages/negotiate` (43). Every one of those polls, arriving with a cookie Radarr cannot
+decrypt, calls ClearSession again. **A left-open Radarr tab is a cookie-deletion engine firing every
+few seconds**, which is why Jackett's sessions die at irregular, short, apparently random intervals.
+
+**A counter-hypothesis was put to me and it does not survive bounding.** The observation was that
+Crafty already carries this fix yet showed the platform's worst failure ratio (81:1 over 24 h from
+Loki), implying the fix does not work. But Crafty's pod started **2026-09-30T03:57:19Z with 0
+restarts**, so a 24 h window is ~10 h of the *pre-fix* pod. Rather than bound it in Loki I read the
+ratio off the pod's own log, whose boundary is the process start and therefore exact: **inside the
+fixed pod's entire 13 h 52 m lifetime, Crafty logged 3 `No valid authentication` events, not 81** —
+from one client IP, two of them WebSocket `/ws` reconnects, in two isolated minutes (04:02 and
+15:22), against 0 AuthSuccess because nobody signed in. Post-fix Crafty is idle, not broken. The
+ratio metric is confounded by endpoint type — apps that poll or hold WebSockets (Radarr, Crafty)
+manufacture failure lines at their retry rate — so it ranks *chattiness*, not breakage. Correctly
+bounded, Crafty is evidence **for** the fix.
+
+**2. Session lifetime pinned to a 5-minute access token.** Measured from the sidecar's own
+`[AuthSuccess]`: `created:15:31:06.391860254` / `expires:` + **300.000 s** exactly, with
+`refresh_token:true` on the same line. `--cookie-refresh` is unset and its default is `0`
+(disabled), so the proxy holds a refresh token it has no mandate to spend.
+
+**3. XHRs answered with 302 instead of 401 — why none of it was visible.** oauth2-proxy's built-in
+AJAX guard fires only on an **exact** `Accept: application/json`, which Jackett's jQuery does not
+consistently send. Measured on `/api/` paths in this pod's log: **33 × 302 against 2 × 401**, and the
+split runs straight through Jackett's own POSTs (`/api/v2.0/server/adminpassword` → 401,
+`/api/v2.0/server/update` → 302). A 302 on an unauthenticated XHR is followed invisibly by the
+browser, satisfied silently by the live Keycloak SSO session, and resolves with HTML where the
+JavaScript expected JSON — no prompt, no error. On a write it is worse: a browser following a 302
+re-issues a POST as a **GET and drops the body**, so the indexer configuration is discarded with
+nothing logged as a failure anywhere.
+
+### The fix has one home, and it is shared — this is the blocker
+
+The brief assigned me `apps/wave-0/jackett.yaml` as exclusively mine, and assigned the other four
+apps to a sibling branch. **That split is not implementable, and I found out by trying it.**
+
+`apps/wave-0/jackett.yaml` is a **generated golden**. The sidecar's argument list is built by one
+shared function, `_oauth2_sidecar_container()` in `catalog/render.py`. `catalog/jackett.yaml`
+declares only `auth.mode` and `auth.groups`, and the `auth` schema is `additionalProperties: false`
+— **there is no per-app override hook**. I wrote the fix into the rendered file first; the merge
+gate rejected it exactly as designed:
+
+    [fail] GOLDEN DRIFT: apps/wave-0/jackett.yaml does not match the canonical render
+    [fail] stage render-golden: FAIL          ->  MERGE GATE: BLOCKED   (rc 1)
+
+So I made the change at its real source instead and **measured the blast radius** before committing
+to it. A change confined to that one function rewrites the sidecar block in **all five** rendered
+apps, and `render-golden` requires all five be regenerated in the same commit:
+
+    CHANGED: apps/wave-0/jackett.yaml
+    CHANGED: apps/wave-0/pihole.yaml
+    CHANGED: apps/wave-0/qbittorrent.yaml
+    CHANGED: apps/wave-1/radarr.yaml
+    CHANGED: apps/wave-1/sonarr.yaml
+
+Four of those five are the files I was explicitly told not to touch, and all four are being edited
+right now on `fix/oauth2-proxy-cookie-collision-fleet`. **The gate and the ownership boundary are in
+direct contradiction**: I cannot fix Jackett's sidecar without regenerating four files another
+branch owns, and two branches rewriting the same generated block across five files is precisely the
+cross-branch conflict this repo's §7 exists to prevent.
+
+I stopped rather than force it. **The cookie fix is one change, not five**, and it should be re-cut
+as a single renderer commit on one branch. The intended content, validated below:
+
+    --cookie-name=_oauth2_proxy_{name}        # per-app, derived
+    --cookie-domain={host}                    # host-scoped, derived
+    --whitelist-domain=.{ZONE}                # unchanged: redirect targets, not cookie scope
+    --cookie-refresh=2m30s                    # half the measured 300 s token lifetime
+    (drop --pass-access-token, --set-xauthrequest)
+
+`--api-route=^/api/` is the one flag I would **not** apply renderer-wide on this evidence: I
+measured the 302/401 split on Jackett only, and PiHole's UI does not serve `/api/` at all. It needs
+either a per-app hook or four more measurements.
+
+**Metrics — asked for, and declined here for the same structural reason, not on merit.** It is true
+and I re-measured it: **no per-app sidecar exposes metrics**. Only the shared proxy carries
+`--metrics-address=0.0.0.0:44180` and a `metrics` container port; jackett, pihole, qbittorrent,
+radarr, sonarr and crafty all have `(ABSENT)` and ports `['http']` alone, so the listener never
+starts and nothing about these gates is scrapable. Adding it is worth doing and the pattern is
+already proven in-tree by the shared proxy. But it is **not** "inside my file": the container's
+`args` *and* its `ports` both come from `_oauth2_sidecar_container()`, so adding
+`--metrics-address` plus a `metrics` port hits the identical five-file blast radius as the cookie
+change. It belongs in the same single renderer commit — which is an argument for cutting that commit
+soon, not for me to hand-edit a golden a second time after the gate already refused the first.
+(Crafty is separately hand-written and could take the metrics flag on its own.)
+
+**Crafty is unaffected by all of this** — its sidecar is hand-written in
+`platform/crafty/deployment.yaml`, not rendered, which is why it could be and was fixed alone on
+2026-09-29.
+
+### What this branch does land
+
+`runbooks/oauth2-proxy-session.md` — the four sidecar log lines that tell Keycloak, the sidecar and
+the app's own login apart (including the upstream `-` field that means the proxy answered a request
+the app never saw), the cookie-collision cross-log diagnosis, the 5-minute-session check, the
+302-vs-401 trap, and the generated-golden warning with its measured blast radius. Registered in the
+runbooks `configMapGenerator` **and** listed in `runbooks/README.md`, so the URL resolves rather
+than the file merely existing.
+
+## What I verified
+
+- **The collision is the live failure.** `[stored_session.go:94] Error loading cookied session:
+  cookie signature not valid, removing session` — **1 occurrence in jackett's log (15:23:44), 8 in
+  radarr's** (15:22:49, 15:23:58, 15:27:59, **15:31:29**, 15:32:29 ×4), and **zero** in pihole,
+  qbittorrent, sonarr, crafty and both shared-proxy replicas. The 15:31:29 → 15:31:32 pair is the
+  operator's whole report.
+- **Six instances / seven pods are in the collision set** — the brief said five; the shared
+  forward-auth proxy in namespace `oauth2-proxy` is the sixth and runs 2 replicas. Read from live
+  pod specs: jackett, pihole, qbittorrent, radarr, sonarr and `oauth2-proxy` all have **no**
+  `--cookie-name` argument and `--cookie-domain=.home.keithjackson.dev`. Only crafty is fixed.
+- **All seven cookie-secrets differ.** Compared in-cluster by sha256, first 12 hex only, values
+  never printed: `d9c5…`/`0b3b…`/`b025…`/`bb6a…`/`25f4…`/`e679…`/`911a…`. So no two of these proxies
+  can read each other's cookie — the precondition for the eviction.
+- **The 5-minute lifetime**, from four `[AuthSuccess]` lines: `expires − created = 300.000 s`, with
+  `refresh_token:true` and `groups:[platform-admins]`. Live args confirm `--cookie-refresh` absent.
+- **The 302/401 split** on `/api/` paths in this pod's log: 33 × 302, 18 × 200, 2 × 401, 2 × 204.
+- **4 of the 6 POSTs never reached Jackett** — the access log's upstream field is `-`, not `/`:
+  `adminpassword` → 401, `adminpassword` → 401, `server/update` → 302, `server/update` → 302. The
+  two that did reach it (`adminpassword`, `indexers/1337x/config`) carry the authenticated email.
+- **The proposed argument list starts, against the pinned image and the real issuer.** Ran
+  `quay.io/oauth2-proxy/oauth2-proxy:v7.8.1` locally with the full proposed args; OIDC discovery
+  against `keycloak.home.keithjackson.dev` succeeded and it printed back its own reading:
+  `Cookie settings: name:_oauth2_proxy_jackett secure(https):true httponly:true expiry:168h0m0s
+  domains:jackett.home.keithjackson.dev path:/ samesite: refresh:after 2m30s` and
+  `API route - Path: ^/api/`.
+- **`--api-route` is singular.** The brief specified `--api-routes`, which does not exist in v7.8.1;
+  the proxy would have refused to start. Read off `--help`.
+- **`--session-cookie-minimal` is mutually exclusive with `--cookie-refresh`** — provoked, not
+  assumed: `invalid configuration: cookie_refresh > 0 requires oauth tokens in sessions.
+  session_cookie_minimal cannot be set`. This kills the brief's premise that dropping
+  `--pass-access-token` shrinks the cookie: only `--session-cookie-minimal` strips tokens from the
+  session, and it is unavailable while refresh is on. Dropping those two flags is justified on
+  **least privilege and inertness**, not size.
+- **`--set-xauthrequest` is inert here**: HTTPRoute `jackett-9117` backendRef is
+  `Service/jackett port 4180`, there is no ext-authz attachment on the `shared` Gateway, so
+  `/oauth2/auth` is never called.
+- **`apps/**` is generated and the fix cannot be scoped to one app** — both halves measured, not
+  inferred: the gate's `render-golden` stage rejected the hand-edit, and re-rendering after a
+  renderer-only change listed all five apps as changed (transcript above).
+- **Crafty's post-fix failure rate, bounded exactly.** Pod `crafty-647c7d8cc5-5spv9`,
+  `startTime=2026-09-30T03:57:19Z`, `restarts=0`, live args `--cookie-name=_oauth2_proxy_crafty` and
+  `--cookie-domain=crafty.home.keithjackson.dev`. Over that pod's whole lifetime: **3**
+  `No valid authentication`, 0 AuthSuccess, one client IP, 2 of the 3 on `/ws`. The 81:1 figure is a
+  24 h window over a 13.9 h pod and is dominated by the pre-fix process.
+- **The ratio metric ranks chattiness, not breakage.** Radarr 156:1 with the top paths being
+  `/api/v3/command`, `/api/v3/queue/details` and `/signalr/messages/negotiate` — UI polling, not 156
+  distinct user-visible failures.
+- **No per-app sidecar exposes metrics.** `--metrics-address` is `(ABSENT)` and `ports` is
+  `['http']` on all six per-app sidecars; only the shared proxy has
+  `--metrics-address=0.0.0.0:44180` and a `metrics` port.
+- **Gate:** `./ci/gate.sh` in the pinned toolbox, **rc 3 — UNDETERMINED, not green**. Green:
+  `spec-immutability`, `approval-policy`, `decisions-log`, `lint`, `l0`, `render-golden`, `l1`.
+  Skipped: `gitea-settings` (no `SETTINGS_API_TOKEN`) and L2/L3 conformance (`CLUSTER_AVAILABLE`
+  unset), leaving touched requirement `R-OBS-04` uncertified. Note `spec-immutability` is **green**,
+  not skipped — mounting the sibling spec repo at `/home-network-k3s` (the path
+  `ci/spec-immutability.sh` actually defaults to) in the ENGINE shim fixes that skip; the commonly
+  circulated shim mounts it at its host path, where `SPEC_REPO` never finds it.
+- **Gitea CI on this PR, read from the commit's `/statuses` rather than from a tick.** Two contexts
+  reported, and **only one of them ran**:
+
+      gate / gate (pull_request)         state=success   desc='Successful in 1m41s'
+      gate / conformance (pull_request)  state=success   desc='Has been skipped'
+
+  That second line is §2b's trap in the wild: a **skipped** Gitea job reports `state=success`. The
+  conformance job did not execute, so the green tick beside it certifies nothing. Read together with
+  the local rc 3 above, the two agree exactly — the offline L0/L1 subset passed and L2/L3 never ran.
+  Anyone reading the PR's check mark alone would conclude otherwise.
+- **`main` is protected**, re-measured this session rather than assumed:
+  `GET /repos/.../branches/main` → `protected=true`, `user_can_push=false`.
+
+## What I did NOT verify
+
+- **The operator's problem is NOT FIXED by this branch.** Only the diagnosis and the runbook land.
+  Jackett's sidecar is unchanged and will keep losing its session to Radarr until the renderer
+  change ships. **This is the single most important line in this entry.**
+- **No browser round-trip.** Every mechanism is measured from logs and specs and the argument list
+  is proven to start, but nobody has added an indexer through the proposed configuration. The human
+  test is below, for whoever lands the renderer change.
+- **Nothing was applied.** No `kubectl apply/edit/patch`, no `flux reconcile`. The running pod is
+  still `jackett-c5bfcbfb8-c75f9` (2/2 Running, 17 d, 0 restarts — unchanged by this session's
+  read-only probes). Note that when the fix does land, merging is what deploys, and it will restart
+  the pod (`strategy: Recreate` on an RWO Longhorn volume), briefly interrupting Radarr's and
+  Sonarr's direct `:9117` access.
+- **Cookie size and chunking: UNMEASURED, and I will not claim either way.** I could not read a
+  `Set-Cookie` header without a browser. I found no `_oauth2_proxy_0`/`_1` chunk evidence and no
+  4 kB warning in 36,287 log lines — but I did **not** establish that v7.8.1 emits such a warning at
+  this log level, so that absence is not evidence. If sessions still misbehave after the fix lands,
+  this is the next thing to measure, with `--session-store-type=redis` as the escape hatch, since
+  `--session-cookie-minimal` is ruled out (above).
+- **`--cookie-refresh=2m30s` has never executed a refresh here.** It depends on Keycloak accepting
+  the refresh token and on the realm's SSO Session Idle/Max, none of which I read (that needs admin
+  credentials). If Keycloak's *refresh* token is also short-lived, the five-minute logout persists
+  and the realm is where to look next.
+- **`--api-route` is measured for Jackett only.** PiHole and qBittorrent were not measured, and
+  PiHole's UI does not serve `/api/`. Do not apply it renderer-wide on my evidence.
+- **The four sibling apps still collide with each other** regardless of what happens to Jackett.
+  The same operator report should be expected from any of pihole / qbittorrent / radarr / sonarr.
+- **L2/L3 conformance did not run** (`CLUSTER_AVAILABLE` unset), so nothing about runtime behaviour
+  is certified by the gate. `gitea-settings` also skipped, so branch protection and runner state are
+  **unverified by me this session** — I did not re-measure them against the Gitea API.
+- **I did not query Loki.** The Crafty bounding above comes from the pod log, which I argue is the
+  better instrument (an exact process boundary rather than a wall-clock window against a 72 h
+  retention). But it means I cannot speak to the 24 h or 12 h *platform-wide* ratios, and a
+  per-app ratio over a window longer than the current pod's life is not something I measured at all.
+- **Collision causation is one adjacency, not a population.** The 15:31:29 → 15:31:32 pair is three
+  seconds and the mechanism is documented and present, but I did not enumerate every jackett failure
+  and match it to a radarr ClearSession. Treat mechanism 1 as **necessary and firing, not proven
+  sufficient** — the coordinator's framing, and I agree with it on this evidence.
+- **The metrics change is NOT in this branch** (reason above). Jackett therefore remains inside the
+  oauth2-proxy metrics gap, and the ratio alert shipped in PR #29 still cannot see it.
+- **The runbook's commands were not all executed verbatim.** The log-reading ones I ran; the
+  per-namespace `for` loops are edited from the scripted forms I actually used.
+
+### The human test — for whoever lands the renderer fix
+
+After it merges and the pod rolls (watch for a **new** pod name at 2/2):
+
+1. **Sign in** at `https://jackett.home.keithjackson.dev/UI/Dashboard`.
+2. **Open Radarr in a second tab** and sign in there too. **This is the step that breaks Jackett
+   today** — it is the 15:31:29 line. Return to the Jackett tab.
+3. **Add an indexer** — *+ Add indexer*, pick one, *Configure*, *Okay*. It must save.
+4. **Wait six minutes on the open dashboard, then save another.** That is the `--cookie-refresh`
+   test; the session currently dies at five.
+5. **If something does still fail, it must now fail LOUDLY** — DevTools → Network, a broken `/api/`
+   call shows **401**, not a 302 followed by HTML. A silent success-looking no-op means the change
+   did not take; check the running pod's args before assuming anything else.
+6. **A plain page reload must still sign you in silently** (`/UI/Dashboard` is outside `^/api/`).
+   A 401 page on reload means the regex is too broad — that is a regression.

decisions/2026-09-30-sd-hardware-capability.md

@@ -1,147 +0,0 @@
----
-date: 2026-09-30
-branch: feat/sd-hardware-capability
-title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` renders the /dev/dri wiring, and the enum is what keeps the mechanism swappable
----
-
-## What changed
-
-A new optional top-level ServiceDefinition property, `hardware`, with one optional member,
-`gpu`, whose enum today has exactly one value: `intel-quicksync`. **No catalog entry uses it.**
-This PR delivers the capability; consuming it (Plex) is separate work on top.
-
-- `schemas/servicedefinition.schema.json` — adds `hardware` (object, `additionalProperties: false`,
-  `minProperties: 1`) with `gpu` (enum `["intel-quicksync"]`). Schema version bumped **1.7.0 →
-  1.8.0**, changelog entry prepended to the schema `description` per that file's own convention.
-- `catalog/render.py` — three module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`,
-  `INTEL_RENDER_GID = 992`) and the emission in `r_deployment`.
-- `schemas/renderer-contract.md` — new `### hardware` field-mapping section, plus cross-field
-  invariant **8** in that file's numbered list.
-- `schemas/README.md` — its `**Schema version:**` line said `1.0.0`; it had been stale since 1.1.0.
-  Synced to 1.8.0 and annotated that the schema `description` is the changelog of record. Added one
-  bullet to "Cross-field rules" for the new enum. That was the whole of the staleness; nothing
-  ballooned.
-
-**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container / pod spec
-only (never a sidecar):
-
-1. a hostPath volume `dri` → `/dev/dri`, `type: DirectoryOrCreate`;
-2. a read-write `volumeMount` of `dri` at `/dev/dri` on the app container (deliberately not
-   `readOnly` — VA-API opens the render node read-write);
-3. `INTEL_RENDER_GID` **merged into** `podSecurityContext.supplementalGroups`, never clobbering
-   what the SD already declares.
-
-It emits **no** `privileged: true` (Kyverno `restrict-host-namespaces` is Enforce on app namespaces
-and it is not needed), **no** `nodeSelector` (placement is a separate concern with its own
-justification) and **no** resource requests (there is no app-container resource render path).
-
-**The design decision worth defending: the field is an enum of intent, not a device path.** The SD
-says *what class of host device the workload needs*; the hostPath, the mount and the GID exist only
-in `catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource
-requests, no hostPath at all) is then a renderer change with zero catalog edits. Both the schema
-`description` and the contract section say this explicitly, because the obvious "simplification" —
-letting an SD name `/dev/dri` — would push a platform implementation detail into the operator-facing
-contract and make that migration a catalog-wide edit.
-
-**`DirectoryOrCreate` is load-bearing.** A node that comes back without the render device (rebuild,
-BIOS change, driver regression, hardware swap) must degrade the workload to software transcoding,
-not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so a failed
-hostPath mount stalls the entire app tier for the sake of a feature that is not load-bearing.
-`/dev` is tmpfs, so the empty directory the kubelet creates on such a node leaves nothing durable
-behind. Do not "tighten" it to `Directory`. The reasoning is recorded at the emission site.
-
-## What I verified
-
-Measured, in this worktree, on this branch:
-
-- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** Three new cases
-  were added in the existing `check(...)` style: the positive (volume + mount + GID, and *no*
-  privileged/nodeSelector), the negative (an SD without `hardware` gains no volume, no volumeMount
-  and no `securityContext`), and the merge case (`supplementalGroups: [44]` + the field →
-  `[44, 992]`, with `fsGroup` preserved).
-- **The self-tests actually bite.** I mutated the guard (`if hw_gpu == "intel-quicksync":` → `if
-  True:`) in a scratch copy and re-ran: rc 1 with three named failures, including "an SD without
-  `hardware` must gain no dri volume, no mount and no securityContext". A test that cannot fail is
-  not a test; this one can.
-- **Re-render is a zero diff.**
-  `python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` then
-  `git diff --stat apps/` → **empty**, and `git status --porcelain apps/` → **empty** (so the two
-  not-onboarded by-products, `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, were
-  rewritten byte-identically and needed no revert). The change is inert until a service opts in.
-- **Schema validity and catalog conformance.** `Draft202012Validator.check_schema` passes; all
-  **12** `catalog/*.yaml` validate with **0** errors. On the new field specifically: `gpu:
-  intel-quicksync` accepted; `gpu: /dev/dri` rejected (enum); `hardware: {gpu: …, path: …}` rejected
-  (`additionalProperties: false`); `hardware: {}` rejected (`minProperties: 1`).
-- **Hand-checked the emitted YAML** by rendering a throwaway SD carrying `hardware`, a Longhorn
-  volume and its own `podSecurityContext: {runAsUser: 1000, supplementalGroups: [44]}`. Result:
-  `volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `securityContext` =
-  `{runAsUser: 1000, supplementalGroups: [44, 992]}`; `volumes` = the PVC then
-  `{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`. That toy SD was never written
-  into `catalog/`.
-- **GATE: `rc 3` — UNDETERMINED, NOT green.** Run in the pinned toolbox from this worktree with
-  both CLAUDE.md §3 workarounds (`DEV_KUBECONFIG=/dev/null`, and an `ENGINE` shim bind-mounting the
-  main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f` … `HEAD @
-  9f0db1495cfd`; touched requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Per stage:
-
-  | stage | result |
-  |---|---|
-  | spec-immutability | **SKIPPED** — sibling spec repo not at `/work/../home-network-k3s`. Expected from a worktree; tolerated by §3. |
-  | decisions-log | pass |
-  | lint | pass |
-  | l0 | pass |
-  | render-golden | pass — *"all 10 onboarded service golden(s) reproduce from the canonical render"* |
-  | l1 | pass — 3 offline checks, `0 failure(s), 0 skip(s), 0 unwired`; coverage 51/51 requirements scaffolded |
-  | approval-policy | pass (`event='none'`, not a pull_request event — N/A locally; it will evaluate for real on the PR) |
-  | gitea-settings | **SKIPPED** — `SETTINGS_API_TOKEN` unset. Expected; branch protection and runner registration are therefore UNVERIFIED by this run. |
-  | conformance (L2/L3) | **SKIPPED** — `CLUSTER_AVAILABLE!=true`. Phase-0 standing skip. |
-
-  The gate's own verdict: *"Stages that did not run: spec-immutability gitea-settings"*. **No stage
-  failed, and no stage skipped for a reason caused by this change.** rc 3 is reported as rc 3; I am
-  not calling this green.
-- **Blast radius of the schema change:** `hardware` is absent from every catalog entry, so
-  `apply_defaults` never materializes it and `s.get("hardware")` is `None` on all 12 — which is what
-  the zero diff above demonstrates rather than asserts.
-- **Touched no sensitive path.** Changes are confined to `schemas/`, `catalog/`, `decisions/`; per
-  `infra/gitea/approval-policy.json` none of those are sensitive. `conformance/**` and `clusters/**`
-  are untouched. I did not run `--emit-waves` (it writes `clusters/base/`, and the set of files
-  under `apps/` did not change).
-
-## What I did NOT verify
-
-- **Whether a non-privileged pod can actually `open()` this hostPath char device at all. This is
-  the largest technical risk in the workstream and nobody has tested it.** `supplementalGroups` gets
-  the DAC permission bits right, but containerd installs a **device cgroup** allow-list per
-  container, and whether `/dev/dri/renderD128` reaching the container as a bind-mounted device node
-  is also permitted by that cgroup on this k3s/containerd version is **unmeasured**. If it is not,
-  the mount will succeed, the group will be present, and the `open()` will still fail `EPERM` — and
-  the design would need the device plugin (which sets the cgroup entry) rather than a hostPath. The
-  first real consumer proves or disproves this; nothing in this PR does.
-- **That a missing device degrades silently — and nothing will tell you.** This is the accepted cost
-  of `DirectoryOrCreate`. The workload will software-transcode and **no signal reports it**: media
-  apps in this catalog declare no `metricsEndpoint`, so no ServiceMonitor exists, and there is no
-  metric to alert on. I deliberately did **not** propose an alert, because an alert that cannot fire
-  is worse than none — see the 2026-09-13 correction in `renderer-contract.md` §`observability`,
-  where twelve monitors discovered zero targets. The only proof of hardware transcode is an operator
-  seeing it in the application's own dashboard.
-- **Whether GID 992 survives a node rebuild.** It was measured on all three prod nodes on
-  2026-09-30 and was identical on each, but `render` is a *systemd dynamic* group allocation, not a
-  Debian/LSB static GID. A reinstall that creates system groups in a different order can move it,
-  and nothing in this repo would notice. Recorded in the constant's comment; re-measure `getent
-  group render` on every node after any rebuild.
-- **Whether `supplementalGroups` is the load-bearing mechanism or belt-and-braces.** A concurrent
-  agent was checking whether the pms-docker image's `45-plex-hw-transcode-and-connected-tuner` init
-  hook resolves the render group dynamically at startup. **No finding reached me**, so the constant's
-  comment describes the GID and its provenance without claiming which mechanism does the work.
-  Update that comment if the answer arrives.
-- **Anything on the live cluster.** I ran no `kubectl`. The node facts above (device present,
-  `root:render` 0660, GID 992, `i915` bound, no device plugin, no `gpu.intel.com/*` advertised) and
-  the Kyverno posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups`
-  restriction; no PSA labels on app namespaces) are **prior recon handed to me, not re-measured by
-  me**. Per CLAUDE.md §2b, treat them as second-hand until re-queried.
-- **That CI exercises the self-tests. It does not.** `git grep -- --selftest ci/` returns nothing —
-  no gate stage invokes the renderer's self-test. They are the executable spec and house style, and
-  I ran them by hand, but a future regression in this code path **will not be caught by the gate**.
-  Wiring `--selftest` into a gate stage is a real gap and is not part of this PR (`ci/` is a
-  sensitive path and would drag in a human reviewer for a change unrelated to this one).
-- **Runtime behaviour of any kind.** The gate proves the L0/L1 offline subset only;
-  `vars.CLUSTER_AVAILABLE` is unset, so L2/L3 conformance does not run in CI.

runbooks/README.md

@@ -115,6 +115,14 @@ Each affected runbook says so on its own page and gives the by-hand equivalent.
   plex.tv/claim — so it is a timed sequence, scripted as `infra/plex/claim-ship.sh`:
   token → `sops` → PR → gate → merge → **forced** `flux reconcile` → pod roll (~2m45s measured).
   Includes why a changed Secret does not restart anything, and why an expired token is harmless.
+- **`oauth2-proxy-session`** — a gated UI loads but its saves quietly do nothing, and you are never
+  re-prompted to log in. Tells the three stacked auth layers apart from the sidecar's own access log
+  (the four lines that matter, including the upstream `-` that means the proxy answered a request the
+  app never saw). Covers the cookie collision between per-app sidecars sharing the default
+  `_oauth2_proxy` name on `.home.keithjackson.dev` — which deletes sessions belonging to apps in
+  *other* namespaces and is only legible with two logs side by side — the 5-minute access-token
+  session with no `--cookie-refresh`, and why an XHR answered with a 302 instead of a 401 loses a
+  POST body silently.
 - **`crafty-admin-credential`** — you signed into Keycloak and Crafty *still* asks for a password.
   Expected: Crafty 4 has no OIDC, so its own local login sits behind the oauth2-proxy gate as a second
   factor. The page tells the two layers apart from the sidecar log, and explains that the bootstrap

runbooks/kustomization.yaml

@@ -85,6 +85,11 @@ configMapGenerator:
       - node-io=node-io.md
       - node-memory=node-memory.md
       - node-memory-swap=node-memory-swap.md
+      # Operator procedure, not alert-linked. Reading the per-app oauth2-proxy sidecar log to tell
+      # Keycloak, the sidecar and the app's own login apart. Cookie collision between two sidecars
+      # presents as random session loss in ONE app's log and is only legible with both side by side,
+      # so the page has to be reachable at 3am without going looking for it in git.
+      - oauth2-proxy-session=oauth2-proxy-session.md
       # Operator procedure, not an alert page (no alert links to it — the
       # alert-runbooks-resolvable check treats that as advisory). It is served
       # because the plex.tv claim window is short and is not a thing to go looking

runbooks/oauth2-proxy-session.md

@@ -0,0 +1,195 @@
+# oauth2-proxy-session — telling the auth layers apart from the sidecar log
+
+**Not alert-driven.** This is an operator procedure for the per-app oauth2-proxy sidecars
+(`jackett`, `pihole`, `qbittorrent`, `radarr`, `sonarr`, `crafty`) and the shared forward-auth
+proxy in namespace `oauth2-proxy`. Use it when a gated UI behaves as though you are signed in and
+signed out at the same time: the page loads, but its buttons and saves quietly do nothing.
+
+> The sidecar log contains **email addresses, session UUIDs and full callback URLs including an
+> OAuth `code`**. Read it, do not paste it. (Repo rule §6: secrets never enter a transcript.)
+
+## What this means here
+
+Every gated app on `home.keithjackson.dev` has **at least two** independent authentications
+stacked on it, and the symptom rarely names the one that refused:
+
+1. **Keycloak** — the `homelab` realm issues the token. Failures here are visible: you see a login
+   form, or an error page from Keycloak's own hostname.
+2. **The oauth2-proxy sidecar** — validates the session cookie and the group claim
+   (`media-users` / `platform-admins`). **This is the layer that fails silently**, and the rest of
+   this page is about reading it.
+3. **The app's own login**, where it has one — Crafty and Radarr/Sonarr each present a local login
+   *behind* the Keycloak gate. See `runbooks/crafty-admin-credential.md` for that layer.
+
+The sidecar's access log distinguishes all three, if you know the four lines to look for. Get it:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n jackett logs \
+      -l app.kubernetes.io/name=jackett -c oauth2-proxy --tail=-1'
+
+Substitute the namespace and label for the app you are chasing. Always pass `--context k3s-prod`:
+`dev.sh` merges both clusters and picks a current-context **per invocation**, and that default has
+already changed once. Do not run two `dev.sh` invocations at the same time.
+
+### The four lines, and what each one means
+
+| Log line | What actually happened | Which layer |
+|---|---|---|
+| `[AuthSuccess] Authenticated via OAuth2: Session{... groups:[...]}` | Keycloak issued a token and the proxy accepted it. **The IdP is fine.** | Keycloak ✔ |
+| `[oauthproxy.go:1026] No valid authentication in request. Initiating login.` | **No session cookie was presented at all.** Either you never had one, or something *deleted* it — see below. | sidecar |
+| `[stored_session.go:94] Error loading cookied session: cookie signature not valid, removing session` | A cookie **of the right name** arrived that this proxy **could not decrypt**. Almost always a *different* proxy's cookie. Note `removing session`: this proxy then **deletes** it. | sidecar, cookie collision |
+| an upstream path of `-` instead of `/` in an access-log line | the **proxy answered the request itself**; it never reached the app. Compare `POST - "/api/..."` against `POST / "/api/..."`. | sidecar |
+
+That last column is the one people miss. The upstream field sits between the method and the quoted
+path. `-` means the proxy handled it (a redirect, a 401, its own `/oauth2/*` endpoints); `/` means
+it was proxied to the app. Count them:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n jackett logs \
+      -l app.kubernetes.io/name=jackett -c oauth2-proxy --tail=-1 \
+      | grep -v "\"/ping\"" | grep -E " (POST|PUT|DELETE) " | cut -c1-140'
+
+If your writes show `-`, the app never saw them. On 2026-09-30, **4 of the 6 POSTs** in jackett's
+log had `-`.
+
+## First commands
+
+**1. Did Keycloak ever succeed?** No `AuthSuccess` at all means this is not a cookie problem —
+go to `runbooks/keycloak-down.md` or `runbooks/keycloak-client-secrets.md`.
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n <ns> logs \
+      -l app.kubernetes.io/name=<app> -c oauth2-proxy --tail=-1 | grep -c AuthSuccess'
+
+**2. How long does a session last?** Subtract `created` from `expires` on an `AuthSuccess` line.
+If it is exactly **5m**, the session's lifetime is Keycloak's access-token lifetime and the proxy
+is not refreshing it:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n <ns> logs \
+      -l app.kubernetes.io/name=<app> -c oauth2-proxy --tail=-1 \
+      | grep AuthSuccess | tail -1 | grep -o "created:[^ ]* [^ ]*\|expires:[^ ]* [^ ]*"'
+
+The same line says `refresh_token:true` — the proxy is *holding* a refresh token. It will only
+spend it if `--cookie-refresh` is set; the default is `0`, meaning disabled.
+
+**3. Is this a cookie collision?** Two checks, and you want both.
+
+*Names and scopes* — every instance that emits the default name `_oauth2_proxy` on the shared
+`.home.keithjackson.dev` domain is in the collision set with every other one:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get pods -A -o json \
+      | grep -E "cookie-name|cookie-domain"'
+
+An instance with **no** `--cookie-name` argument is using the default. That is the failure — the
+flag being absent, not wrong.
+
+*The smoking gun* — search **every** proxy, not just the one you are debugging, for the removal
+line, and put the timestamps next to each other:
+
+    infra/devcontainer/dev.sh run bash -c 'for ns in jackett pihole qbittorrent radarr sonarr crafty; do \
+      echo "== $ns"; kubectl --context k3s-prod -n $ns logs -l app.kubernetes.io/name=$ns \
+      -c oauth2-proxy --tail=-1 2>/dev/null | grep stored_session; done'
+
+**4. Do the cookie-secrets actually differ?** They should — but confirm the collision is between
+instances that genuinely cannot read each other. Compare **hashes**, never values:
+
+    infra/devcontainer/dev.sh run bash -c 'for ns in jackett pihole qbittorrent radarr sonarr crafty; do \
+      printf "%s " "$ns"; kubectl --context k3s-prod -n $ns get secret $ns-oauth2-proxy \
+      -o jsonpath="{.data.cookie-secret}" | sha256sum | cut -c1-12; done'
+
+Never print `.data` or `.stringData` itself. A password has been leaked into a transcript here
+before.
+
+### Do not rank apps by their failure ratio
+
+Counting `No valid authentication` against `AuthSuccess` per app looks like a health metric and is
+not one. **It ranks chattiness, not breakage.** An app whose UI polls or holds a WebSocket
+manufactures one failure line per retry, so a single dead cookie in one idle browser tab becomes
+scores of log lines, while an app nobody is using scores zero no matter how broken it is. Measured
+2026-09-30: Radarr showed **156** unauthenticated requests against **1** `AuthSuccess`, but the top
+paths were `/api/v3/command` (62), `/api/v3/queue/details` (46) and `/signalr/messages/negotiate`
+(43) — polling, not 156 user-visible failures.
+
+**Bound the window to the current pod, not to wall-clock.** A ratio measured over 24 h across a pod
+that restarted 13 h ago is mostly about the *previous* configuration, and that is exactly how a
+deployed fix gets written off as ineffective. The pod's own log has an exact boundary — its process
+start — so read it there:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n <ns> get pod \
+      -l app.kubernetes.io/name=<app> -o jsonpath="{.items[*].status.startTime}{\"\n\"}"'
+
+then count within that same pod's log. On 2026-09-30 this took Crafty from an apparent 81:1 over 24 h
+to **3 failures in its fixed pod's entire 13 h 52 m life**, two of them `/ws` reconnects.
+
+## Likely causes, most likely first
+
+1. **Cookie collision between two per-app proxies.** Several instances emit the default
+   `_oauth2_proxy` on `.home.keithjackson.dev` with different `cookie-secret`s. Whoever
+   authenticates last overwrites the rest; the losers log `cookie signature not valid, removing
+   session` and — because `removing session` clears the cookie on *their* `--cookie-domain`, which
+   is the whole zone — **delete a session belonging to an app in another namespace**. The victim
+   then logs `No valid authentication` with no cookie at all, seconds after a clean login, while
+   its own `expires` is still minutes away. **Read both logs side by side; one app's log alone
+   makes this look like random session loss.** Fix: a unique `--cookie-name` and a host-scoped
+   `--cookie-domain` on the per-app sidecar. The shared forward-auth proxy keeps the zone-wide
+   scope — cross-host auth is its job.
+2. **Session pinned to a 5-minute access token with no refresh.** `--cookie-refresh` unset means
+   the proxy never spends the refresh token it holds, so the session expires with the access token.
+   Set it to about half the measured token lifetime.
+3. **An XHR got a 302 instead of a 401.** The app's JavaScript follows the redirect invisibly, the
+   live Keycloak SSO session satisfies it without a prompt, and the XHR resolves with **HTML where
+   it expected JSON**. Nothing is logged as an error and the operator is never re-prompted — this
+   is what "it just doesn't save, and reloading doesn't help" looks like. Worse on a write: a
+   browser following a 302 re-issues a POST as a GET and **drops the body**, so the change is
+   discarded silently. oauth2-proxy's built-in AJAX guard only fires on an exact
+   `Accept: application/json`, which most app JavaScript does not send; make it deterministic with
+   `--api-route=<regex>` (**singular** — `--api-routes` does not exist and the proxy refuses to
+   start). Keep browser-navigable paths *out* of the regex so a plain page load still signs you in.
+4. **Group membership.** An `AuthSuccess` followed by `403` is the `--allowed-group` gate, not a
+   session problem. Check the `groups:[...]` field on the `AuthSuccess` line.
+5. **The app's own second login.** Crafty, Radarr and Sonarr each have one. `AuthSuccess` followed
+   by the app's own `/login` page is layer 3, not layer 2.
+
+## When to escalate or stop
+
+- **Stop before you `kubectl edit`/`patch`/`apply` anything.** Flux reconciles `main`; a
+  hand-applied flag is drift that the next reconcile reverts or, worse, adopts. Argument changes
+  go through a branch and a PR. Reading logs is always safe; changing the Deployment is not an
+  agent action.
+- **Do not hand-edit `apps/**/<app>.yaml` — those files are GENERATED.** The five per-app sidecars
+  (jackett, pihole, qbittorrent, radarr, sonarr) get their argument list from one shared function,
+  `_oauth2_sidecar_container()` in `catalog/render.py`; `catalog/<app>.yaml` declares only
+  `auth.mode` and `auth.groups`, and the `auth` schema is `additionalProperties: false`, so there
+  is **no per-app override hook**. A hand-edit to the rendered file is caught by the merge gate's
+  `render-golden` stage ("GOLDEN DRIFT ... does not match the canonical render") and would be
+  erased by the next render anyway. Change the renderer, then regenerate:
+
+      python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation
+
+  **Budget for the blast radius**: because the function is shared, any change to it rewrites the
+  sidecar block in **all five** rendered apps, and the gate requires all five be regenerated in
+  the same commit. There is no way to change one app's sidecar arguments alone. Measured
+  2026-09-30. Coordinate before starting — this is a single change, not five.
+- **Crafty is the exception** and is not rendered from the catalog: its sidecar is hand-written in
+  `platform/crafty/deployment.yaml`, so it can be, and was, fixed independently.
+- **You cannot currently alert on any of this**, and that is worth knowing before you promise
+  monitoring. Measured 2026-09-30: no per-app sidecar sets `--metrics-address` and none declares a
+  `metrics` container port, so oauth2-proxy's metrics listener never starts — the only oauth2-proxy
+  series in Prometheus come from the shared proxy in namespace `oauth2-proxy`, which does set both.
+  Until the renderer emits them for the per-app sidecars, every diagnosis on this page is
+  log-reading only.
+- **Validate an argument change before you open the PR**, not after. The pinned image will tell
+  you what it understood, without touching the cluster:
+
+      podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help
+
+  Running it with your full proposed argument list prints a `Cookie settings:` line and an
+  `API route - Path:` line back at you, and rejects mutually exclusive flags outright
+  (`--session-cookie-minimal` with `--cookie-refresh`, for one). A flag typo here is a CrashLoop
+  on a service you have just told someone is fixed.
+- **Do not rotate a `cookie-secret` to "clear" a bad session.** It signs every live session out of
+  that app and does nothing about the collision, which is caused by the *names* matching, not the
+  secrets differing.
+- **Escalate to Keycloak** (`runbooks/keycloak-down.md`, `runbooks/keycloak-client-secrets.md`) if
+  there is no `AuthSuccess` in any proxy's log — that is an IdP or client-secret problem and no
+  cookie flag will help.
+- **Hand off rather than guess** if you see `cookie signature not valid` on a proxy that already
+  has a unique `--cookie-name`. That combination should not occur; it means something is writing
+  that name which is not in this inventory. Record what you measured.

schemas/README.md

@@ -15,7 +15,7 @@ docs/13 exactly. Do not diverge from it here — escalate spec problems, don't f
 | `servicedefinition.schema.json` | Formal JSON Schema (draft 2020-12) for a ServiceDefinition. Enforces enums, required fields, patterns, and cross-field invariants. |
 | `renderer-contract.md` | Authoritative field → rendered-resource mapping, with requirement IDs. The contract other workstreams consume. |
 
-**Schema version:** 1.8.0 (in the schema `description`, which carries the full per-version changelog — this line had said `1.0.0` since 1.1.0 and is now synced). Treat any change as an interface change:
+**Schema version:** 1.0.0 (in the schema `description`). Treat any change as an interface change:
 propose on the task board, update dependent renderers/tests in the same change, never silently break a
 consumer (coordination-protocol §2).
 
@@ -35,9 +35,6 @@ consumer (coordination-protocol §2).
 - `ingressRoutes[].host` must match `<label>.home.keithjackson.dev`.
 - `storage[].class ∈ {longhorn, bulk}`; `exposure ∈ {internal, public}`; `egress ∈ {internet, vpn}`;
   `auth.mode ∈ {oidc, forward-auth, native, none}`.
-- `hardware.gpu ∈ {intel-quicksync}` — a host **capability class**, never a device path. The enum
-  is the contract: the mechanism that satisfies it lives only in `catalog/render.py` (see
-  `renderer-contract.md` §`hardware`).
 - `additionalProperties: false` everywhere — unknown keys are rejected (catches typos in review/CI).
 
 ## Validate examples against the schema — L1, no cluster (R-TEST-02)

schemas/renderer-contract.md

@@ -160,45 +160,6 @@ platform must render, and cites the requirement each mapping satisfies. Changing
 - **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication),
   **R-STATE-04** (bulk tier). (docs/04)
 
-### `hardware` (`gpu`) — host device capability
-- **The field is INTENT, never MECHANISM.** `hardware.gpu` is an **enum of capability classes**
-  (today: `intel-quicksync`), not a device path, not a hostPath, not a resource request. The
-  ServiceDefinition says *what class of host device this workload needs*; how the platform hands it
-  over is `catalog/render.py`'s business and lives nowhere else. **Do not "simplify" this into a raw
-  `/dev/...` path.** The entire payoff is that migrating from the hostPath mechanism below to the
-  Intel GPU device plugin (`resources.limits: gpu.intel.com/i915`) is a renderer change with **zero**
-  catalog edits and zero re-review of per-service device wiring. Extend the enum for a genuinely
-  different capability class (e.g. `nvidia-nvenc`); never to name a second route to the same one.
-- `gpu: intel-quicksync` → on the **app container / pod spec only** (never a sidecar):
-  1. a **hostPath volume** `dri` → `/dev/dri`, **`type: DirectoryOrCreate`**;
-  2. a **read-write volumeMount** of `dri` at `/dev/dri` on the app container (VA-API opens the
-     render node read-write, so it is deliberately not `readOnly`);
-  3. the host `render` **GID merged into `podSecurityContext.supplementalGroups`** — merged, never
-     clobbered, so an SD that declares its own supplemental groups keeps all of them and gains one.
-     `/dev/dri/renderD128` is `root:render` mode `0660`, so a non-root container needs the group.
-     The GID is the single constant `INTEL_RENDER_GID` in `catalog/render.py` (992, **measured
-     2026-09-30 on all three prod nodes**; it is a *systemd dynamic* allocation, not a static GID,
-     so a node rebuild can move it — re-measure `getent group render` before assuming).
-- **`DirectoryOrCreate` is load-bearing — do not tighten it to `Directory`.** A node that comes back
-  without the render device (rebuild, BIOS change, driver regression, hardware swap) must degrade the
-  workload to **software transcoding**, not fail the mount. `apps-wave-0` is `wait: true` and
-  `apps-wave-1` `dependsOn` it, so a failed hostPath mount stalls the **whole app tier** for the sake
-  of a feature that is not load-bearing. `/dev` is tmpfs, so the empty directory the kubelet creates
-  on such a node leaves nothing durable behind.
-  - **The cost of that choice, stated plainly:** a missing device is **silent**. The workload
-    software-transcodes and nothing reports it. There is no metric to alert on — media apps here
-    render no `metricsEndpoint`, so no ServiceMonitor exists — and an alert that cannot fire is worse
-    than none (see §`observability`'s 2026-09-13 correction). The only proof of hardware transcode is
-    an operator observing it in the application's own dashboard.
-- **Deliberately NOT rendered** from this field: no `privileged: true` (Kyverno
-  `restrict-host-namespaces` is Enforce on app namespaces and would reject it, and the render node
-  does not need it); no `nodeSelector` (placement is a separate concern with its own justification —
-  this field is about the device, not about where the pod lands); no resource requests/limits (there
-  is no app-container resource render path, and inventing one here would be scope creep).
-- Absent ⇒ **nothing at all** is rendered; every service that does not opt in stays byte-identical.
-  Ignored for `kind: vm` — a VM receives devices through KubeVirt's own `domain.devices`.
-- **Requirements:** **R-PLAT-02** (the PaaS provides the host capability; the app declares need).
-
 ### `ingressRoutes[]` (`host` / `port`) — HTTPRoute + Certificate + DNS
 - Each entry renders an **HTTPRoute** attaching `host` → the service `port` on the shared Gateway.
 - TLS: the Gateway terminates with the cert-manager **wildcard `*.home.keithjackson.dev`** certificate
@@ -325,11 +286,6 @@ platform must render, and cites the requirement each mapping satisfies. Changing
    `external-dns.alpha.kubernetes.io/controller: none`, and no `public` service's route carries it.
    Derived from `exposure` alone; enforced on every gate run by
    `conformance/tests/l0-static/internal-services-publish-no-public-dns` (R-DNS-03).
-8. **`hardware` is inert until opted into, and its mechanism is renderer-local.** An SD with no
-   `hardware` key renders no volume, no volumeMount and no `securityContext` — proven by
-   `render.py --selftest`. The `/dev/dri` hostPath, the `DirectoryOrCreate` type and
-   `INTEL_RENDER_GID` appear in `catalog/render.py` and nowhere else, so the capability's
-   implementation can be replaced without touching a single ServiceDefinition.
 
 ## Requirement traceability (contract-level)
 Primary **R-GITOPS-02** (single reviewable object wires every requirement). Field-level: R-NET-01/02

schemas/servicedefinition.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
   "title": "ServiceDefinition",
-  "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with a /dev/dri hostPath volume (`DirectoryOrCreate`), a read-write /dev/dri volumeMount on the app container, and the host `render` GID merged into podSecurityContext.supplementalGroups, and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
+  "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
   "type": "object",
   "additionalProperties": false,
   "required": ["name", "source"],
@@ -139,19 +139,6 @@
         "supplementalGroups": { "type": "array", "items": { "type": "integer" }, "description": "Additional GIDs added to the first process in each container." }
       }
     },
-    "hardware": {
-      "type": "object",
-      "description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume + the host's `render` GID in supplementalGroups, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",
-      "additionalProperties": false,
-      "minProperties": 1,
-      "properties": {
-        "gpu": {
-          "type": "string",
-          "description": "GPU capability class this service needs. `intel-quicksync` = the host's Intel Quick Sync Video render device (the iGPU's /dev/dri/renderD* node) for hardware video transcode/decode. Enum, not a free-form device path, so the satisfying mechanism stays swappable (see the parent description). Extend the enum when a genuinely different capability class appears (e.g. nvidia-nvenc), never to name a different path to the same one.",
-          "enum": ["intel-quicksync"]
-        }
-      }
-    },
     "exposure": {
       "type": "string",
       "description": "internal = LAN-only (internal DNS + Gateway on LAN); public = additionally edge port-forwarded 80/443 (docs/09; R-DNS/R-NET).",