diff --git a/apps/wave-0/jackett.yaml b/apps/wave-0/jackett.yaml index 9d2574d..03e1fe5 100644 --- a/apps/wave-0/jackett.yaml +++ b/apps/wave-0/jackett.yaml @@ -100,11 +100,14 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_jackett + - --cookie-domain=jackett.${BASE_DOMAIN} - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -125,6 +128,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/apps/wave-0/pihole.yaml b/apps/wave-0/pihole.yaml index 4d65465..2541c87 100644 --- a/apps/wave-0/pihole.yaml +++ b/apps/wave-0/pihole.yaml @@ -108,11 +108,14 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://pihole.${BASE_DOMAIN}/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_pihole + - --cookie-domain=pihole.${BASE_DOMAIN} - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -133,6 +136,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/apps/wave-0/plex.yaml b/apps/wave-0/plex.yaml index 5c3b21f..2928eaa 100644 --- a/apps/wave-0/plex.yaml +++ b/apps/wave-0/plex.yaml @@ -143,6 +143,8 @@ spec: - name: media mountPath: /media readOnly: true + - name: dri + mountPath: /dev/dri volumes: - name: config persistentVolumeClaim: @@ -150,6 +152,10 @@ spec: - name: media persistentVolumeClaim: claimName: plex-media + - name: dri + hostPath: + path: /dev/dri + type: DirectoryOrCreate --- # Service apiVersion: v1 diff --git a/apps/wave-0/qbittorrent.yaml b/apps/wave-0/qbittorrent.yaml index 6551fcf..e6f19d8 100644 --- a/apps/wave-0/qbittorrent.yaml +++ b/apps/wave-0/qbittorrent.yaml @@ -171,11 +171,14 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_qbittorrent + - --cookie-domain=qbittorrent.${BASE_DOMAIN} - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -196,6 +199,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index e84f565..2cef42c 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -153,11 +153,14 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_radarr + - --cookie-domain=radarr.${BASE_DOMAIN} - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -178,6 +181,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index 5b0d98b..3771672 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -153,11 +153,14 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_sonarr + - --cookie-domain=sonarr.${BASE_DOMAIN} - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -178,6 +181,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/catalog/plex.yaml b/catalog/plex.yaml index caf51b1..0b14b29 100644 --- a/catalog/plex.yaml +++ b/catalog/plex.yaml @@ -80,6 +80,58 @@ env: # remote access are UNVERIFIED. Advertising a wrong/internal-only URL breaks remote access for ALL # clients, so per "prefer safety over completeness" it is left unset. FOLLOW-UP: once remote # reachability through the Gateway is confirmed, set ADVERTISE_IP: "https://plex.home.keithjackson.dev:443". +# HARDWARE TRANSCODING — the host iGPU's render device (schema 1.8.0 `hardware`; R-PLAT-02). +# INTENT ONLY. This says "this workload needs an Intel Quick Sync render device"; it does NOT name a +# device path. The renderer currently satisfies it with exactly two things — a `/dev/dri` hostPath +# volume (`type: DirectoryOrCreate`) and a read-write `/dev/dri` volumeMount on the app container — +# and nothing else: no nodeSelector, no privileged, no resources (schemas/renderer-contract.md +# `### hardware`). If that mechanism is later replaced by the Intel GPU device plugin +# (`gpu.intel.com/i915`), it is a `catalog/render.py` change and THIS BLOCK DOES NOT MOVE. Do not +# "simplify" it into a raw `/dev/...` path; that is the whole point of the enum. +# +# THE DEVICE IS PRESENT ON EVERY PROD NODE, so nothing had to be installed and Plex can land +# anywhere. Measured on all three nodes 2026-09-30: `/dev/dri/renderD128` present, `root:render`, +# GID 992, mode 0660, driver `i915` bound (NOT `xe`), UHD Graphics 630 on an i7-9700T, media engines +# `vcs0`/`vecs0` live, iGPU enabled in BIOS on each. Plex happens to run on k3s-prod-2 today, but +# that is scheduler history rather than a requirement — this field deliberately renders no +# nodeSelector, because placement is a separate concern with its own justification. +# +# NO `podSecurityContext.supplementalGroups`, AND ADDING ONE WOULD BE INERT — do not "fix" this. +# The device is `root:render` 0660 and this process is uid/gid 1000, so the obvious move is to push +# the host `render` GID in as a supplemental group. It cannot work here. MEASURED IN THE RUNNING +# CONTAINER, 2026-09-30: the image's own +# `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` stats every char device under +# `/dev/dri`, `groupadd`s a matching group INSIDE the container when none exists, and +# `usermod -a -G`s the `plex` user into it — every start, nothing hardcoded, which also makes a host +# GID renumbering self-healing (no GID is pinned anywhere in this repo). The service then execs via +# `s6-setuidgid plex`, and `s6-setuidgid` REPLACES the process group list from `/etc/group` rather +# than appending to it: `Plex Media Server` runs with `Groups: 100 1000`, and root's gid `0` — which +# PID 1 (`s6-svscan`) does hold — is GONE. Anything Kubernetes supplies therefore reaches s6-svscan +# and is discarded before Plex ever starts. Mounting the device is sufficient; the group is the +# image's job and it does it correctly. `render.py --selftest` asserts the ABSENCE of a pod +# securityContext for this field precisely so nobody adds an inert one back later. +# +# `DirectoryOrCreate` IS THE SAFE FAILURE MODE, not laziness. A node that comes back without the +# device (rebuild, BIOS reset, driver regression, hardware swap) gets an empty directory and Plex +# degrades to SOFTWARE transcoding. `Directory` would fail the mount instead — and because +# `apps-wave-0` is `wait: true` with `apps-wave-1` `dependsOn` it, that would stall the WHOLE app +# tier for a feature that is not load-bearing. The price, stated plainly: a missing device is +# SILENT. This SD renders no `metricsEndpoint`, so there is no ServiceMonitor, no transcode-mode +# metric, and no alert that could ever fire. The only proof is an operator reading the Plex +# dashboard. +# +# THE LAST MILE IS A MANUAL OPERATOR STEP AND CANNOT LIVE IN THIS REPO. Mounting the device does +# not enable hardware transcoding — Plex's own `HardwareAcceleratedCodecs` preference does, and it +# is ABSENT from `Preferences.xml` today (the toggle has never been on). NO environment variable can +# set it: a grep of the running image's entire init surface (`/etc/cont-init.d`, `/etc/services.d`, +# `/etc/fix-attrs.d`, `/plex-common.sh`, `/usr/local/bin`) found ZERO `PLEX_PREFERENCE*` support, +# and the six prefs the image does write (`MachineIdentifier`, `ProcessedMachineIdentifier`, +# `PlexOnlineToken`, `customConnections`, `allowedNetworks`, `TranscoderTempDirectory`) are all +# unrelated. So there is no GitOps path to the toggle; an operator clicks it once in the web UI. +# `runbooks/plex-quicksync.md` is that procedure, plus how to verify it took and the +# device-cgroup failure mode to expect first. +hardware: + gpu: intel-quicksync exposure: public # edge port-forward + public DNS; clients + Seerr reach it (docs/09; R-DNS/R-NET) egress: internet # normal WAN egress (docs/03; R-NET-05) storage: diff --git a/catalog/render.py b/catalog/render.py index cc6c9a5..3d2db72 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -133,6 +133,45 @@ OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1" OAUTH2_PROXY_PORT = 4180 KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab" +# --- session lifetime ----------------------------------------------------------------- +# Keycloak's `homelab` realm issues access tokens with a 300-SECOND lifetime. MEASURED on a +# live session, not read off a config page: `expires - created = 300.000s` exactly, with +# `refresh_token:true` on the same line — so the proxy HOLDS a refresh token it was never +# told to spend. `--cookie-refresh` defaults to 0, which the v7.8.1 binary's own help spells +# "0 to disable", so every one of these sidecars dropped its session five minutes after +# sign-in and sent the user back to Keycloak. Refresh at HALF the lifetime: two chances to +# renew before expiry, so one failed refresh is not an immediate logout. +# +# NOTE for anyone tempted to add `--session-cookie-minimal` to shrink the cookie: it is +# MUTUALLY EXCLUSIVE with this (provoked: "cookie_refresh > 0 requires oauth tokens in +# sessions"). Dropping --pass-access-token is a least-privilege argument, not a size fix. +OAUTH2_ACCESS_TOKEN_LIFETIME_S = 300 +OAUTH2_COOKIE_REFRESH = "2m30s" + +# --- metrics -------------------------------------------------------------------------- +# oauth2-proxy serves /metrics only when --metrics-address is given; with it absent the +# listener never starts, so NO per-app sidecar was scrapable at all (measured: `ports` was +# `['http']` on all six). 44180 is the platform's existing convention, not a guess — the +# SHARED proxy already runs `--metrics-address=0.0.0.0:44180` with a `metrics` port and 2 +# ready endpoints. The flag is confirmed present in the PINNED v7.8.1 binary (`--help`), +# which matters because the shared proxy is on v7.15.3 and could not have vouched for it. +# +# THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED. A container port is not a +# Service port: these sidecars' Services expose the app port and :4180, and each namespace's +# default-deny baseline has `ingress: []`, so Prometheus can neither resolve nor reach this +# port until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor +# are added. Do not read this flag as "metrics are working". +OAUTH2_PROXY_METRICS_PORT = 44180 + +# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------ +# The ServiceDefinition declares the CAPABILITY (schemas/servicedefinition.schema.json +# `hardware.gpu`); everything below is the MECHANISM that currently satisfies it and is +# deliberately confined to this file. Swapping to the Intel GPU device plugin +# (`gpu.intel.com/i915` resource requests instead of a hostPath) is a change here and +# nowhere else -- no catalog entry moves. +DRI_VOLUME_NAME = "dri" +DRI_DEVICE_DIR = "/dev/dri" + # ============================================================================= # Defaults (renderer-contract.md "Default posture") — materialize BEFORE mapping. @@ -483,6 +522,46 @@ def r_deployment(s: dict) -> dict: "persistentVolumeClaim": {"claimName": f"{s['name']}-{vol['name']}"}, } ) + # hardware capability (schema-optional): `hardware.gpu: intel-quicksync` means this + # workload needs the host iGPU's render node for hardware video transcode. The SD says + # only that; the wiring below is the platform's current mechanism for satisfying it. + # + # `type: DirectoryOrCreate` is DELIBERATE and load-bearing -- do NOT "tighten" it to + # `Directory`. A node that comes back without the render device (rebuild, BIOS change, + # driver regression, hardware swap) must degrade the workload to SOFTWARE transcoding, + # not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` dependsOn it, so a + # failed hostPath mount stalls the entire app tier for the sake of a feature that is not + # load-bearing. `DirectoryOrCreate` turns that failure mode into "transcodes in + # software" instead of "the app tier does not reconcile". /dev is tmpfs, so the empty + # directory the kubelet creates on such a node leaves nothing durable behind. + # + # The mount is deliberately NOT readOnly: VA-API opens the render node read-write. + # + # `podSecurityContext.supplementalGroups` is DELIBERATELY NOT SET here, and adding it + # would be a regression, not a hardening. MEASURED in the running container on prod-2, + # 2026-09-30: the image's own cont-init hook + # `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` `find`s every char device + # under /dev/dri, `stat -c '%g'` each one's GID, `groupadd`s a matching group INSIDE the + # container when none exists, and `usermod -a -G` the app user into it -- every start, + # nothing hardcoded. cont-init.d completes before services.d, so the account database + # s6 reads at exec time already carries the group. The service is then started with + # `exec s6-setuidgid plex ...`, and `s6-setuidgid` REPLACES the process group list from + # /etc/group rather than appending to it: `Plex Media Server` runs with + # `Uid: 1000 Gid: 1000 Groups: 100 1000` -- root's gid 0, which PID 1 (s6-svscan) does + # have, is GONE. Anything Kubernetes puts in supplementalGroups reaches s6-svscan and is + # then discarded before the app ever starts. MOUNTING THE DEVICE IS SUFFICIENT; the + # group handling is the image's job and it does it correctly. A supplementalGroups entry + # here would be inert AND would mislead whoever debugs this next. + hw_gpu = (s.get("hardware") or {}).get("gpu") + if hw_gpu == "intel-quicksync": + volume_mounts.append({"name": DRI_VOLUME_NAME, "mountPath": DRI_DEVICE_DIR}) + volumes.append( + { + "name": DRI_VOLUME_NAME, + "hostPath": {"path": DRI_DEVICE_DIR, "type": "DirectoryOrCreate"}, + } + ) + if volume_mounts: container["volumeMounts"] = volume_mounts @@ -971,7 +1050,11 @@ def _oauth2_sidecar_container(s: dict) -> dict: auth.group (R-IDP-06: a valid session is not enough — the user must be in an authorized group, else /oauth2 returns 403). client-id/secret + cookie-secret come by name from the SOPS-encrypted `-oauth2-proxy` Secret (R-IDP-04). Single provider => --skip-provider-button 302s straight - to Keycloak. The sidecar->app hop is pod-local (127.0.0.1) and not subject to network policy.""" + to Keycloak. The sidecar->app hop is pod-local (127.0.0.1) and not subject to network policy. + + --cookie-name and --cookie-domain are DERIVED per app (see the block comment on those two + args): every instance holds a different cookie-secret, so sharing one cookie name across the + zone makes the instances evict each other's sessions.""" route = s["ingressRoutes"][0] host = route["host"] upstream_port = route["port"] @@ -991,11 +1074,64 @@ def _oauth2_sidecar_container(s: dict) -> dict: args += [ "--skip-provider-button=true", f"--redirect-url=https://{host}/oauth2/callback", - f"--cookie-domain=.{ZONE}", + # ── COOKIE ISOLATION (fleet fix 2026-09-30; crafty's 2026-09-29 fix, generalized) ────── + # EVERY oauth2-proxy on this platform — this sidecar, its four siblings, and the SHARED + # forward-auth proxy (platform/oauth2-proxy) — is an INDEPENDENT instance holding its OWN + # `cookie-secret`. Measured on prod 2026-09-30: seven `*-oauth2-proxy` Secrets, seven + # DISTINCT cookie-secret values, so no instance can decrypt any other's cookie. + # + # Left at oauth2-proxy's DEFAULTS (`_oauth2_proxy` / `_oauth2_proxy_csrf`) and scoped to + # `.${BASE_DOMAIN}`, the browser therefore keeps ONE cookie pair for ALL of them: + # whichever proxy authenticates last OVERWRITES every other's session. The loser logs + # "No valid authentication in request" and bounces to Keycloak, which — the SSO session + # still being live — redirects straight back and overwrites the cookie again. The + # user-visible symptom is "I signed into Keycloak but I'm still not signed in", and it + # sends you hunting in Keycloak, where nothing is wrong. Silent, intermittent, and + # misattributed: the exact shape this repo keeps getting burned by. + # + # WHY IT WAS *INTERMITTENT*, AND WHY RELOADING NEVER HELPED. The loser does not merely + # fail to read the cookie — it calls ClearSession, which EXPIRES the cookie on its own + # --cookie-domain, i.e. the whole zone. And the *arr UIs poll: measured on prod, radarr's + # rejections are `/api/v3/command`, `/api/v3/queue/details` and `/signalr/negotiate`, + # 155 of them in 12h against 1 successful auth. Every poll carrying a cookie radarr + # cannot decrypt calls ClearSession again — so A LEFT-OPEN RADARR TAB DELETES JACKETT'S + # COOKIE EVERY FEW SECONDS, from another namespace, with no user action at all. That is + # the intermittency, and it is why signing in again fixed nothing for longer than the + # gap between two polls. Host-scoping the cookie (below) is what contains the blast + # radius of a ClearSession to the app that issued it. + # + # DERIVED, NOT LISTED. Both flags come from the ServiceDefinition (`s['name']` and the + # route host), so a NEW forward-auth app is isolated the moment it is onboarded. A + # per-service table would leave the next app defaulting into the collision, which is how + # twelve ServiceMonitors acquired one identical wrong port name in a single edit. + # conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct asserts both properties + # over the WHOLE tree and asserts THIS function derives rather than lists. + # + # --cookie-name a unique name per instance, so two sessions cannot alias at all. + # This is the fix; the one below is defence in depth. + # --cookie-domain HOST-scoped. A per-app reverse proxy has no reason to broadcast its + # session cookie to every other host in the zone — a zone-wide cookie + # is transmitted to every sibling app on `.${BASE_DOMAIN}`, so any + # FUTURE name collision would again be platform-wide instead of + # contained. The SHARED forward-auth proxy keeps the wide scope, + # because cross-host forward-auth is precisely its job. + # + # --whitelist-domain deliberately stays ZONE-WIDE: it governs permitted post-login + # REDIRECT targets, not cookie scope. Narrowing it here would break the sign-in bounce. + f"--cookie-name=_oauth2_proxy_{s['name']}", + f"--cookie-domain={host}", f"--whitelist-domain=.{ZONE}", "--cookie-secure=true", + # Spend the refresh token the proxy already holds, at half the 300s access-token + # lifetime. Without this the session dies 5 minutes after sign-in. See + # OAUTH2_ACCESS_TOKEN_LIFETIME_S above for the measurement and the + # --session-cookie-minimal incompatibility. + f"--cookie-refresh={OAUTH2_COOKIE_REFRESH}", "--pass-access-token=true", "--set-xauthrequest=true", + # Start the /metrics listener. Prerequisite only — not yet reachable by Prometheus; + # see OAUTH2_PROXY_METRICS_PORT above for exactly what is still missing. + f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}", ] secret = f"{s['name']}-oauth2-proxy" return { @@ -1017,7 +1153,10 @@ def _oauth2_sidecar_container(s: dict) -> dict: "valueFrom": {"secretKeyRef": {"name": secret, "key": "cookie-secret"}}, }, ], - "ports": [{"name": "http", "containerPort": OAUTH2_PROXY_PORT, "protocol": "TCP"}], + "ports": [ + {"name": "http", "containerPort": OAUTH2_PROXY_PORT, "protocol": "TCP"}, + {"name": "metrics", "containerPort": OAUTH2_PROXY_METRICS_PORT, "protocol": "TCP"}, + ], "resources": { "requests": {"cpu": "25m", "memory": "64Mi"}, "limits": {"memory": "128Mi"}, @@ -2440,6 +2579,83 @@ def _selftest() -> int: "an SD without podSecurityContext must not gain a pod securityContext", ) + # 6b) hardware.gpu: intel-quicksync -> a /dev/dri hostPath volume and a read-write mount + # on the app container, and NOTHING ELSE. The SD declares the CAPABILITY; these + # assertions pin the MECHANISM that currently satisfies it -- including what it + # deliberately does NOT emit -- so a change to it is a deliberate, visible edit. + hw_docs = render( + { + "name": "hw", + "source": {"image": "a:1"}, + "hardware": {"gpu": "intel-quicksync"}, + } + ) + hw_pod = next( + d[1] for d in hw_docs if d[1]["kind"] == "Deployment" + )["spec"]["template"]["spec"] + check( + { + "name": "dri", + "hostPath": {"path": "/dev/dri", "type": "DirectoryOrCreate"}, + } + in hw_pod.get("volumes", []), + "hardware.gpu=intel-quicksync must render a /dev/dri hostPath volume " + "with type DirectoryOrCreate (a GPU-less node must degrade, not fail the mount)", + ) + hw_mounts = hw_pod["containers"][0].get("volumeMounts", []) + check( + {"name": "dri", "mountPath": "/dev/dri"} in hw_mounts, + "hardware.gpu=intel-quicksync must mount /dev/dri READ-WRITE on the app container", + ) + # The device mount is the WHOLE feature: no pod securityContext is invented. The image's + # own cont-init hook adds the app user to the device's group inside the container, and + # s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway (see the + # emission site). Asserting the absence is what stops someone adding an inert one back. + check( + "securityContext" not in hw_pod, + "hardware.gpu must NOT invent a pod securityContext (supplementalGroups is inert " + "under s6-setuidgid; the image's cont-init hook handles the device group)", + ) + # Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce) + # rejects `privileged: true` in an app namespace, and placement is a separate concern + # with its own justification -- this field is about the DEVICE only. + check( + "privileged" not in hw_pod.get("securityContext", {}) + and "securityContext" not in hw_pod["containers"][0] + and "nodeSelector" not in hw_pod, + "hardware.gpu must emit no privileged securityContext and no nodeSelector", + ) + # The regression that matters: absence renders NOTHING. Mirrors the podSecurityContext + # negative above -- an SD that does not opt in must be byte-identical to before. + nohw_pod = next( + d[1] for d in render({"name": "nohw", "source": {"image": "a:1"}}) + if d[1]["kind"] == "Deployment" + )["spec"]["template"]["spec"] + check( + "volumes" not in nohw_pod + and "volumeMounts" not in nohw_pod["containers"][0] + and "securityContext" not in nohw_pod, + "an SD without `hardware` must gain no dri volume, no mount and no securityContext", + ) + # An SD that declares BOTH keeps its own podSecurityContext exactly as written -- the + # hardware field neither adds to it nor rewrites it. + hwm_pod = next( + d[1] + for d in render( + { + "name": "hwm", + "source": {"image": "a:1"}, + "hardware": {"gpu": "intel-quicksync"}, + "podSecurityContext": {"fsGroup": 1001, "supplementalGroups": [44]}, + } + ) + if d[1]["kind"] == "Deployment" + )["spec"]["template"]["spec"] + check( + hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]}, + "hardware.gpu must leave the SD's own podSecurityContext untouched", + ) + # 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that # fronts the Gateway path (fail-closed), while oidc/native/none apps are unchanged. fa_docs = render( @@ -2475,6 +2691,86 @@ def _selftest() -> int: not any(a.startswith("--ssl-upstream-insecure-skip-verify") for a in side["args"]), "http upstream must NOT skip TLS verify (that flag is only for crafty's self-signed https)", ) + # COOKIE ISOLATION. Every oauth2-proxy instance holds its own cookie-secret, so two instances + # sharing a cookie NAME on a shared DOMAIN evict each other's sessions silently. Both flags + # must be DERIVED from this service, not defaulted and not listed. + check( + "--cookie-name=_oauth2_proxy_fa" in side["args"], + "sidecar must declare a cookie name DERIVED from the service name — the oauth2-proxy " + "default (_oauth2_proxy) is shared with every other instance on the zone", + ) + check( + "--cookie-domain=fa.home.keithjackson.dev" in side["args"], + "sidecar cookie-domain must be HOST-scoped (the app's own host), not zone-wide", + ) + check( + f"--whitelist-domain=.{ZONE}" in side["args"], + "whitelist-domain must stay ZONE-wide — it governs post-login redirect targets, not " + "cookie scope; narrowing it breaks the sign-in bounce", + ) + # SESSION LIFETIME. `--cookie-refresh` must be present AND strictly shorter than the + # measured 300s access-token lifetime, or the session dies mid-use. Asserted as a parsed + # duration rather than a string match, so "2m30s" -> "150s" -> "2.5m" stays green and a + # silent widening past the token lifetime does not. + refresh = next((a.split("=", 1)[1] for a in side["args"] + if a.startswith("--cookie-refresh=")), None) + check(refresh is not None, + "sidecar must set --cookie-refresh; the v7.8.1 default is 0 = DISABLED, so the " + "session dies when the access token expires and the user is bounced to Keycloak") + if refresh is not None: + _m = re.fullmatch(r"(?:(\d+)m)?(?:(\d+(?:\.\d+)?)s)?", refresh) + _secs = (int(_m.group(1) or 0) * 60 + float(_m.group(2) or 0)) if _m else -1 + check(0 < _secs < OAUTH2_ACCESS_TOKEN_LIFETIME_S, + f"--cookie-refresh={refresh} parses to {_secs}s, which is not inside " + f"(0, {OAUTH2_ACCESS_TOKEN_LIFETIME_S}s). A refresh at or past the access-token " + f"lifetime never renews in time, which is the same logout it was added to stop") + check( + not any(a.startswith("--session-cookie-minimal") for a in side["args"]), + "--session-cookie-minimal is MUTUALLY EXCLUSIVE with --cookie-refresh " + "('cookie_refresh > 0 requires oauth tokens in sessions') — the proxy refuses to start", + ) + # METRICS. The listener does not exist unless the flag is given, and the port must be + # DECLARED or nothing can ever select it (neither a Service port nor a PodMonitor). + check( + f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}" in side["args"], + "sidecar must start the /metrics listener; without --metrics-address it is not merely " + "unscraped, it is not serving at all", + ) + check( + [p["name"] for p in side["ports"]] == ["http", "metrics"] + and {p["containerPort"] for p in side["ports"]} + == {OAUTH2_PROXY_PORT, OAUTH2_PROXY_METRICS_PORT}, + "sidecar must declare BOTH the http and metrics container ports — an undeclared port " + "cannot be named by a Service port or a PodMonitor, so the flag alone is inert", + ) + # Two forward-auth apps must not be able to collide. Renders a SECOND one and compares, so a + # future refactor to a constant (or to a per-service table with a copy-paste slip) fails here + # rather than silently in a browser six weeks later. + fa2_side = next( + c + for c in next( + d[1] + for d in render( + { + "name": "fa2", + "source": {"image": "a:1"}, + "ingressRoutes": [{"host": "fa2.home.keithjackson.dev", "port": 9118}], + "auth": {"mode": "forward-auth", "groups": ["media-users"]}, + } + ) + if d[1]["kind"] == "Deployment" + )["spec"]["template"]["spec"]["containers"] + if c["name"] == "oauth2-proxy" + ) + + def _cookie_name(container): + return next(a for a in container["args"] if a.startswith("--cookie-name=")) + + check( + _cookie_name(side) != _cookie_name(fa2_side), + "two forward-auth apps rendered IDENTICAL cookie names — they would overwrite each " + "other's session on every sign-in (the 2026-09-29 crafty defect, fleet-wide)", + ) check( side["env"][0]["valueFrom"]["secretKeyRef"]["name"] == "fa-oauth2-proxy", "sidecar creds must come by name from the per-app -oauth2-proxy Secret (R-IDP-04)", diff --git a/conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/chainsaw-test.yaml b/conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/chainsaw-test.yaml new file mode 100644 index 0000000..841bf86 --- /dev/null +++ b/conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/chainsaw-test.yaml @@ -0,0 +1,152 @@ +# ============================================================================= +# L0 STATIC — no two oauth2-proxy instances can share a browser cookie +# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster. +# +# THE DEFECT THIS CLOSES. Every oauth2-proxy on this platform is an INDEPENDENT instance with +# its OWN `cookie-secret`: five per-app reverse-proxy sidecars (jackett, pihole, qbittorrent, +# radarr, sonarr), crafty's, and the SHARED forward-auth proxy. Measured on prod 2026-09-30: +# seven `*-oauth2-proxy` Secrets, SEVEN DISTINCT cookie-secret values — so no instance can +# decrypt any other's cookie. +# +# They were all emitting oauth2-proxy's DEFAULT cookie names (`_oauth2_proxy` and +# `_oauth2_proxy_csrf`), scoped to `.${BASE_DOMAIN}`. A cookie is keyed by (name, domain, path), +# so the browser kept ONE pair for ALL of them. Whichever proxy authenticated last OVERWROTE +# every other's session; the loser logged `No valid authentication in request`, bounced to +# Keycloak, and — the SSO session still being live — was redirected straight back and overwrote +# the cookie again. Six live instances, one cookie, mutual eviction. +# +# WHY IT IS THE WORST KIND OF BUG. The symptom is "I signed into Keycloak but I'm still not +# signed in", which points the investigator at Keycloak, where nothing is wrong. It is +# INTERMITTENT — it only appears once a browser has visited a SECOND gated app, so the obvious +# single-app reproduction succeeds — and it is SILENT: every Deployment stays Ready, every +# Kustomization stays Ready, the proxy's own log line names an authentication failure rather +# than a collision. It went unnoticed across five services for the whole life of the pattern, +# and was found only as a side effect of investigating something else (2026-09-29, crafty). +# +# WHY A COMMIT IS NOT ENOUGH, AND THIS IS A GATE STAGE. The collision is a property of the +# PATTERN, not of any one app: the next forward-auth service onboarded inherits the defect by +# doing nothing at all, because a missing flag IS the bug. Nothing about its absence is loud. +# This is the same shape as the twelve inert ServiceMonitors in this tree — one omission, +# copied, invisible for weeks — and the same remedy: make the omission fail the merge. +# +# GIVEN every oauth2-proxy instance this repo authors — a container in any manifest (the +# rendered sidecars in apps/, crafty's hand-authored one in platform/) and the shared +# proxy's HelmRelease values, +# WHEN each instance's EFFECTIVE cookie name is resolved (its declared `--cookie-name` / +# `OAUTH2_PROXY_COOKIE_NAME`, or oauth2-proxy's built-in default `_oauth2_proxy` when it +# declares none — an omission is a NAME, not an absence, and that is the whole bug), +# THEN (1) no two instances share an effective cookie name; +# (2) no instance's effective name is another's name + `_csrf` — oauth2-proxy derives the +# CSRF cookie that way, so `_oauth2_proxy` and `_oauth2_proxy_csrf` as two SESSION +# names would collide on the second cookie while looking distinct on the first; +# (3) the DEFAULT name has exactly ONE owner and it is the instance declared in +# SHARED_DEFAULT_COOKIE_OWNER. This is the deliberate ASYMMETRY. The shared +# forward-auth proxy MUST keep `_oauth2_proxy`: it is the single session every +# OTHER forward-auth-gated surface on the platform rides on, so renaming it signs +# every one of them out at once. The rule is therefore NOT "every instance must +# declare a name" — it is "the default is a NAME LIKE ANY OTHER, held by exactly one +# REGISTERED owner". That formulation needs no exemption from (1), and it closes the +# hole an exemption would leave: if the shared proxy is ever removed, an app silently +# inheriting the default would still FAIL here instead of quietly becoming its heir; +# (4) every instance that is NOT that owner is HOST-SCOPED: its `--cookie-domain` equals +# the host of its own `--redirect-url`. See the argument below; +# (5) catalog/render.py DERIVES both flags from the ServiceDefinition (the service name +# and the route host) and does not carry a per-service table. A list is how twelve +# ServiceMonitors acquired one identical wrong port name in a single edit, and a list +# leaves the NEXT app defaulting into the collision; +# (6) the SHIPPED renderer is proven END-TO-END on synthetic ServiceDefinitions, run as a +# subprocess: two forward-auth fixtures must yield two different, non-default cookie +# names, each scoped to its own host. Without this, once the tree is compliant (1)-(4) +# are satisfied by a tree with nothing left to catch, and a guard whose only evidence +# is "nothing failed" cannot be told apart from one that stopped working; +# AND (7) the check is LOAD-BEARING: this Test DECLARES `conformance.home.k3s/scope: +# whole-repo`, so ci/l0.sh Stage 3b's `--include-scope whole-repo` unions it into +# EVERY gate run, and ci/gate.sh blocks the merge when the l0 stage fails. It +# deliberately adds NO stage to ci/l0.sh — tests/l0-static/whole-repo-invariants-wired +# fails the gate if anyone adds one. +# +# WHY SCOPE, NOT REQUIREMENTS: ci/touched-requirements.sh maps `apps/*` to R-IDP-02/R-IDP-06 +# today, but the instances live in THREE unrelated subtrees (apps/, platform/crafty/, +# platform/oauth2-proxy/) and the file most likely to reintroduce the defect is +# catalog/render.py, which maps to R-GITOPS-02 and to no identity requirement at all. A +# requirement-scoped selection would filter this check out of exactly the change that breaks it. +# +# ON ASSERTION (4), HOST-SCOPED COOKIE-DOMAIN — the argument, and the decision. +# AGAINST asserting it (overreach): distinct NAMES already close the eviction bug completely. +# Two cookies with different names do not overwrite each other however wide their domains are, +# so (4) fixes nothing that (1) has not already fixed. `--cookie-domain` is also a legitimate +# knob — a future proxy fronting two hostnames of one app would need a wider value — and a +# check that fails correct configurations is a check someone eventually disables. +# FOR asserting it (and this wins): a zone-wide cookie is TRANSMITTED BY THE BROWSER TO EVERY +# HOST IN THE ZONE. qBittorrent's session token is therefore sent to pihole, to radarr, to +# sonarr, and to every other name under `.${BASE_DOMAIN}` — including hosts behind no proxy at +# all. That is a session-credential disclosure across trust boundaries that no one needs and +# nobody asked for; the sidecar's session is only ever meaningful to the one app it fronts. +# And it is what makes a FUTURE name collision catastrophic rather than contained: with +# host-scoped cookies, two instances that accidentally share a name cannot even see each +# other's cookie, so the failure this whole test exists to prevent degrades from platform-wide +# to nothing. Defence in depth on the exact axis that already failed once. +# DECISION: assert it, for every instance except the registered shared proxy — whose whole +# job IS cross-host forward-auth, and which is exempted by name and with a reason, not by +# loosening the rule. `--whitelist-domain` is deliberately NOT touched: it governs permitted +# post-login REDIRECT targets, not cookie scope, and narrowing it breaks the sign-in bounce. +# +# STATUS: EXECUTABLE. `check.sh` is an offline assertion script (PyYAML, as catalog/render.py +# and the sibling l0-static checks use); conformance/run.sh --layer l0 and ci/l0.sh run it +# directly, and this Chainsaw Test wraps the SAME script. +# +# NOT proven (honest gaps; this is STATIC, with no cluster and no browser): +# - That anyone can actually sign in. No browser round-trip is performed, and none can be +# from L0. The 2026-09-29 crafty entry recorded the same gap and it is still open: the +# symptom has never been REPRODUCED in a browser, so this is a latent-bug fix reasoned from +# measured configuration, not a fix watched to work. +# - That the LIVE pods carry what the tree says. git is not the cluster (CLAUDE.md §2b). The +# live flags were measured separately, once, by hand; nothing here re-measures them, and a +# hand-applied drift would not be visible to this check. +# - That the cookie-secrets really differ. That is a Secret comparison and belongs in-cluster; +# it was measured out of band (seven distinct values, compared without printing any of +# them). If they were ever made IDENTICAL the collision would become harmless — but sharing +# one session-encryption key across trust zones is a worse idea than the bug it would mask, +# so this check does not offer it as an alternative and does not look for it. +# - Anything about the CONTENT of a session, group authorization, or the CSRF cookie's own +# behaviour. Names and domains only. +# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06). +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: oauth2-proxy-cookie-names-distinct + labels: + conformance.home.k3s/R-IDP-02: "true" + conformance.home.k3s/R-IDP-06: "true" + annotations: + conformance.home.k3s/requirements: "R-IDP-02, R-IDP-06" + conformance.home.k3s/layer: "L0" + # WHOLE-REPO SCOPE — the instances span apps/wave-0, apps/wave-1, platform/crafty and + # platform/oauth2-proxy, and the generator that emits five of them is catalog/render.py. + # No single requirement path-maps to that set, and the renderer maps to no identity + # requirement at all, so a requirement-scoped selection would skip this check on precisely + # the change that can reintroduce the defect fleet-wide in one edit. + conformance.home.k3s/scope: "whole-repo" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + No two oauth2-proxy instances can share a browser cookie: every instance's effective + cookie name is unique, the built-in default belongs to exactly one registered owner (the + shared forward-auth proxy, which cannot be renamed without signing every gated app out), + every other instance is host-scoped, and catalog/render.py derives both flags from the + ServiceDefinition rather than from a per-service list — proven end-to-end on synthetic + fixtures so the guard stays falsifiable once the tree is clean. +spec: + steps: + - name: assert-no-two-oauth2-proxy-instances-share-a-cookie + description: >- + Inventory every oauth2-proxy container and HelmRelease in the tree, resolve each one's + effective cookie name (declared, or oauth2-proxy's default when omitted), assert + pairwise distinctness including the session/CSRF derivation, ratchet the single + registered owner of the default name, assert host-scoped cookie-domain on every other + instance, assert catalog/render.py derives rather than lists, and prove the shipped + renderer on two synthetic forward-auth ServiceDefinitions. + try: + - script: + content: bash ./check.sh diff --git a/conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/check.sh b/conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/check.sh new file mode 100755 index 0000000..d05fa53 --- /dev/null +++ b/conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/check.sh @@ -0,0 +1,498 @@ +#!/usr/bin/env bash +# ============================================================================= +# L0 STATIC CHECK — no two oauth2-proxy instances can share a browser cookie. +# See chainsaw-test.yaml's header for WHY this exists, what it measured, and the argument +# for assertion (4). In one paragraph: +# +# Every oauth2-proxy here holds its OWN cookie-secret (seven distinct values on prod, +# 2026-09-30), so two instances sharing a cookie NAME on a shared DOMAIN cannot read each +# other's session — they OVERWRITE it. The loser bounces to Keycloak, is redirected straight +# back by the live SSO session, and overwrites the winner in turn. The symptom is "I signed +# into Keycloak but I'm still not signed in", it only shows up after a second gated app is +# visited, and nothing in the cluster reports an error. A missing --cookie-name IS the bug, +# so the next forward-auth app inherits it by doing nothing. +# +# Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, which the runner +# turns into rc 3 = UNWIRED, never green, because this test declares the whole-repo scope) +# only if the tree contains no oauth2-proxy instance at all — genuinely nothing to audit. +# ============================================================================= +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root +PYTHON="${PYTHON:-python3}" + +# Honest SKIP (never green for a scope-declaring test) if there is nothing to audit at all. +# Matches the IMAGE every instance runs and the CHART the shared one installs; if NEITHER +# appears anywhere, the forward-auth pattern has left this repo and the check has no subject. +# +# ANCHORED ON `image:` DELIBERATELY. A bare `oauth2-proxy/oauth2-proxy:` also matches +# platform/oauth2-proxy/forward-auth-cec.yaml's Envoy cluster reference +# (`serviceName: oauth2-proxy/oauth2-proxy:80` — namespace/service:port, not a registry path). +# That file configures the ext_authz listener that CALLS the proxy; it runs no proxy and holds +# no cookie flags, so counting it made the grep/parse cross-check below report a phantom +# unaudited instance. +if ! grep -rlE 'image:[[:space:]]*[^[:space:]]*oauth2-proxy|chart:[[:space:]]*oauth2-proxy' \ + "${ROOT}/apps" "${ROOT}/platform" "${ROOT}/catalog" >/dev/null 2>&1; then + echo "check SKIPPED: no oauth2-proxy image or chart found under apps/, platform/ or catalog/ — nothing to audit" >&2 + exit 3 +fi + +cd "${ROOT}" +"${PYTHON}" - "${ROOT}" <<'PY' +"""Static audit: no two oauth2-proxy instances can share a browser cookie. + +PyYAML (the same dependency as catalog/render.py and the sibling l0-static checks) rather +than line-based parsing: a container's args live in a nested structure and a HelmRelease's +flags live in a values map, and a line-based reader of either is precisely the approximate +parser that silently stops matching — the failure class this check exists to prevent. +""" +from __future__ import annotations + +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", + file=sys.stderr) + raise SystemExit(1) + +ROOT = Path(sys.argv[1]) +sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree + +FAILURES: list[str] = [] +CHECKS = 0 + + +def check(cond, msg): + global CHECKS + CHECKS += 1 + if not cond: + FAILURES.append(msg) + + +def read(p: Path) -> str: + try: + return p.read_text() + except (OSError, UnicodeDecodeError): + return "" + + +# oauth2-proxy's OWN built-in defaults (pkg/apis/options/cookie.go): the session cookie is +# `_oauth2_proxy` and the CSRF cookie is that name with `_csrf` appended. An instance that +# declares nothing is NOT unconfigured — it has claimed this exact name, which is why the +# inventory below treats the default as a name like any other rather than as an absence. +DEFAULT_COOKIE_NAME = "_oauth2_proxy" +CSRF_SUFFIX = "_csrf" + +# ============================================================================================= +# REGISTER — SHARED_DEFAULT_COOKIE_OWNER. Exactly one entry, and it must match a real instance. +# +# The one instance permitted to hold DEFAULT_COOKIE_NAME, and the one permitted a zone-wide +# cookie-domain. Both exemptions belong to the SAME object for the SAME reason, so they are one +# register rather than two: the shared forward-auth proxy is not a per-app sidecar, it is the +# single session that every forward-auth-gated surface on the platform rides on. Renaming its +# cookie signs all of them out simultaneously, and host-scoping it would break the cross-host +# forward-auth that is its entire job. +# +# Shrink-only, and exact in BOTH directions: an entry that matches no instance FAILS as stale +# (the proxy was renamed or removed and the exemption is now a hole), and an instance holding +# the default WITHOUT an entry FAILS as an unregistered heir. Adding an entry is a review event +# on a sensitive path. +# +# Keyed by ":". +# ============================================================================================= +SHARED_DEFAULT_COOKIE_OWNER: dict[str, str] = { + "platform/oauth2-proxy/helmrelease.yaml:oauth2-proxy": + "The SHARED forward-auth proxy (ADR-0014, platform/oauth2-proxy/README.md). It is the " + "single authenticator behind every forward-auth-gated surface that is NOT fronted by " + "its own sidecar, so its cookie is the platform session: renaming it signs every one " + "of them out at once, and that is why the 2026-09-29 crafty fix changed only the " + "sidecar side. Its cookie-domain is zone-wide for the same structural reason — " + "cross-host forward-auth is what it is for.", +} + + +def is_test_fixture(p: Path) -> bool: + """conformance/tests/**/{golden,fixtures}/** is assertion DATA, not deployed manifests. + + Auditing a golden would be auditing a copy of something already audited through apps/. + Same exclusion the sibling l0-static checks make, and the same "reference, not + declaration" distinction ci/touched-requirements.sh draws for fixture paths. + """ + parts = p.parts + return "conformance" in parts and ("golden" in parts or "fixtures" in parts) + + +def iter_containers(node): + """Yield every container mapping under any `containers`/`initContainers` list, at any depth. + + Depth-agnostic on purpose: a sidecar sits under spec.template.spec.containers in a + Deployment, but the same container shape appears under a CronJob's jobTemplate, a + DaemonSet, or a Pod, and a reader that hardcoded one path would silently stop seeing an + instance the day one moved. + """ + if isinstance(node, dict): + for key, val in node.items(): + if key in ("containers", "initContainers") and isinstance(val, list): + for c in val: + if isinstance(c, dict): + yield c + else: + yield from iter_containers(val) + elif isinstance(node, list): + for item in node: + yield from iter_containers(item) + + +def flag_from_args(args: list, flag: str): + """Value of `--flag=value` or `--flag value` in an argv list, else None.""" + pref = "--" + flag + for i, a in enumerate(args): + if not isinstance(a, str): + continue + if a.startswith(pref + "="): + return a.split("=", 1)[1] + if a == pref and i + 1 < len(args) and isinstance(args[i + 1], str): + return args[i + 1] + return None + + +def host_of(url): + """Host part of an https://host/path URL, else None.""" + if not isinstance(url, str): + return None + m = re.match(r"^https?://([^/]+)", url) + return m.group(1) if m else None + + +# ============================================================ (0) INVENTORY +# instances: list of dicts {rel, ident, kind, declared_name, cookie_domain, redirect_host} +instances: list[dict] = [] +unparseable: list[str] = [] +# Files the grep gate says contain an instance, so the parse can be cross-checked against it. +grep_files: set[str] = set() + +for p in sorted(ROOT.rglob("*.yaml")): + relparts = p.relative_to(ROOT).parent.parts + # NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES + # such as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent + # runs, so a bare rglob audited up to 18 duplicates of every file — which both invents + # FAILs and, worse, can satisfy a "something declares X" check from a copy rather than from + # real source. Same fix as egress-paths-selected-by-rules. .claude/ is gitignored. + if any(part.startswith(".") for part in relparts) or is_test_fixture(p) or ".sops." in p.name: + continue + txt = read(p) + if not txt: + continue + rel = str(p.relative_to(ROOT)) + # Same anchoring as check.sh's skip gate: `image:` (or a chart name), never a bare path — + # see the comment there for the Envoy `serviceName: oauth2-proxy/oauth2-proxy:80` near-miss. + if re.search(r"image:\s*\S*oauth2-proxy|chart:\s*oauth2-proxy", txt): + grep_files.add(rel) + elif "oauth2-proxy" not in txt: + continue + try: + docs = [d for d in yaml.safe_load_all(txt) if isinstance(d, dict)] + except yaml.YAMLError as exc: + unparseable.append(f"{rel} ({exc})") + continue + for d in docs: + # --- shape 1: a container running the oauth2-proxy image ------------------------- + for c in iter_containers(d): + image = str(c.get("image") or "") + name = str(c.get("name") or "") + if "oauth2-proxy" not in image and name != "oauth2-proxy": + continue + args = [a for a in (list(c.get("args") or []) + list(c.get("command") or []))] + env = {e.get("name"): e.get("value") + for e in (c.get("env") or []) if isinstance(e, dict) and "value" in e} + owner = (d.get("metadata") or {}).get("name") or "?" + instances.append({ + "rel": rel, + "ident": f"{owner}/{name or 'oauth2-proxy'}", + "kind": "container", + "declared_name": flag_from_args(args, "cookie-name") + or env.get("OAUTH2_PROXY_COOKIE_NAME"), + "cookie_domain": flag_from_args(args, "cookie-domain") + or env.get("OAUTH2_PROXY_COOKIE_DOMAIN"), + "redirect_host": host_of(flag_from_args(args, "redirect-url") + or env.get("OAUTH2_PROXY_REDIRECT_URL")), + }) + # --- shape 2: the shared proxy's HelmRelease values ------------------------------ + if d.get("kind") != "HelmRelease": + continue + spec = d.get("spec") or {} + chart = (((spec.get("chart") or {}).get("spec") or {}).get("chart") or "") + if "oauth2-proxy" not in str(chart): + continue + values = spec.get("values") or {} + extra = values.get("extraArgs") or {} + if isinstance(extra, list): # the chart also accepts a bare argv list + extra = {a.lstrip("-").split("=", 1)[0]: a.split("=", 1)[1] + for a in extra if isinstance(a, str) and "=" in a} + cfg = values.get("config") or {} + get = lambda k: extra.get(k, cfg.get(k)) # noqa: E731 - two-source lookup, used 3x + instances.append({ + "rel": rel, + "ident": str((d.get("metadata") or {}).get("name") or "oauth2-proxy"), + "kind": "helmrelease", + "declared_name": get("cookie-name"), + "cookie_domain": get("cookie-domain"), + "redirect_host": host_of(get("redirect-url")), + }) + +for u in unparseable: + check(False, f"{u} is not parseable YAML — every oauth2-proxy instance in it went unaudited") + +for ins in instances: + ins["key"] = f"{ins['rel']}:{ins['ident']}" + # THE CORE MODELLING DECISION: an omitted --cookie-name is not "unset", it is a CLAIM on + # oauth2-proxy's built-in default. Resolving it here is what lets assertion (1) below be a + # plain distinctness test with no special case, and it is exactly the step whose absence + # made the defect invisible to every reviewer who read these manifests. + ins["effective_name"] = ins["declared_name"] or DEFAULT_COOKIE_NAME + +# Anti-vacuous. The floor is deliberately low (a real removal should not turn this check red), +# so the load-bearing guard is the grep/parse CROSS-CHECK beneath it: every file the cheap +# text scan says holds an instance must yield at least one PARSED instance. A parser that +# quietly stopped matching would keep the floor and fail this. +check(len(instances) >= 5, + f"only {len(instances)} oauth2-proxy instance(s) parsed out of the tree. This repo " + f"authors more than that (five rendered sidecars, crafty's, and the shared proxy), so " + f"the inventory has stopped seeing instances and every assertion below would pass " + f"vacuously while a collision went unaudited") + +parsed_files = {ins["rel"] for ins in instances} +for rel in sorted(grep_files - parsed_files): + check(False, + f"{rel} contains an oauth2-proxy image or chart reference but NO instance was parsed " + f"out of it. Either it uses a shape iter_containers()/the HelmRelease reader does not " + f"recognise, or the file stopped parsing. Its cookie configuration is UNAUDITED, which " + f"is the same state the whole fleet was in before this check existed") + +# ============================================================ (1) DISTINCT EFFECTIVE NAMES +by_name: dict[str, list[dict]] = {} +for ins in instances: + by_name.setdefault(ins["effective_name"], []).append(ins) + +for nm, group in sorted(by_name.items()): + if len(group) < 2: + continue + who = ", ".join(sorted(g["key"] for g in group)) + default_note = ( + " Every one of them OMITS --cookie-name, so they are all claiming oauth2-proxy's " + "built-in default. Omitting the flag is not neutral: it IS this collision." + if nm == DEFAULT_COOKIE_NAME else "") + check(False, + f"{len(group)} oauth2-proxy instances share the cookie name '{nm}': {who}.{default_note}\n" + f" WHY THIS IS A BUG AND NOT A STYLE RULE: these are INDEPENDENT instances with " + f"DIFFERENT cookie-secrets (measured on prod 2026-09-30: seven distinct values), so " + f"neither can DECRYPT the other's cookie — it can only OVERWRITE it. A browser keys a " + f"cookie by (name, domain, path), so signing into one app destroys the session of every " + f"other app sharing that key. The loser logs 'No valid authentication in request', " + f"bounces to Keycloak, is redirected straight back because the SSO session is still " + f"live, and overwrites the winner in turn. The user sees 'I signed into Keycloak but " + f"I'm still not signed in' and goes looking in Keycloak, where nothing is wrong.\n" + f" FIX: give each per-app instance its own --cookie-name. For a service rendered " + f"from catalog/, do NOT hand-edit the golden — catalog/render.py derives the flag from " + f"the service name; re-render instead (ci/render-golden.sh's header has the canonical " + f"command). Exactly one instance may hold '{DEFAULT_COOKIE_NAME}', and it must be the " + f"one registered in SHARED_DEFAULT_COOKIE_OWNER.") + +# ============================================================ (2) SESSION vs CSRF DERIVATION +# oauth2-proxy names the CSRF cookie _csrf. So a SESSION name that happens to equal +# another instance's name + "_csrf" collides on the CSRF cookie while assertion (1) sees two +# perfectly distinct strings. This is the near-miss the obvious fix invites: naming the shared +# proxy's neighbour `_oauth2_proxy_csrf` reads as "clearly different" and is not. +effective = {ins["effective_name"] for ins in instances} +for ins in sorted(instances, key=lambda i: i["key"]): + base = ins["effective_name"] + if not base.endswith(CSRF_SUFFIX): + continue + victim = base[: -len(CSRF_SUFFIX)] + check(victim not in effective, + f"{ins['key']} uses the SESSION cookie name '{base}', which is exactly the CSRF cookie " + f"oauth2-proxy derives for the instance named '{victim}' (it appends '{CSRF_SUFFIX}'). " + f"The two session names look distinct and the CSRF cookies collide, so sign-in breaks " + f"with a state-mismatch instead of a session loss — the same bug wearing a different " + f"error message. Pick a name that is not another instance's name + '{CSRF_SUFFIX}'") + +# ============================================================ (3) THE DEFAULT HAS ONE OWNER +default_holders = sorted(ins["key"] for ins in instances + if ins["effective_name"] == DEFAULT_COOKIE_NAME) +registered = set(SHARED_DEFAULT_COOKIE_OWNER) + +for key in sorted(registered - set(ins["key"] for ins in instances)): + check(False, + f"SHARED_DEFAULT_COOKIE_OWNER registers '{key}', but no such oauth2-proxy instance " + f"exists in the tree. The register is SHRINK-ONLY and exact: a stale entry is a " + f"standing exemption for an object that is gone, so the next instance to inherit the " + f"default name would be silently excused. Remove the entry, or restore the instance") + +for key in default_holders: + check(key in registered, + f"{key} holds oauth2-proxy's DEFAULT cookie name '{DEFAULT_COOKIE_NAME}' but is NOT " + f"registered in SHARED_DEFAULT_COOKIE_OWNER. Exactly ONE instance may hold the default, " + f"and it is the SHARED forward-auth proxy — whose cookie is the platform-wide session " + f"every forward-auth-gated surface rides on, so renaming IT signs them all out at once. " + f"Every other instance must declare its own --cookie-name. If this instance really is " + f"the new shared proxy, register it here with the reason; if it is a per-app sidecar, " + f"give it a unique name") + +check(len(default_holders) <= 1, + f"{len(default_holders)} instances hold the default cookie name: {', '.join(default_holders)}. " + f"The default is a NAME, not an absence — it can have exactly one owner") + +# ============================================================ (4) HOST-SCOPED COOKIE DOMAIN +# See the chainsaw-test header for the argument (distinct names alone already fix the eviction; +# host-scoping is asserted anyway because a zone-wide cookie is TRANSMITTED to every host in the +# zone, and because it makes a future name collision contained instead of platform-wide). +for ins in sorted(instances, key=lambda i: i["key"]): + if ins["key"] in registered: + continue # the shared proxy: zone-wide by design, registered + dom, rhost = ins["cookie_domain"], ins["redirect_host"] + check(dom is not None, + f"{ins['key']} declares no --cookie-domain. oauth2-proxy then scopes the cookie to the " + f"request host, which happens to be correct — but it is correct BY ACCIDENT and says " + f"nothing to the next reader. Declare it explicitly as this app's own host") + if dom is None: + continue + check(not dom.startswith("."), + f"{ins['key']} sets --cookie-domain={dom}. A leading dot makes the cookie ZONE-WIDE: " + f"the browser sends this app's session token to every other host under that domain, " + f"including hosts behind no proxy at all, and it is what turns any future cookie-name " + f"collision into a platform-wide outage instead of a contained one. Scope it to this " + f"app's own host. Only the registered shared forward-auth proxy may be zone-wide, " + f"because cross-host forward-auth is its job") + if rhost: + check(dom == rhost, + f"{ins['key']} sets --cookie-domain={dom} but its --redirect-url host is {rhost}. " + f"The cookie is then set for a domain the sign-in flow never returns to (or for a " + f"wider one than this app owns), so the session either does not stick or leaks " + f"beyond the app. Both flags describe the SAME app: make them agree") + +# ============================================================ (5) THE RENDERER DERIVES +# Five of these instances are GENERATED. Auditing only the goldens would let someone "fix" the +# tree by hand and leave the generator emitting the defect for the next service onboarded — and +# the next re-render would then silently revert the hand fix. +render_py = ROOT / "catalog" / "render.py" +check(render_py.is_file(), + "catalog/render.py is missing — the generator of five of these instances cannot be audited") +if render_py.is_file(): + src = read(render_py) + m = re.search(r"def _oauth2_sidecar_container\(.*?\n(?=\n*def |\n*# =)", src, re.S) + fn = m.group(0) if m else "" + check(bool(fn), + "could not locate _oauth2_sidecar_container() in catalog/render.py. It was renamed or " + "removed; assertions (5) below cannot run, and the generator is unaudited") + if fn: + cookie_name_lines = [ln for ln in fn.splitlines() if "--cookie-name" in ln] + check(bool(cookie_name_lines), + "catalog/render.py's _oauth2_sidecar_container() emits NO --cookie-name at all, so " + "every service it renders claims oauth2-proxy's default and collides with every " + "other. This is the defect itself, back in the one place that reintroduces it " + "fleet-wide in a single edit") + check(any("s['name']" in ln or 's["name"]' in ln for ln in cookie_name_lines), + "catalog/render.py emits --cookie-name without DERIVING it from the " + "ServiceDefinition's own name. A constant collides with every sibling by " + "construction; a per-service table leaves the NEXT app onboarded defaulting into " + "the collision, which is how twelve ServiceMonitors acquired one identical wrong " + "port name in a single edit. Derive it from s['name']") + cookie_dom_lines = [ln for ln in fn.splitlines() if "--cookie-domain" in ln] + check(any("{host}" in ln for ln in cookie_dom_lines), + "catalog/render.py's sidecar --cookie-domain is not derived from the route host. A " + "zone-wide literal (`.{ZONE}`) broadcasts every app's session cookie across the " + "whole zone — see assertion (4)") + check(not any(re.search(r"--cookie-name=_oauth2_proxy_[a-z0-9]", ln) for ln in cookie_name_lines), + "catalog/render.py hardcodes a literal service name into --cookie-name. Whatever " + "service that is, every OTHER service now renders the same constant and they all " + "collide. Derive the name instead") + +# ============================================================ (6) PROVE THE SHIPPED RENDERER +# Once the tree is compliant, (1)-(4) have nothing left to catch and their passing is evidence +# of nothing. This runs the SHIPPED renderer as a SUBPROCESS on two synthetic ServiceDefinitions +# and asserts the property directly, so the guard stays falsifiable. +def _sd(name: str, port: int) -> str: + return yaml.safe_dump({ + "name": name, + "kind": "container", + "source": {"image": "example/app:1"}, + "exposure": "internal", + "ingressRoutes": [{"host": f"{name}.home.keithjackson.dev", "port": port}], + "auth": {"mode": "forward-auth", "groups": ["platform-admins"]}, + }, sort_keys=False) + + +def _sidecar_args(rendered: str) -> list: + for doc in yaml.safe_load_all(rendered): + if not isinstance(doc, dict) or doc.get("kind") != "Deployment": + continue + for c in iter_containers(doc): + if c.get("name") == "oauth2-proxy": + return list(c.get("args") or []) + return [] + + +if render_py.is_file(): + fixtures = {"cookiefixa": 8111, "cookiefixb": 8112} + rendered_args: dict[str, list] = {} + with tempfile.TemporaryDirectory() as td: + for svc, port in fixtures.items(): + f = Path(td) / f"{svc}.yaml" + f.write_text(_sd(svc, port)) + proc = subprocess.run([sys.executable, str(render_py), str(f)], + capture_output=True, text=True) + check(proc.returncode == 0, + f"the shipped catalog/render.py failed on a synthetic forward-auth fixture " + f"({svc}): rc={proc.returncode} {proc.stderr.strip()[:200]}") + if proc.returncode == 0: + rendered_args[svc] = _sidecar_args(proc.stdout) + + names, domains = {}, {} + for svc, args in rendered_args.items(): + names[svc] = flag_from_args(args, "cookie-name") + domains[svc] = flag_from_args(args, "cookie-domain") + check(bool(args), + f"the shipped renderer produced no oauth2-proxy sidecar for synthetic forward-auth " + f"fixture {svc} — the sidecar is how a forward-auth app is gated at all (R-IDP-02)") + check(names[svc] not in (None, DEFAULT_COOKIE_NAME), + f"the shipped renderer gave synthetic fixture {svc} the cookie name " + f"{names[svc]!r}. A newly-onboarded forward-auth app would therefore default " + f"straight into the fleet collision — which is exactly how five services acquired " + f"this defect without anyone making a decision") + check(domains[svc] == f"{svc}.home.keithjackson.dev", + f"the shipped renderer scoped synthetic fixture {svc}'s cookie to " + f"{domains[svc]!r} rather than to its own host — a newly-onboarded app would " + f"broadcast its session cookie across the whole zone") + check(any(isinstance(a, str) and a.startswith("--whitelist-domain=.") for a in args), + f"synthetic fixture {svc} lost its ZONE-WIDE --whitelist-domain. That flag governs " + f"permitted post-login REDIRECT targets, not cookie scope; narrowing it with the " + f"cookie domain breaks the sign-in bounce") + + if len(rendered_args) == 2: + a, b = sorted(rendered_args) + check(names[a] != names[b], + f"the shipped renderer gave two DIFFERENT synthetic services the SAME cookie name " + f"({names[a]!r}). Whatever the committed goldens currently look like, the generator " + f"will collide every forward-auth app it renders from now on") + check(domains[a] != domains[b], + f"the shipped renderer gave two different synthetic services the same cookie domain " + f"({domains[a]!r}) — the cookies would share a scope even with distinct names") + +# ============================================================ VERDICT +inv = " | ".join(f"{i['key']} -> {i['effective_name']} @ {i['cookie_domain'] or ''}" + for i in sorted(instances, key=lambda x: x["key"])) +print(f"oauth2-proxy instances audited ({len(instances)}): {inv}") +if FAILURES: + print(f"\nCHECK FAIL (oauth2-proxy-cookie-names-distinct): {len(FAILURES)} failed, " + f"{CHECKS - len(FAILURES)} passed", file=sys.stderr) + for f in FAILURES: + print(f" FAIL {f}", file=sys.stderr) + raise SystemExit(1) +print(f"check OK: oauth2-proxy-cookie-names-distinct — {CHECKS} assertions (R-IDP-02, R-IDP-06)") +PY diff --git a/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml b/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml index a8d1db4..bf3e82d 100644 --- a/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml +++ b/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml @@ -76,11 +76,14 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://paas-demo.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_paas-demo + - --cookie-domain=paas-demo.home.keithjackson.dev - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -101,6 +104,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml b/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml index a13b7c2..b32ea49 100644 --- a/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml @@ -99,11 +99,14 @@ spec: - --allowed-group=media-admins - --skip-provider-button=true - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_sonarr + - --cookie-domain=sonarr.home.keithjackson.dev - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -124,6 +127,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml b/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml index dbccb46..692e8dd 100644 --- a/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml @@ -99,11 +99,14 @@ spec: - --allowed-group=media-admins - --skip-provider-button=true - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.${BASE_DOMAIN} + - --cookie-name=_oauth2_proxy_sonarr + - --cookie-domain=sonarr.home.keithjackson.dev - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true + - --cookie-refresh=2m30s - --pass-access-token=true - --set-xauthrequest=true + - --metrics-address=0.0.0.0:44180 env: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: @@ -124,6 +127,9 @@ spec: - name: http containerPort: 4180 protocol: TCP + - name: metrics + containerPort: 44180 + protocol: TCP resources: requests: cpu: 25m diff --git a/decisions/2026-09-30-jackett-oauth2-proxy-session.md b/decisions/2026-09-30-jackett-oauth2-proxy-session.md new file mode 100644 index 0000000..d8c0259 --- /dev/null +++ b/decisions/2026-09-30-jackett-oauth2-proxy-session.md @@ -0,0 +1,273 @@ +--- +date: 2026-09-30 +branch: fix/jackett-oauth2-proxy-session +title: Jackett — radarr's oauth2-proxy deletes jackett's session cookie; the fix cannot be made per-app +--- + +## What changed + +Reported as *"GET requests work, but most PUT/POST requests to add indexers fail. Reloading the page +never fixes it, and I am never re-prompted for login if it is a login issue."* + +**This branch lands the diagnosis and a runbook. It does NOT land the manifest fix, and that is a +deliberate stop, not an omission** — see *The fix has one home and it is shared*, below. + +Three mechanisms, all measured. **Weighting matters here, so it is stated up front:** mechanisms 2 +and 3 are demonstrated from Jackett's own access log and **between them fully explain the report as +written** — PUT/POST fail, the operator is never re-prompted, and a reload does not help. Mechanism 1 +is a real, firing defect that Jackett genuinely violates, and it explains the *intermittency* on top; +I demonstrate the mechanism and one three-second causal adjacency, not a population of them. Fix all +three; do not rest the case on the first. + +**1. Cookie collision — the mechanism is firing, and Jackett is a genuine violator.** Six oauth2-proxy +instances (seven pods) emit oauth2-proxy's **default** cookie name `_oauth2_proxy` scoped to +`.home.keithjackson.dev` while holding seven **different** cookie-secrets. The ping-pong is in two +pods' own logs, interleaved to the second: + + 15:23:17 radarr [AuthSuccess] -> radarr writes _oauth2_proxy + 15:23:44 jackett "cookie signature not valid, removing session" -> re-login #1 + 15:23:58 radarr "cookie signature not valid, removing session" -> radarr DELETES jackett's + 15:27:17 jackett [AuthSuccess] -> re-login #2 + 15:27:59 radarr "cookie signature not valid, removing session" + 15:31:06 jackett [AuthSuccess] -> re-login #3 + 15:31:29 radarr "cookie signature not valid, removing session" <- deletes it again + 15:31:32 jackett "No valid authentication in request" <- 3 SECONDS later, no cookie + +`removing session` is oauth2-proxy calling ClearSession, which expires the cookie on **its own** +`--cookie-domain` — the whole zone. So jackett's session is not merely overwritten, it is **deleted +by a process in another namespace**, 26 s after a clean login whose own `expires` was five minutes +out. That is the answer to the open question in the brief ("26 s is well inside the 5-minute window, +so plain expiry does not explain it"): it was never expiry, and it was never Jackett. + +It also explains the half of the report that looked like user error. **Reloading genuinely does not +help**, because the operator's Radarr tab re-clears the cookie seconds after each new login — and +Radarr is not a passive tab. Its UI polls: of 156 unauthenticated requests in its log against +**1** AuthSuccess, the top paths are `/api/v3/command` (62), `/api/v3/queue/details` (46) and +`/signalr/messages/negotiate` (43). Every one of those polls, arriving with a cookie Radarr cannot +decrypt, calls ClearSession again. **A left-open Radarr tab is a cookie-deletion engine firing every +few seconds**, which is why Jackett's sessions die at irregular, short, apparently random intervals. + +**A counter-hypothesis was put to me and it does not survive bounding.** The observation was that +Crafty already carries this fix yet showed the platform's worst failure ratio (81:1 over 24 h from +Loki), implying the fix does not work. But Crafty's pod started **2026-09-30T03:57:19Z with 0 +restarts**, so a 24 h window is ~10 h of the *pre-fix* pod. Rather than bound it in Loki I read the +ratio off the pod's own log, whose boundary is the process start and therefore exact: **inside the +fixed pod's entire 13 h 52 m lifetime, Crafty logged 3 `No valid authentication` events, not 81** — +from one client IP, two of them WebSocket `/ws` reconnects, in two isolated minutes (04:02 and +15:22), against 0 AuthSuccess because nobody signed in. Post-fix Crafty is idle, not broken. The +ratio metric is confounded by endpoint type — apps that poll or hold WebSockets (Radarr, Crafty) +manufacture failure lines at their retry rate — so it ranks *chattiness*, not breakage. Correctly +bounded, Crafty is evidence **for** the fix. + +**2. Session lifetime pinned to a 5-minute access token.** Measured from the sidecar's own +`[AuthSuccess]`: `created:15:31:06.391860254` / `expires:` + **300.000 s** exactly, with +`refresh_token:true` on the same line. `--cookie-refresh` is unset and its default is `0` +(disabled), so the proxy holds a refresh token it has no mandate to spend. + +**3. XHRs answered with 302 instead of 401 — why none of it was visible.** oauth2-proxy's built-in +AJAX guard fires only on an **exact** `Accept: application/json`, which Jackett's jQuery does not +consistently send. Measured on `/api/` paths in this pod's log: **33 × 302 against 2 × 401**, and the +split runs straight through Jackett's own POSTs (`/api/v2.0/server/adminpassword` → 401, +`/api/v2.0/server/update` → 302). A 302 on an unauthenticated XHR is followed invisibly by the +browser, satisfied silently by the live Keycloak SSO session, and resolves with HTML where the +JavaScript expected JSON — no prompt, no error. On a write it is worse: a browser following a 302 +re-issues a POST as a **GET and drops the body**, so the indexer configuration is discarded with +nothing logged as a failure anywhere. + +### The fix has one home, and it is shared — this is the blocker + +The brief assigned me `apps/wave-0/jackett.yaml` as exclusively mine, and assigned the other four +apps to a sibling branch. **That split is not implementable, and I found out by trying it.** + +`apps/wave-0/jackett.yaml` is a **generated golden**. The sidecar's argument list is built by one +shared function, `_oauth2_sidecar_container()` in `catalog/render.py`. `catalog/jackett.yaml` +declares only `auth.mode` and `auth.groups`, and the `auth` schema is `additionalProperties: false` +— **there is no per-app override hook**. I wrote the fix into the rendered file first; the merge +gate rejected it exactly as designed: + + [fail] GOLDEN DRIFT: apps/wave-0/jackett.yaml does not match the canonical render + [fail] stage render-golden: FAIL -> MERGE GATE: BLOCKED (rc 1) + +So I made the change at its real source instead and **measured the blast radius** before committing +to it. A change confined to that one function rewrites the sidecar block in **all five** rendered +apps, and `render-golden` requires all five be regenerated in the same commit: + + CHANGED: apps/wave-0/jackett.yaml + CHANGED: apps/wave-0/pihole.yaml + CHANGED: apps/wave-0/qbittorrent.yaml + CHANGED: apps/wave-1/radarr.yaml + CHANGED: apps/wave-1/sonarr.yaml + +Four of those five are the files I was explicitly told not to touch, and all four are being edited +right now on `fix/oauth2-proxy-cookie-collision-fleet`. **The gate and the ownership boundary are in +direct contradiction**: I cannot fix Jackett's sidecar without regenerating four files another +branch owns, and two branches rewriting the same generated block across five files is precisely the +cross-branch conflict this repo's §7 exists to prevent. + +I stopped rather than force it. **The cookie fix is one change, not five**, and it should be re-cut +as a single renderer commit on one branch. The intended content, validated below: + + --cookie-name=_oauth2_proxy_{name} # per-app, derived + --cookie-domain={host} # host-scoped, derived + --whitelist-domain=.{ZONE} # unchanged: redirect targets, not cookie scope + --cookie-refresh=2m30s # half the measured 300 s token lifetime + (drop --pass-access-token, --set-xauthrequest) + +`--api-route=^/api/` is the one flag I would **not** apply renderer-wide on this evidence: I +measured the 302/401 split on Jackett only, and PiHole's UI does not serve `/api/` at all. It needs +either a per-app hook or four more measurements. + +**Metrics — asked for, and declined here for the same structural reason, not on merit.** It is true +and I re-measured it: **no per-app sidecar exposes metrics**. Only the shared proxy carries +`--metrics-address=0.0.0.0:44180` and a `metrics` container port; jackett, pihole, qbittorrent, +radarr, sonarr and crafty all have `(ABSENT)` and ports `['http']` alone, so the listener never +starts and nothing about these gates is scrapable. Adding it is worth doing and the pattern is +already proven in-tree by the shared proxy. But it is **not** "inside my file": the container's +`args` *and* its `ports` both come from `_oauth2_sidecar_container()`, so adding +`--metrics-address` plus a `metrics` port hits the identical five-file blast radius as the cookie +change. It belongs in the same single renderer commit — which is an argument for cutting that commit +soon, not for me to hand-edit a golden a second time after the gate already refused the first. +(Crafty is separately hand-written and could take the metrics flag on its own.) + +**Crafty is unaffected by all of this** — its sidecar is hand-written in +`platform/crafty/deployment.yaml`, not rendered, which is why it could be and was fixed alone on +2026-09-29. + +### What this branch does land + +`runbooks/oauth2-proxy-session.md` — the four sidecar log lines that tell Keycloak, the sidecar and +the app's own login apart (including the upstream `-` field that means the proxy answered a request +the app never saw), the cookie-collision cross-log diagnosis, the 5-minute-session check, the +302-vs-401 trap, and the generated-golden warning with its measured blast radius. Registered in the +runbooks `configMapGenerator` **and** listed in `runbooks/README.md`, so the URL resolves rather +than the file merely existing. + +## What I verified + +- **The collision is the live failure.** `[stored_session.go:94] Error loading cookied session: + cookie signature not valid, removing session` — **1 occurrence in jackett's log (15:23:44), 8 in + radarr's** (15:22:49, 15:23:58, 15:27:59, **15:31:29**, 15:32:29 ×4), and **zero** in pihole, + qbittorrent, sonarr, crafty and both shared-proxy replicas. The 15:31:29 → 15:31:32 pair is the + operator's whole report. +- **Six instances / seven pods are in the collision set** — the brief said five; the shared + forward-auth proxy in namespace `oauth2-proxy` is the sixth and runs 2 replicas. Read from live + pod specs: jackett, pihole, qbittorrent, radarr, sonarr and `oauth2-proxy` all have **no** + `--cookie-name` argument and `--cookie-domain=.home.keithjackson.dev`. Only crafty is fixed. +- **All seven cookie-secrets differ.** Compared in-cluster by sha256, first 12 hex only, values + never printed: `d9c5…`/`0b3b…`/`b025…`/`bb6a…`/`25f4…`/`e679…`/`911a…`. So no two of these proxies + can read each other's cookie — the precondition for the eviction. +- **The 5-minute lifetime**, from four `[AuthSuccess]` lines: `expires − created = 300.000 s`, with + `refresh_token:true` and `groups:[platform-admins]`. Live args confirm `--cookie-refresh` absent. +- **The 302/401 split** on `/api/` paths in this pod's log: 33 × 302, 18 × 200, 2 × 401, 2 × 204. +- **4 of the 6 POSTs never reached Jackett** — the access log's upstream field is `-`, not `/`: + `adminpassword` → 401, `adminpassword` → 401, `server/update` → 302, `server/update` → 302. The + two that did reach it (`adminpassword`, `indexers/1337x/config`) carry the authenticated email. +- **The proposed argument list starts, against the pinned image and the real issuer.** Ran + `quay.io/oauth2-proxy/oauth2-proxy:v7.8.1` locally with the full proposed args; OIDC discovery + against `keycloak.home.keithjackson.dev` succeeded and it printed back its own reading: + `Cookie settings: name:_oauth2_proxy_jackett secure(https):true httponly:true expiry:168h0m0s + domains:jackett.home.keithjackson.dev path:/ samesite: refresh:after 2m30s` and + `API route - Path: ^/api/`. +- **`--api-route` is singular.** The brief specified `--api-routes`, which does not exist in v7.8.1; + the proxy would have refused to start. Read off `--help`. +- **`--session-cookie-minimal` is mutually exclusive with `--cookie-refresh`** — provoked, not + assumed: `invalid configuration: cookie_refresh > 0 requires oauth tokens in sessions. + session_cookie_minimal cannot be set`. This kills the brief's premise that dropping + `--pass-access-token` shrinks the cookie: only `--session-cookie-minimal` strips tokens from the + session, and it is unavailable while refresh is on. Dropping those two flags is justified on + **least privilege and inertness**, not size. +- **`--set-xauthrequest` is inert here**: HTTPRoute `jackett-9117` backendRef is + `Service/jackett port 4180`, there is no ext-authz attachment on the `shared` Gateway, so + `/oauth2/auth` is never called. +- **`apps/**` is generated and the fix cannot be scoped to one app** — both halves measured, not + inferred: the gate's `render-golden` stage rejected the hand-edit, and re-rendering after a + renderer-only change listed all five apps as changed (transcript above). +- **Crafty's post-fix failure rate, bounded exactly.** Pod `crafty-647c7d8cc5-5spv9`, + `startTime=2026-09-30T03:57:19Z`, `restarts=0`, live args `--cookie-name=_oauth2_proxy_crafty` and + `--cookie-domain=crafty.home.keithjackson.dev`. Over that pod's whole lifetime: **3** + `No valid authentication`, 0 AuthSuccess, one client IP, 2 of the 3 on `/ws`. The 81:1 figure is a + 24 h window over a 13.9 h pod and is dominated by the pre-fix process. +- **The ratio metric ranks chattiness, not breakage.** Radarr 156:1 with the top paths being + `/api/v3/command`, `/api/v3/queue/details` and `/signalr/messages/negotiate` — UI polling, not 156 + distinct user-visible failures. +- **No per-app sidecar exposes metrics.** `--metrics-address` is `(ABSENT)` and `ports` is + `['http']` on all six per-app sidecars; only the shared proxy has + `--metrics-address=0.0.0.0:44180` and a `metrics` port. +- **Gate:** `./ci/gate.sh` in the pinned toolbox, **rc 3 — UNDETERMINED, not green**. Green: + `spec-immutability`, `approval-policy`, `decisions-log`, `lint`, `l0`, `render-golden`, `l1`. + Skipped: `gitea-settings` (no `SETTINGS_API_TOKEN`) and L2/L3 conformance (`CLUSTER_AVAILABLE` + unset), leaving touched requirement `R-OBS-04` uncertified. Note `spec-immutability` is **green**, + not skipped — mounting the sibling spec repo at `/home-network-k3s` (the path + `ci/spec-immutability.sh` actually defaults to) in the ENGINE shim fixes that skip; the commonly + circulated shim mounts it at its host path, where `SPEC_REPO` never finds it. +- **Gitea CI on this PR, read from the commit's `/statuses` rather than from a tick.** Two contexts + reported, and **only one of them ran**: + + gate / gate (pull_request) state=success desc='Successful in 1m41s' + gate / conformance (pull_request) state=success desc='Has been skipped' + + That second line is §2b's trap in the wild: a **skipped** Gitea job reports `state=success`. The + conformance job did not execute, so the green tick beside it certifies nothing. Read together with + the local rc 3 above, the two agree exactly — the offline L0/L1 subset passed and L2/L3 never ran. + Anyone reading the PR's check mark alone would conclude otherwise. +- **`main` is protected**, re-measured this session rather than assumed: + `GET /repos/.../branches/main` → `protected=true`, `user_can_push=false`. + +## What I did NOT verify + +- **The operator's problem is NOT FIXED by this branch.** Only the diagnosis and the runbook land. + Jackett's sidecar is unchanged and will keep losing its session to Radarr until the renderer + change ships. **This is the single most important line in this entry.** +- **No browser round-trip.** Every mechanism is measured from logs and specs and the argument list + is proven to start, but nobody has added an indexer through the proposed configuration. The human + test is below, for whoever lands the renderer change. +- **Nothing was applied.** No `kubectl apply/edit/patch`, no `flux reconcile`. The running pod is + still `jackett-c5bfcbfb8-c75f9` (2/2 Running, 17 d, 0 restarts — unchanged by this session's + read-only probes). Note that when the fix does land, merging is what deploys, and it will restart + the pod (`strategy: Recreate` on an RWO Longhorn volume), briefly interrupting Radarr's and + Sonarr's direct `:9117` access. +- **Cookie size and chunking: UNMEASURED, and I will not claim either way.** I could not read a + `Set-Cookie` header without a browser. I found no `_oauth2_proxy_0`/`_1` chunk evidence and no + 4 kB warning in 36,287 log lines — but I did **not** establish that v7.8.1 emits such a warning at + this log level, so that absence is not evidence. If sessions still misbehave after the fix lands, + this is the next thing to measure, with `--session-store-type=redis` as the escape hatch, since + `--session-cookie-minimal` is ruled out (above). +- **`--cookie-refresh=2m30s` has never executed a refresh here.** It depends on Keycloak accepting + the refresh token and on the realm's SSO Session Idle/Max, none of which I read (that needs admin + credentials). If Keycloak's *refresh* token is also short-lived, the five-minute logout persists + and the realm is where to look next. +- **`--api-route` is measured for Jackett only.** PiHole and qBittorrent were not measured, and + PiHole's UI does not serve `/api/`. Do not apply it renderer-wide on my evidence. +- **The four sibling apps still collide with each other** regardless of what happens to Jackett. + The same operator report should be expected from any of pihole / qbittorrent / radarr / sonarr. +- **L2/L3 conformance did not run** (`CLUSTER_AVAILABLE` unset), so nothing about runtime behaviour + is certified by the gate. `gitea-settings` also skipped, so branch protection and runner state are + **unverified by me this session** — I did not re-measure them against the Gitea API. +- **I did not query Loki.** The Crafty bounding above comes from the pod log, which I argue is the + better instrument (an exact process boundary rather than a wall-clock window against a 72 h + retention). But it means I cannot speak to the 24 h or 12 h *platform-wide* ratios, and a + per-app ratio over a window longer than the current pod's life is not something I measured at all. +- **Collision causation is one adjacency, not a population.** The 15:31:29 → 15:31:32 pair is three + seconds and the mechanism is documented and present, but I did not enumerate every jackett failure + and match it to a radarr ClearSession. Treat mechanism 1 as **necessary and firing, not proven + sufficient** — the coordinator's framing, and I agree with it on this evidence. +- **The metrics change is NOT in this branch** (reason above). Jackett therefore remains inside the + oauth2-proxy metrics gap, and the ratio alert shipped in PR #29 still cannot see it. +- **The runbook's commands were not all executed verbatim.** The log-reading ones I ran; the + per-namespace `for` loops are edited from the scripted forms I actually used. + +### The human test — for whoever lands the renderer fix + +After it merges and the pod rolls (watch for a **new** pod name at 2/2): + +1. **Sign in** at `https://jackett.home.keithjackson.dev/UI/Dashboard`. +2. **Open Radarr in a second tab** and sign in there too. **This is the step that breaks Jackett + today** — it is the 15:31:29 line. Return to the Jackett tab. +3. **Add an indexer** — *+ Add indexer*, pick one, *Configure*, *Okay*. It must save. +4. **Wait six minutes on the open dashboard, then save another.** That is the `--cookie-refresh` + test; the session currently dies at five. +5. **If something does still fail, it must now fail LOUDLY** — DevTools → Network, a broken `/api/` + call shows **401**, not a 302 followed by HTML. A silent success-looking no-op means the change + did not take; check the running pod's args before assuming anything else. +6. **A plain page reload must still sign you in silently** (`/UI/Dashboard` is outside `^/api/`). + A 401 page on reload means the regex is too broad — that is a regression. diff --git a/decisions/2026-09-30-node-health-render-device.md b/decisions/2026-09-30-node-health-render-device.md new file mode 100644 index 0000000..d5438d9 --- /dev/null +++ b/decisions/2026-09-30-node-health-render-device.md @@ -0,0 +1,283 @@ +--- +date: 2026-09-30 +branch: feat/node-health-render-device +title: The Intel render device is now self-re-measuring, not a fact someone checked once +--- + +## What changed + +### Why this probe has value at all + +**Nothing else in the platform would notice the device going missing.** Plex exposes no +ServiceMonitor, so no alert fires on a silent fall back to software transcoding. And the +manifest side mounts the device with `hostPath type: DirectoryOrCreate`, which means a node +*without* the device mounts an **empty directory** — Plex starts, runs, and degrades quietly +rather than failing loudly. There is no crash loop, no failed mount, no alert: just a hot CPU. +This check is the only thing in the system that would surface it, which is the actual +justification for the role carrying it. + +### The change + +`roles/node_health` grew a graphics / render-device probe. Nothing was installed, no module +was loaded, no group was changed, nothing was rebooted: the prod nodes already have a working +Intel render device (measured by hand on all three on 2026-09-30) and the point of this change +is that the measurement now **re-runs forever** instead of decaying silently in a handoff note. + +The failure this exists to catch is silent by construction. Plex hardware-transcodes through +`/dev/dri/renderD128`; if that node stops existing — node rebuild, a BIOS change that disables +the onboard IGD, a kernel that binds a different DRM driver, a driver regression — Plex falls +back to software transcoding on its own and the only symptom is a hot CPU. + +Four files: + +- `files/node_health_collect.py` — new `collect_graphics()`, wired into the **unprivileged** + pass (`main()`). Everything it reads is world-readable: `lstat()` on `/dev/dri/*`, + `grp.getgrgid()` for the owning group name, the `/sys/class/drm/card*/device/driver` symlink + target for the bound driver, `/sys/bus/pci/devices/*/class` for display-class (PCI base class + `0x03`) devices whether or not a driver is bound, `/sys/class/drm/card*/engine/*` for the + `vcs*`/`vecs*` media engines, `/proc/modules`, `/proc/cmdline`, and + `/sys/module/{i915,xe}/parameters/enable_guc`. `lspci` and `vainfo` were added to the + existing tool probe; both are optional and neither is depended on. +- `tasks/assess.yml` — one shared derive task plus three checks: `graphics.render_device` + (PASS/FAIL/SKIP), `graphics.drm_driver` (PASS/WARN/FAIL/SKIP), `graphics.media_stack` + (INFO/SKIP). +- `defaults/main.yml` — `node_health_graphics_expected_driver: i915`, so the one comparison the + check makes is data rather than task logic. +- `templates/summary.txt.j2` — a `GRAPHICS / RENDER DEVICE` section, and + `roles/node_health/README.md` gained a section explaining the design and two new "known + limits". + +Three design rulings worth stating, because each is the kind of thing a later edit would get +wrong: + +1. **The render node's GID is reported, never asserted.** It read `root:render`, gid **992**, + mode `0660` on all three prod nodes. The check does *not* compare against `992`. `render` is + a **systemd dynamic** allocation (`systemd-sysusers`), not a Debian static gid from + `base-passwd`, so a rebuild can legitimately move it — and Plex's container entrypoint stats + the device and joins whichever group owns it, so drift genuinely is not a fault. The + observed group name and number appear in the detail line, which makes drift *visible* + without making it an error. A hard `!= 992` would be a false alarm waiting for a reinstall. +2. **The PCI scan is deliberately separate from the DRM scan.** `/sys/class/drm/card*` exists + only once a driver has *bound*, so a GPU with nothing attached has no `card0` and would + otherwise be indistinguishable from a machine with no GPU. Scanning the PCI bus separately + is what lets "no display device at all" be an honest **SKIP** (a VM guest — the same + convention `thermal.max` already uses for "no sensor on this hardware") while "a GPU is on + the bus and there is still no render node" is a **FAIL**. +3. **GuC/HuC and host VAAPI userspace are INFO and must stay INFO.** GuC/HuC off + (`i915.enable_guc=0`) is an accepted state here and costs low-power H.264 encode *quality* + only. No host `vainfo` is not a defect at all — Plex ships its own libva and media driver + inside its container. Grading either would put a permanent WARN on a chosen configuration, + or invite someone to `apt install vainfo` on a node to clear a health check, and this role + installs nothing. + +`xe` bound instead of `i915` is a **WARN**, not a FAIL: the device still works, but the two +drivers do not share a userspace media stack. On these nodes `xe.ko.xz` exists on disk and is +bound to nothing. + +## What I verified + +### The real prod run (read-only, all three nodes) + +`--list-hosts` first, every time. It named exactly three hosts and nothing else: + + play #1 (k3s_prod): Inspect node hardware health (read-only) + pattern: ['k3s_prod'] + hosts (3): k3s-prod-3 k3s-prod-2 k3s-prod-1 + +Then `--syntax-check` (rc 0), then the run: + + ansible-playbook playbooks/node-health.yml -e node_health_hosts=k3s_prod --limit k3s_prod + + k3s-prod-1 : ok=51 changed=3 unreachable=0 failed=0 skipped=1 + k3s-prod-2 : ok=51 changed=2 unreachable=0 failed=0 skipped=1 + k3s-prod-3 : ok=51 changed=2 unreachable=0 failed=0 skipped=1 + PLAY rc=0 + +(The `changed=` counts are the *controller-side* artifact writes under +`infra/ansible/_artifacts/node-health/`, delegated to localhost. Nothing on any node changed — +the collector task is `changed_when: false` and the role touches nothing.) + +All three nodes graded **PASS / PASS / INFO** on the new checks, 0 FAIL overall: + + k3s-prod-1 VERDICT 0 FAIL · 1 WARN · 18 PASS · 2 SKIP · 6 INFO + k3s-prod-2 VERDICT 0 FAIL · 3 WARN · 15 PASS · 3 SKIP · 6 INFO + k3s-prod-3 VERDICT 0 FAIL · 2 WARN · 17 PASS · 2 SKIP · 6 INFO + + [PASS] graphics.render_device — DRM render node present for hardware transcoding (/dev/dri/renderD*) + [PASS] graphics.drm_driver — DRM driver bound to the display device (expected i915) + [INFO] graphics.media_stack — Media engines, GuC/HuC firmware and host VAAPI userspace (informational) + +(The pre-existing WARNs and SKIPs are other checks — SMART, memattr, resolvers — and are not +mine; none of them moved.) + +The new report section, **byte-identical on all three nodes**: + + -------------------------------------------------------------------------------- + GRAPHICS / RENDER DEVICE (the Plex hardware-transcoding path) + -------------------------------------------------------------------------------- + pci 0000:00:02.0 00:02.0 VGA compatible controller [0300]: Intel Corporation CoffeeLake-S GT2 [UHD Graphics 630] [8086:3e98] (rev 02) + driver i915 + drm card0 driver i915 at 0000:00:02.0 engines bcs0 rcs0 vcs0 vecs0 + dev card0 character special mode 0660 root:video gid 44 + dev renderD128 character special mode 0660 root:render gid 992 + GuC/HuC NOT READ — /sys/module/i915/parameters/enable_guc is not + readable without root (i915 module parameters are mode 0400) + no i915/xe parameter on the kernel cmdline + modules amdgpu=false, i915=true, nouveau=false, nvidia=false, radeon=false, xe=false (/proc/modules — corroboration only; the bound-driver symlink above decides) + vainfo NOT installed on this host — not a defect: Plex ships its own libva/media driver inside its container + +That reproduces every item of the 2026-09-30 hand measurement independently: `renderD128` +character special, `root:render`, **gid 992**, mode `0660`; `card0` `root:video` gid 44 mode +`0660`; `i915` bound; `xe` **not** loaded and bound to nothing; `[8086:3e98] (rev 02)` +CoffeeLake-S GT2 UHD Graphics 630 at `00:02.0`; `vcs0` and `vecs0` live; no host `vainfo`. + +**One measured correction to my own phase-1 assumption:** `lspci` **is** present on all three +prod nodes (`/usr/bin/lspci` — Debian ships `pciutils` by default). I had written that it was +absent. It is still *treated* as optional, because `pciutils` is genuinely not declared in +`node_baseline_packages` and depending on an undeclared package would be wrong — but the +comments and README that asserted it was missing were factually wrong and are now corrected. +Its presence is why the `pci` line above carries a readable device name rather than bare IDs. + +### The gate + + DEV_KUBECONFIG=/dev/null ENGINE= ./infra/devcontainer/dev.sh run bash -c './ci/gate.sh' + GATE rc=3 + +**rc 3 — UNDETERMINED. NOT green.** Nothing failed, and nothing is certified either. Exactly +three stages did not run, all three the known local ceiling from a worktree, and **no fourth +skip**: + + [skip] spec-immutability spec repo not present at /work/../home-network-k3s (set SPEC_REPO) + [skip] gitea-settings SETTINGS_API_TOKEN is not set — cannot read the live Gitea rule + [skip] conformance no cluster configured (CLUSTER_AVAILABLE!=true) — L2/L3 NOT RUN + + [ ok ] decisions-log: pass [ ok ] lint: pass [ ok ] l0: pass + [ ok ] render-golden: pass [ ok ] l1: pass [ ok ] approval-policy: pass + +Both CLAUDE.md §3 worktree workarounds were needed and both worked: `DEV_KUBECONFIG=/dev/null` +(the credential in the main checkout was **not** copied) and an `ENGINE` shim that bind-mounts +the main checkout's `.git` read-only. No `tracked_files()` traceback appeared, which is how I +know the shim took effect rather than the git stages silently degrading. + +`ansible-lint` was **not** run, because **`ci/` does not invoke it anywhere** — I grepped +`ci/` and `.gitea/` and the lint stage is `yamllint -c ci/yamllint.yaml` only. I am not +claiming to have matched a lint convention that does not exist. + +### The CI gate on the PR — a green tick that is exit 3, read from the log + +The required context `gate / gate (pull_request)` reported **`success`, "Successful in 1m40s"** +on commit `a78f010`. That description matters: a *skipped* Gitea job reports `success` with +"Has been skipped", so this one genuinely ran, and 1m40s matches CLAUDE.md §2's ~1m35s. + +**But the green tick does NOT mean `gate.sh` returned 0.** `.gitea/workflows/gate.yml:138` is +`run: ./ci/gate.sh || [ "$?" -eq 3 ]` — the job tolerates rc 3 by design. So I fetched the job +log (`/actions/runs/173/jobs/0/logs`; the `/api/v1/.../tasks/{id}/logs` endpoints 404 on this +Gitea, the web route serves it) and read the gate's own headline: + + [fail] MERGE GATE: UNDETERMINED — greenness was NOT ESTABLISHED. + [fail] Stages that did not run: spec-immutability + +**So CI is also rc 3, with exactly ONE stage skipped — `spec-immutability`.** The green Gitea +check is the tolerance swallowing that 3, not a rc 0. + +**CI verified two things my local run could not, and both came back green:** + + [ ok ] stage gitea-settings: pass + [ ok ] enable_push = false enable_status_check = true + [ ok ] required_approvals = 1 block_admin_merge_override = true + [ ok ] block_on_rejected_reviews = true dismiss_stale_approvals = true + [ ok ] runners online: 1 (>= 1) + [ ok ] Gitea settings match the declaration. + [ ok ] stage approval-policy: pass + [ ok ] no architecturally/security-relevant paths touched — the GATE is the approver. + +My local run **skipped** `gitea-settings` (no `SETTINGS_API_TOKEN`) and could not give +`approval-policy` a PR number. CI has the secret and `REQUIRE_GITEA_SETTINGS: "true"`, so a skip +there would have failed the job. That is measured confirmation that the live Gitea matches +`infra/gitea/expected-branch-protection.json`, and that `infra/ansible/**` needs no human +reviewer. `lint` passed over 405 files. `conformance` SKIPPED in CI too +(`CLUSTER_AVAILABLE != true`), so L2/L3 remains unverified there as well. + +Corroborated independently: `GET /repos/.../branches/main` → +`protected=true user_can_push=false enable_status_check=true required_approvals=1`, +contexts `["gate / gate (pull_request)"]`; `GET /admin/actions/runners` → `incluster-dind +status=online`. And note the commit's **combined** state is `pending`, not success, because +`gate / conformance (pull_request)` sits at "Blocked by required conditions" — that context is +not in the required list, so it does not block, but the PR is not "all green" either. + +The gate's `lint` stage passed with one `line too long` **warning** on +`roles/node_health/tasks/assess.yml:189` — that is pre-existing, in the SMART block, well +above my insertion point, and warnings do not fail the stage. + +### Offline work (before the lock was released) + +- `python3 -m py_compile` on the collector: clean. +- `collect_graphics()` executed for real against this workstation's hardware (an unrelated + machine, Raptor Lake-S UHD Graphics + an NVIDIA discrete GPU) and returned the whole + structure with no exception, correctly reporting two cards, two render nodes, the bound + driver per card, the `vcs0/vcs1/vecs0` media engines, and `enable_guc` as *not readable + without root* rather than as a fabricated `0`. +- That last point is a real finding and changed the design: `/sys/module/i915/parameters/*` is + **mode 0400**, so an unprivileged pass cannot read `enable_guc`. It now reports + `NOT READ — ` and `/proc/cmdline` is read as the world-readable second source. An + earlier draft would have silently reported a null. +- Two other bugs found and fixed by running it rather than reasoning about it: + `/dev/dri/by-path` is a **directory** and was being `lstat()`ed as a device; and + `/sys/class/drm` carries one entry per *connector* (`card0-HDMI-A-1`), which had to be + excluded from the card scan. +- All three checks and the new template section were rendered through a standalone Jinja + harness (Ansible is not on the host — CLAUDE.md §5) against **seven** fact sets: the real + workstation; GPU present with the render node removed; no display device at all; `xe` bound + in place of `i915`; a display device with **no** driver bound; `renderD128` present but not a + character device; and a shape with neither `lspci` nor `vainfo`. Zero template errors, and + the statuses came out as designed — `FAIL/PASS/INFO`, `SKIP/SKIP/SKIP`, `PASS/WARN/INFO`, + `FAIL/FAIL/INFO`, `FAIL/PASS/INFO` respectively. The harness supplies stand-ins for the + Ansible-only filters (`bool`, `dict2items`, `flatten`, `zip`, `regex_replace`) and matches + Ansible's `trim_blocks=True`. + **Read that for exactly what it is: evidence that the LOGIC is right, and no evidence + whatsoever that the HARDWARE DETECTION is.** Six of those seven fact sets were hand-mutated + dictionaries, not machines. They are not hardware tests and must not be written up as if + they were. + +## What I did NOT verify + +- **The FAIL path has never fired on a real node.** Every prod node has a working render + device, and this role is read-only, so a device cannot be taken away to prove the check + catches its absence. The failure branches were exercised against **synthetic fact sets + only** (see above). I am not claiming the failure path is tested against hardware. +- **`xe` detection is likewise untested against reality.** No node here has `xe` bound — the + module exists on disk as `xe.ko.xz` and is bound to nothing — so the WARN branch was only + ever driven by a mutated fact set. If `xe` ever does bind, the *shape* of + `/sys/class/drm/card*` under it is something I have assumed matches i915's (same + `device/driver` symlink, same `engine/*` directories). That assumption is unverified. +- **The media-engine sysfs path is confirmed on these nodes but not proven portable.** + `vcs0`/`vecs0` were read from `/sys/class/drm/card0/engine/*` on all three prod nodes, so it + is measured here. I did **not** confirm against kernel source that no other kernel version or + driver publishes engines elsewhere. If one does, the check reports "none exposed" — an INFO + line, not a false verdict, which is why this failure mode is tolerable. +- **`enable_guc` is still unread on prod.** The report says `NOT READ — mode 0400`, which is + honest but means I have **not** confirmed by measurement that GuC/HuC is off on these nodes; + I only know the kernel cmdline carries no `i915.enable_guc=` override. The brief's ground + truth says `i915.enable_guc=0`; my check does not corroborate that value, it declines to + guess it. Reading it would need root, and it is INFO either way, so I left it alone. +- **Not a transcode test.** These checks prove the render *device* exists, is a character + device, and which driver owns it. They do **not** prove a transcode succeeds — that needs a + VAAPI round trip inside Plex's own container with Plex's own libva, and is out of scope for a + node-level check that installs nothing. +- **Greenness was never established, locally or in CI.** Both runs returned **rc 3**. My local + run skipped three stages (`spec-immutability`, `gitea-settings`, `conformance`); CI skipped + one (`spec-immutability`) and its own headline says `MERGE GATE: UNDETERMINED`. **I am not + describing this PR as green**, and the green Gitea tick must not be read as one: the workflow + line is `./ci/gate.sh || [ "$?" -eq 3 ]`, so it is a tolerated 3. +- **L2/L3 conformance did not run anywhere** — `CLUSTER_AVAILABLE != true` in both. Whatever + runtime requirements this touches remain UNVERIFIED; only the L0/L1 offline subset passed. +- **`spec-immutability` ran in neither.** The sibling spec repo is checked out in neither my + worktree nor the CI workspace, so nothing confirms this change does not drift from the spec. + It is the expected, tolerated skip — but it is still a skip, not a pass. +- **No claim about the live platform beyond what was printed.** Per CLAUDE.md §2b this repo is + not evidence about prod; the only prod facts I assert are the ones quoted verbatim above. + Branch protection and runner state I did query directly, and the CI `gitea-settings` stage + passed — so those two I assert on measurement, not on this file's prose. +- **The stale dev inventory was left alone deliberately.** `hosts.ini` still defines the + `k3s-dev-*` groups as the default target of most playbooks and the dev cluster no longer + exists. That is real, it is out of this change's scope, and it is unclaimed work. diff --git a/decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md b/decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md new file mode 100644 index 0000000..edd3b25 --- /dev/null +++ b/decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md @@ -0,0 +1,378 @@ +--- +date: 2026-09-30 +branch: fix/oauth2-proxy-cookie-collision-fleet +title: The cookie collision was never Crafty's — it was the renderer's, and it shipped to five apps +--- + +## What changed + +Follow-up #3 of `decisions/2026-09-29-crafty-oauth2-proxy-signin.md`: *"Any further per-app +reverse-proxy sidecar needs a unique `--cookie-name` too. The collision is a property of the +pattern, not of Crafty."* + +It is a property of the pattern, and this branch found where the pattern actually lives. + +**Read the scope claim carefully before the detail.** This branch fixes THREE defects in the +forward-auth sidecar contract, all in one function, all landing on all five apps at once: + +| Defect | Fix | Measured evidence | +|---|---|---| +| Six independent instances, seven different cookie-secrets, **one** cookie name on **one** zone-wide domain — so they overwrite and `ClearSession`-delete each other's sessions | `--cookie-name=_oauth2_proxy_` + host-scoped `--cookie-domain` | 7/7 distinct secrets; 6 instances on the default name; radarr logs the eviction fingerprint 8× in 12h | +| The session dies **5 minutes** after sign-in — `--cookie-refresh` defaults to 0 = *disabled*, so the refresh token the proxy already holds is never spent | `--cookie-refresh=2m30s` (half the lifetime) | access-token lifetime = **300.000s exactly**, `refresh_token:true`, flag absent | +| **No** per-app sidecar serves `/metrics` at all — without `--metrics-address` the listener never starts | `--metrics-address=0.0.0.0:44180` + a `metrics` container port | `ports` was `['http']` on all six | + +Rejection *volume* is dominated by the second defect, not the first: ~95% of radarr's rejections +carry no cookie at all. **Do not merge this expecting one number to drop** — three mechanisms were +tangled together, and separating them is most of the value here. + +**Boundary note.** My brief originally reserved `apps/wave-0/jackett.yaml` for a concurrent branch. +That split is not achievable and has been revoked: the file is generated, the fix is one renderer +edit, and `render-golden` requires all five goldens regenerated in the *same commit*. I own all +five. The proof I ran before touching jackett is under "What I verified". + +**The defect was not in the four manifests it was reported in.** `apps/wave-0/pihole.yaml`, +`apps/wave-0/qbittorrent.yaml`, `apps/wave-1/radarr.yaml`, `apps/wave-1/sonarr.yaml` — and +`apps/wave-0/jackett.yaml` — are **generated goldens**. One function emits all five sidecars: +`catalog/render.py::_oauth2_sidecar_container()`, which hardcoded `--cookie-domain=.{ZONE}` and +emitted no `--cookie-name` at all. Hand-editing a golden is not a fix twice over: `ci/render-golden.sh` +FAILS on the drift (provoked, not assumed — see below), and the next re-render silently reverts it. + +So the fix is in the generator, and both flags are now **derived** from the ServiceDefinition: + + --cookie-name=_oauth2_proxy_{s['name']} unique per service, by construction + --cookie-domain={host} the app's own route host, not the zone + +`--whitelist-domain` deliberately stays zone-wide — it governs permitted post-login *redirect* +targets, not cookie scope. `catalog/render.py --selftest` gained four assertions, including one +that renders a **second** synthetic forward-auth service and requires the two cookie names to +differ, so a future refactor to a constant fails in the renderer's own test. + +Re-rendering with the canonical command regenerated **five** goldens, not four. Jackett is the +fifth, and this is the one thing in this branch the PM brief did not anticipate — see +"the jackett collision" below. The three `conformance/tests/l1-rendering/*/golden/` fixtures are +expected-output copies of the same renderer, so they are regenerated too; that is not collateral +damage, it is the L1 stage doing its job (it FAILED on the first gate run and named the drift). + +**The new gate check** is `conformance/tests/l0-static/oauth2-proxy-cookie-names-distinct/`, +declaring `conformance.home.k3s/scope: whole-repo` so `ci/l0.sh`'s `--include-scope` union runs it +on **every** gate invocation. No bespoke `ci/l0.sh` stage was added — `whole-repo-invariants-wired` +fails the gate if anyone adds one, and the scope annotation is the supported mechanism. + +It inventories every oauth2-proxy instance the repo authors (containers at any depth, plus the +shared proxy's HelmRelease values) and resolves each one's **effective** cookie name — its declared +name, **or oauth2-proxy's built-in `_oauth2_proxy` when it declares none**. That single modelling +decision is the whole check: an omitted flag is not an absence, it is a *claim on a name*, and +treating it as an absence is precisely why five reviewers read these manifests and saw nothing. +Then: (1) no two effective names collide; (2) no name is another's name + `_csrf`, because that is +how oauth2-proxy derives the CSRF cookie and two such session names look distinct while their CSRF +cookies collide; (3) the default name has exactly one owner, and it is the one registered in +`SHARED_DEFAULT_COOKIE_OWNER`; (4) every non-registered instance is host-scoped; (5) +`catalog/render.py` derives rather than lists; (6) the **shipped** renderer is proven end-to-end on +two synthetic ServiceDefinitions run as a subprocess. + +**On the shared proxy — "distinct" or "explicitly declared"?** The brief asked me to decide, and +the answer is *neither, quite*. The shared forward-auth proxy at `platform/oauth2-proxy/` **must +keep** `_oauth2_proxy`: it is the single session every forward-auth-gated surface without its own +sidecar rides on, so renaming it signs all of them out at once. Requiring *every* instance to +declare a name would therefore need an exemption, and an exemption is a hole: delete the shared +proxy and whichever app next omitted the flag would silently inherit the default with the rule +still satisfied. So the assertion is plain **distinctness over effective names**, with the default +treated as a name like any other — held by **exactly one owner, named in a shrink-only register +with its reason**. The shared proxy needs no exemption from the rule; it is simply the registered +holder of that one name. A stale register entry FAILS (the exemption would outlive its object) and +an unregistered holder FAILS (no silent heirs). Both were provoked and are quoted below. + +**On host-scoped `--cookie-domain` — overreach, or not?** Asserted, and here is the argument both +ways. *Against:* distinct names alone close the eviction bug completely — two differently-named +cookies do not overwrite each other however wide their domains are — so this assertion fixes +nothing (1) has not already fixed, and a check that fails a legitimate configuration is a check +someone eventually disables. *For, and this wins:* a zone-wide cookie is **transmitted by the +browser to every host in the zone**. qBittorrent's session token was being sent to pihole, radarr, +sonarr and every other name under the zone, including hosts behind no proxy at all — a +session-credential disclosure across trust boundaries that nothing needs. And it is what makes a +*future* name collision catastrophic rather than contained: with host-scoped cookies, two +instances that accidentally share a name cannot see each other's cookie at all. Defence in depth +on the exact axis that already failed once. The escape hatch is the same register, by name and +with a reason, rather than a loosened rule. + +### The jackett boundary, and why it could not hold + +`apps/wave-0/jackett.yaml` carries the identical defect and was initially assigned to a concurrent +branch, with an instruction to me not to touch it. **That separation is not achievable** — the +ownership split was drawn across a generated file — and I did not discover this by reasoning about +it, I provoked it. With the renderer fixed and jackett's golden left at its `main` content, +`./ci/render-golden.sh` exits **1**: + + [fail] GOLDEN DRIFT: apps/wave-0/jackett.yaml does not match the canonical render: + - - --cookie-domain=.${BASE_DOMAIN} + + - --cookie-name=_oauth2_proxy_jackett + + - --cookie-domain=jackett.${BASE_DOMAIN} + [fail] render==golden check FAILED + +Jackett's golden is an *output* of the file that contains the defect. There are exactly three +options and two of them are worse: hand-edit four goldens and leave the generator broken (fails +`render-golden`, and reverts on the next render); add a per-service carve-out to the renderer so +jackett alone keeps defaulting (ships the exact "list, don't derive" anti-pattern this check +exists to forbid, and leaves jackett broken *by design*); or fix the generator and regenerate all +five. I took the third. Jackett's golden here is **machine output of a one-line generator change**, +not an independent edit of Mira's file, and it is byte-identical to what her branch must produce. + +**Consequence:** the jackett branch has no cookie work left in it — this commit fixes jackett +identically and by the same mechanism, and both branches independently arrived at the *same two +values* (`_oauth2_proxy_jackett`, `jackett.${BASE_DOMAIN}`), which is a good sign for the +diagnosis. Its hand-edit of the rendered file will fail `render-golden` on its own terms, so it +should be closed rather than rebased. **What must not be lost with it** is its evidence: the +interleaved radarr/jackett log showing `removing session` deleting a clean session 26s after login, +and the `expires − created = 300.000s` measurement that produced `--cookie-refresh` here. Both are +carried into this entry and into the renderer's comments; the branch itself is now redundant, the +work behind it is not. + +**The general lesson, which is worth more than this bug:** an ownership split across +`apps/wave-*/*.yaml` cannot hold, because those files are *outputs*. Two agents assigned two +rendered apps are assigned the same function and will collide in `render-golden` no matter how +carefully they stay in their own files. Ownership of a rendered app is ownership of +`catalog/render.py`, and briefs should say so. + +### What this branch deliberately does NOT do + +- **`--api-route=^/api/` is NOT added**, and not because of scope discipline alone: it would be + **wrong** renderer-wide. It was measured on jackett, and **PiHole does not serve `/api/`** — so + a renderer constant applies one app's path shape to five. Doing it properly means declaring API + path prefixes per service, and the `auth` block is `additionalProperties: false`, so that is a + *schema change*. Separate PR. (Note the flag is **`--api-route`, singular** — verified in the + pinned binary's own `--help`. The plural `--api-routes` does not exist in v7.8.1 and the proxy + refuses to start with it. That error was propagating through briefs; it stops here.) +- **`--session-cookie-minimal` is NOT added** and must not be, as a cookie-size fix: it is + **mutually exclusive** with `--cookie-refresh` (*"cookie_refresh > 0 requires oauth tokens in + sessions"*). Dropping `--pass-access-token` is a least-privilege argument, never a size one. +- **Metrics are NOT scraped, only served.** The flag and container port are the prerequisite; + a container port is not a Service port. These Services expose the app port and `:4180`, and each + namespace's default-deny baseline is `ingress: []`, so Prometheus can neither resolve nor reach + `:44180` until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor + exist. That wiring is someone else's follow-up and I have deliberately not faked its start. + **Nobody should read this commit as "the sidecars have metrics now."** +- **The new check was NOT widened to cover cookie-refresh or metrics.** Its subject is cookie + *identity*; those two share neither its inputs nor its failure mode. Bundling them would make + the check's name a lie and couple unrelated ratchets, so a future metrics decision could only + be expressed by weakening a *cookie* check. The cost of that discipline is named honestly in + the follow-ups: those two flags are currently **ungated**. + +## What I verified + +- **The premise, in-cluster, rather than inherited from the handoff** (CLAUDE.md §2b). Read-only, + `kubectl --context k3s-prod`, through the pinned toolbox. + - **Seven** `*-oauth2-proxy` Secrets carry a `cookie-secret`: crafty, jackett, pihole, + qbittorrent, radarr, sonarr, and the shared `oauth2-proxy-credentials`. **All seven values are + DISTINCT.** Compared without printing anything: each value was hashed **in-cluster-side script, + never echoed**, and only the first 8 hex characters of each `sha256` were emitted — enough to + say same-or-different, recovering nothing. Secret *key names* were listed; no `.data` or + `.stringData` was printed (CLAUDE.md §6). **So no instance can decrypt another's cookie — it can + only overwrite it. The premise holds.** + - **The default cookie names really were in use.** Of 8 live oauth2-proxy containers, **6 declare + no `--cookie-name` at all** — jackett, pihole, qbittorrent, radarr, sonarr, and both replicas of + the shared proxy — so all six emit `_oauth2_proxy` / `_oauth2_proxy_csrf`. All six also carry + `--cookie-domain=.home.keithjackson.dev`. **One cookie pair, six independent instances.** + - Crafty's live pod (`crafty-647c7d8cc5-5spv9`) already carries + `--cookie-name=_oauth2_proxy_crafty` and `--cookie-domain=crafty.home.keithjackson.dev`, so + PR #22 is merged and reconciled. It is the only compliant per-app instance on the cluster. +- **THE COLLISION IS REAL. IT IS NOT ESTABLISHED AS THE CAUSE OF THE REPORTED FAILURES.** These + are two different claims and only the first is supported. I was handed counter-evidence — + crafty, the app already fixed, showed the *worst* rejection ratio of the set over a 24h Loki + window (`crafty 81:1`, `radarr 67:1`, `jackett 39:4`) — and it is right to refuse the causal + claim. What I could do was bound the window. Read from **each sidecar's own log** rather than + Loki, because `kubectl logs` on a pod that has not restarted cannot reach a pod older than + itself, so a window inside the pod's age is inside the fixed spec's lifetime *by construction*: + + namespace age(h) rst reject success ratio cleared badsig + crafty 13.9 0 3 0 n/a 0 0 <- FIXED spec, whole life + radarr 422.5 0 155 1 155.0:1 8 8 + jackett 422.6 0 39 4 9.8:1 1 1 + pihole 395.7 0 0 0 0:0 0 0 + qbittorrent 404.1 0 0 0 0:0 0 0 + sonarr 422.5 0 0 0 0:0 0 0 + oauth2-proxy 422.6 0 0 0 0:0 0 0 (both replicas) + + `reject` = `No valid authentication in request`; `success` = `AuthSuccess`; + `cleared` = `removing session` (oauth2-proxy's `ClearSession`); `badsig` = `cookie signature + not valid`. Window `--since=24h`, which exceeds every pod's age only for crafty — whose pod is + 13.9h old with **0 restarts**, so these are its complete post-fix figures. + + Then I read crafty's four auth-relevant lines rather than counting them, because at n=3 the + content is the evidence: + + 03:57:20 Cookie settings: name:_oauth2_proxy_crafty secure(https):true httponly:true + expiry:168h0m0s domains:crafty.home.keithjackson.dev path:/ + 04:02:55 No valid authentication in request. Initiating login. + 15:22:39 No valid authentication in request. Initiating login. + 15:22:44 No valid authentication in request. Initiating login. + + 6743 log lines; those are all four. **Zero** `removing session`, **zero** `cookie signature + not valid`, zero anything else. + + Four conclusions, kept separate: + 1. **`crafty 81:1` is a window artefact. It is not evidence against the fix and must not be + cited as such.** The fixed pod emitted **3** rejections in 13h 52m; the 81 came from a 24h + window straddling it, i.e. from the pre-fix process. + 2. **The eviction FINGERPRINT is gone where the fix is deployed.** `cookie signature not + valid` + `removing session` is what a clobbered instance logs — a cookie *is* present, + written with a key this instance does not hold, so it cannot decrypt it and then + `ClearSession` expires it on its own `--cookie-domain`. Fixed crafty: **0 and 0** across + its whole life. Unfixed radarr: **8 and 8** in 12h. Unfixed jackett: 1 and 1. That is a + real, directional signal and it is the strongest evidence available that the fix works. + 3. **It is still not a demonstrated successful sign-in.** Crafty logged **0 `AuthSuccess`** — + so what is shown is *absence of the failure mode*, not a session observed working + end-to-end. Those three rejections are consistent with benign first-hits (one client, two + of them 5s apart, i.e. a page plus a `/ws` reconnect). I am comfortable saying the fix + works; I am not willing to say it has been watched working, and the distinction is the + whole reason this section exists. + 4. **The collision is a MINORITY of the rejection volume.** radarr: 8 of 155 (5%) carry the + fingerprint; ~95% carry **no cookie at all**. Those are the 5-minute session expiry and + `302`-on-XHR — the first of which this branch now also fixes, the second of which it + deliberately does not. + + Two cautions about the instrument. `reject` ranks **chattiness, not breakage**: radarr's are + `/api/v3/command`, `/api/v3/queue/details` and `/signalr/negotiate` polls, so a busy *arr + inflates the count with nobody inconvenienced. And that same polling is the *amplifier* that + made this intermittent — every poll carrying an undecryptable cookie calls `ClearSession` + again on the zone-wide domain, so **a left-open Radarr tab deleted Jackett's cookie every few + seconds, from another namespace, with no user action at all.** That is why signing in again + never helped for longer than the gap between two polls. It is now a comment in the renderer, + because it is the part of this bug that no one would reconstruct from the flags alone. +- **The four target files are generated, not authored.** Rendering `catalog/` with the canonical + command into a temp dir and diffing against `apps/` reported all 10 onboarded goldens + byte-identical before any change. Hand-editing one is drift by definition. +- **`render-golden` FAILS if jackett is excluded** — provoked (output quoted above), rc **1**. + Restoring the regenerated golden returns it to `all 10 onboarded service golden(s) reproduce`, + rc **0**. +- **Every flag verified against the PINNED v7.8.1 binary**, not against docs, memory, or the + shared proxy (which runs **v7.15.3** and therefore could not vouch for v7.8.1): + `podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help`. + - `--cookie-name string ... (default "_oauth2_proxy")` — **this confirms the constant my whole + check is built on, from the binary itself.** The check's central claim, that an omitted flag + is a *claim on a name* rather than an absence, now rests on the implementation's own help + text rather than on my reading of its source. + - `--cookie-refresh duration refresh the cookie after this duration; 0 to disable` — confirms + the default really is disabled. + - `--metrics-address string` — present in v7.8.1, so the new flag cannot crashloop the pods. + - `--api-route strings` — **singular**, confirming the brief's plural was wrong. + - `--session-cookie-minimal` — exists, and is the flag that conflicts with `--cookie-refresh`. +- **44180 is the platform's existing convention, measured, not chosen.** The shared proxy already + runs `--metrics-address=0.0.0.0:44180` with a `metrics` container port, a `metrics:44180` + Service port, a ServiceMonitor whose endpoint is `metrics`, and **2 ready endpoint addresses** — + i.e. it demonstrably works on this platform. +- **`catalog/render.py --selftest`: PASS**, now including the two-service distinctness assertion, + a `--cookie-refresh` assertion that PARSES the duration and requires `0 < refresh < 300s` (so + `2m30s` -> `150s` -> `2.5m` all stay green but a silent widening past the token lifetime does + not), a `--session-cookie-minimal` mutual-exclusion guard, and both metrics assertions. +- **The check is selected by the whole-repo union.** `conformance/run.sh --layer l0 --requirements + r-nosuchreq-99 --include-scope whole-repo --print-selection` lists + `tests/l0-static/oauth2-proxy-cookie-names-distinct` among 20 tests — i.e. it runs even when the + touched-requirement filter matches nothing, which is the blind spot it was scoped for. +- **The check was tested in BOTH directions**, against throwaway trees outside the repo, never by + mutating the branch: + - **Compliant tree (this branch): rc 0, 39 assertions, 7 instances audited.** + - **Pre-fix tree** (`git archive HEAD` + this test dropped in): **rc 1, 26 failures**, naming all + five violators, the 6-way `_oauth2_proxy` collision, the zone-wide domains, the renderer + emitting no `--cookie-name`, and both synthetic fixtures rendering `None`. + - **Targeted mutations on a compliant copy**, one assertion class each, all fired and nothing + else did: duplicate name (sonarr←radarr) → 1 failure; CSRF derivation + (`_oauth2_proxy_sonarr_csrf` as a session name) → fired; zone-wide `--cookie-domain` on pihole + → 2 failures; crafty's `--cookie-name` deleted → unregistered-default-heir + 2-way collision; + shared proxy renamed → **stale register entry** failure. + - A **false positive was found and fixed** by this testing: `platform/oauth2-proxy/forward-auth-cec.yaml` + contains `serviceName: oauth2-proxy/oauth2-proxy:80` — an Envoy `namespace/service:port` + reference, not an image — which the first inventory scan counted as an unaudited instance. The + scan is now anchored on `image:`, and the near-miss is documented in the check. +- **Gate:** toolbox `./ci/gate.sh` — exit code and stage list in the next section, because it is + not a pass and that belongs there. + +## What I did NOT verify + +- **Local gate: rc 3 (UNDETERMINED), not 0.** Green: `spec-immutability`, `decisions-log`, `lint`, + `l0`, `render-golden`, `l1`, `approval-policy`. **Skipped:** `gitea-settings` (no + `SETTINGS_API_TOKEN` locally) and **L2/L3 conformance** (`CLUSTER_AVAILABLE` unset). Nothing + FAILED; nothing runtime was certified either. **rc 3 is UNDETERMINED, never green.** Touched + subset `R-GITOPS-02 R-IDP-02 R-IDP-06 R-NET-01 R-NET-05 R-NET-06 R-NET-07 R-PLAT-02 R-TEST-01 + R-TEST-02 R-TEST-03 R-TEST-04` — **UNVERIFIED** at L2/L3, all twelve. +- **`spec-immutability` was skipping because of a BUG IN MY TOOLING, not the environment.** The + worktree ENGINE shim in circulation mounts only the main checkout's `.git`, and `dev.sh` + computes the spec path from its own `REPO_ROOT` — which in a worktree is + `.claude/worktrees/agent-*/`, so `../home-network-k3s` does not resolve and it adds no spec + mount at all. The gate then reported the skip, and the skip read as "the one tolerated + environmental skip" while actually being a path bug being waved through. Adding + `-v /var/home/keith/source/home-network-k3s:/home-network-k3s:ro -e SPEC_REPO=/home-network-k3s` + makes **R-TEST-06 actually run and pass**. Worth putting in every worktree agent brief; the + two-mount shim in CLAUDE.md §3 should say so. +- **CI: `gate / gate (pull_request)` reports FAILURE on `75a4029`, for exactly ONE reason, and it + is not this change.** Read from the job log rather than the tick: + + [fail] approval-policy: a sensitive change lacks a non-author approval -> blocks merge + [fail] MERGE GATE: BLOCKED + + Every substantive stage is **green in CI** — `gitea-settings: pass` (CI has the token my local + run does not), `decisions-log`, `lint`, `l0`, `render-golden`, `l1`. Skipped: `spec-immutability` + (the workflow does not check the spec repo out — the documented tolerated skip) and + `conformance` (declared). So CI is a **stricter, more complete** run than my local one on every + axis except the spec repo, and it agrees. + `infra/gitea/approval-policy.json` lists `conformance/**` as sensitive — *"the tests and the + harness; changing them changes what green MEANS"* — so this PR is **fail-closed pending a + non-author approving review**, which is CLAUDE.md §1 working, not a defect to route around. + **The red tick is the policy, and it clears when a human reviews it.** Note also that + `gate / conformance (pull_request)` reports `success` with description **"Has been skipped"** — + the precise trap CLAUDE.md §2b names; it certifies nothing. +- **A SUCCESSFUL SIGN-IN, ANYWHERE.** The strongest post-fix evidence is the *absence* of the + eviction fingerprint on crafty (0 `removing session`, 0 `cookie signature not valid`, across + 13h 52m) against radarr's 8-and-8. That is directional and I believe it. But crafty logged + **0 `AuthSuccess`** in the same window, so no session has been observed working end to end on a + fixed instance. Whoever closes this out should re-measure crafty **after real sign-in traffic**. +- **No browser round-trip by me at all**, for any of the three defects. Reproducing the collision + needs two gated apps visited in one session. The concurrent jackett branch does carry an observed + log-level reproduction (interleaved radarr/jackett `removing session` deleting a clean session + 26s after login) — real evidence for the MECHANISM, and it is theirs, not mine. +- **`--cookie-refresh=2m30s` has never been exercised.** The value is derived from a measured + 300.000s access-token lifetime and is asserted to sit inside `(0, 300s)`, but no session has been + held past the five-minute cliff to watch a refresh succeed. If Keycloak's refresh-token or SSO + idle timeouts are shorter than expected, the symptom moves rather than disappears. **This is the + fix most likely to need a second pass**, and the first thing to watch after merge. +- **The `metrics` port has never been fetched.** The listener is asserted to start by flag + presence only; nobody has curled `:44180` on a rendered pod. And it is unreachable by design + until the Service/policy/ServiceMonitor wiring lands — see above. +- **Nothing was applied.** No `kubectl apply/edit/patch`, no `flux reconcile`. Cluster access was + read-only throughout; the one container I ran was the oauth2-proxy image locally under podman to + read its `--help`. The running pods are still the pre-fix spec. **Merging is what deploys**, and + it will roll five Deployments; every user's existing session on those five apps is invalidated by + the cookie rename and they will re-authenticate once. Radarr/sonarr/qbittorrent use + `strategy: Recreate` on RWO volumes, so there is a brief gap per app. Pick the moment — and + note this now changes **three** flags plus a port on all five, so a bad `--cookie-refresh` value + would surface as five apps misbehaving, not one. +- **The live flags are not re-measured by the check.** git is not the cluster. I measured the live + pods by hand, once; nothing in the gate re-reads them, so hand-applied drift would be invisible + to it. A live-state assertion is L2 and is not written. +- **`--cookie-refresh` and `--metrics-address` are UNGATED, and I found out why.** Nothing in + `ci/` or `.gitea/` runs `catalog/render.py --selftest` — I grepped for it. So the twenty-odd + assertions in that selftest, including the six I just added, are dev-time only: a future edit + that drops either flag *and* re-renders the goldens would pass the entire merge gate. + `render-golden` catches accidental drift (golden vs renderer) but not a deliberate removal of + both together, and my L0 check deliberately does not cover these two flags. **The repo already + has the precedent for the fix and the one-line shape of it:** + `conformance/tests/l0-static/auditability-proofs-producible/check.sh:291` drives + `conformance/lib/ --selftest` as a subprocess for exactly this reason — its own header + says it does so "because nothing [else] drives them". Adding `catalog/render.py` to that list, + or giving the renderer its own scope-declaring L0 test, closes it. I did not do it here because + it widens an already review-blocked PR into a third subject; it is the highest-value follow-up + in this entry. +- **Log-derived counts are a weak instrument and I did not strengthen them.** `kubectl logs + --since=` returns what the kubelet still retains, so an old, busy container's window may be + truncated; and `No valid authentication in request` counts unauthenticated first hits and XHR + polls indiscriminately. The `cookie signature not valid` / `removing session` pair is the only + collision-specific signal in there, and it is the one I leaned on. None of this is gated — + there is no alert on it and no L2 check reads it. +- **The cookie-secrets' distinctness is not gated either.** It was measured out of band and is + asserted nowhere. If they were ever made identical the collision would become harmless — but + sharing one session-encryption key across trust zones is a worse idea than the bug it would mask, + so the check does not offer it as an alternative and does not look for it. +- **The shared proxy's own configuration is unchanged and untested by me.** I did not touch + `platform/oauth2-proxy/**` (owned by another agent this cycle); the check only reads it. +- **Nothing about group authorization, CSRF behaviour, or session contents.** Names and domains only. +- **`vars.CLUSTER_AVAILABLE` is still unset in CI**, so the PR's green tick — if it goes green — + proves the L0/L1 offline subset and nothing about runtime. diff --git a/decisions/2026-09-30-plex-quicksync.md b/decisions/2026-09-30-plex-quicksync.md new file mode 100644 index 0000000..35b9241 --- /dev/null +++ b/decisions/2026-09-30-plex-quicksync.md @@ -0,0 +1,174 @@ +--- +date: 2026-09-30 +branch: feat/plex-quicksync +title: Plex opts into `hardware.gpu: intel-quicksync` — the device mount is GitOps, the Plex-side toggle is a documented manual step, and hardware transcoding has NOT been observed working +--- + +## What changed + +Three things, and the third is the point of the branch. + +1. **`catalog/plex.yaml`** — added the four lines the whole feature reduces to: + + ```yaml + hardware: + gpu: intel-quicksync + ``` + + placed between `env` and `exposure` to match the schema's own property order, with ~48 lines of + comment in the file's established style. The comment records: the field is **intent, not + mechanism** (a later move to the Intel device plugin is a `render.py` change and this block does + not move); that `/dev/dri/renderD128` is present on **all three** prod nodes with `i915` bound; + **why a `supplementalGroups` entry would be inert** (the image's cont-init hook does the group + work inside the container and `s6-setuidgid` replaces the process group list from `/etc/group`); + why `DirectoryOrCreate` is the load-bearing safe failure mode (`apps-wave-0` is `wait: true` with + `apps-wave-1` depending on it, so a failed mount would stall the whole app tier for a feature + that is not load-bearing); and that the Plex-side toggle is a **separate manual operator step**, + pointing at the new runbook. + +2. **`apps/wave-0/plex.yaml`** — re-rendered. Six added lines, nothing else, in the whole `apps/` + tree: + + ``` + - name: dri + mountPath: /dev/dri + - name: dri + hostPath: + path: /dev/dri + type: DirectoryOrCreate + ``` + +3. **`runbooks/plex-quicksync.md`** (new), registered in `runbooks/kustomization.yaml`'s + `configMapGenerator` as `- plex-quicksync=plex-quicksync.md` and indexed in `runbooks/README.md` + under "Operator procedures". **The file alone would have been a 404** — the pod serves a + ConfigMap, not the git tree, and `conformance/tests/l0-static/alert-runbooks-resolvable` exists + precisely because "the file is in git" and "the URL resolves" are different claims. The page + covers: the GitOps/manual boundary as a table; the toggle (Settings → Transcoder → "Use hardware + acceleration when available", preference key `HardwareAcceleratedCodecs`); the real neighbouring + keys and the widely-cited one that **does not exist**; that the only proof is `Transcode (hw)` on + the Plex dashboard **and why no alert is possible**; the device-cgroup failure to expect first + with a concrete open() diagnostic and the Intel-device-plugin fallback; and the floating-`:latest` + caveat with a digest check. + +**Stacked on PR #27 (`feat/sd-hardware-capability`), which is open and unmerged.** This branch was +cut from it rather than from `main`, so its one commit (`d9baa2a`, the schema + renderer + contract) +is in this PR's range. That is deliberate: merging #27 is not an agent's action (CLAUDE.md §1), and +duplicating its changes here would have been worse than showing them. **This must merge after #27.** + +Not done, deliberately: no `--emit-waves` (it writes into `clusters/base/`, a sensitive path that +would force a human reviewer, and it is a no-op unless the *set* of files under `apps/` changes — it +does not here). No `podSecurityContext`. No `nodeSelector`. No alert rule. No `kubectl apply`. + +### Two follow-ups left unclaimed, with the reasoning + +- **Transcode scratch is on the wrong medium, and it is invisible.** The image sets + `TranscoderTempDirectory=/transcode`, which is **not a volume** — it is the container's writable + overlay, i.e. the node's `/data`. That is NVMe on prod-2 but **SATA on prod-1 and prod-3**, so + transcode scratch performance changes silently on reschedule, with nothing reporting it. Both + candidate fixes need a **schema** change and so neither belongs in this PR: pin Plex to prod-2 + (needs a `nodeSelector` field the schema does not have), or give `/transcode` a `medium: Memory` + emptyDir and transcode in RAM (~31 GiB per node; needs an emptyDir field the schema does not + have). **The RAM option is the better one** — it removes the node asymmetry outright instead of + working around it — which is exactly why it deserves its own PR and its own review rather than + being smuggled in behind a four-line catalog change. Documented in the runbook's "Known gaps". +- **GuC/HuC firmware is not loaded** (`i915.enable_guc=0`). The cost is low-power H.264 **encode** + quality only; decode and this feature are unaffected. Changing it needs a kernel parameter plus a + **serialized rolling reboot** — an operator action under CLAUDE.md §5, on a cluster with a + documented history of that going badly (a prod node hung on a warm reboot on 2026-09-03). Kept + well clear of this PR. + +### Rebase note (2026-10-01, Ruth Okonkwo) — resolution only, no redesign + +Rebased onto `main` @ `08deb20` (was cut from `0fbf544`), restacked on the rebased +`feat/sd-hardware-capability` (PR #27) so the stack stays coherent — this branch still carries +exactly PR #27's commit plus its own, with no duplicate of anything already in `main`. The branch +ref was force-updated with `--force-with-lease`, so PR #32 is the same PR with a new head. + +**The `catalog/render.py` conflict disappeared** once PR #27 was rebased; it was never this +branch's own conflict. **One conflict remained, in `runbooks/README.md`, purely textual.** PR #30 +(`fix/jackett-oauth2-proxy-session`, merged as `08deb20`) appended an `oauth2-proxy-session` index +entry under *Operator procedures* at the same point this branch appended its `plex-quicksync` entry. +**Both entries were kept, in full.** Ordering follows this file's existing convention, which is +grouped/narrative rather than alphabetical (the *Nodes* section runs down/io/memory/disk/cpu): +`plex-quicksync` sits next to `plex-claim` as the other Plex page, leaving `oauth2-proxy-session` +adjacent to `crafty-admin-credential`, which cross-references the same sidecar gate. The change to +`runbooks/README.md` is an 8-line pure insertion, so PR #30's entry is untouched. +`runbooks/kustomization.yaml` auto-merged and **both** `configMapGenerator` entries are present. + +Re-verified after the resolution, on the rebased stack: `--selftest` PASS (rc 0); a full re-render +reproduces every committed golden byte-identically; and the **only** `apps/` diff against the new +`main` is `apps/wave-0/plex.yaml`, confined to the expected six lines (the `dri` mount and its +`DirectoryOrCreate` hostPath volume). No other service's golden moved — PR #28 added ~176 lines of +oauth2-proxy sidecar rendering, and this confirms by measurement rather than assumption that Plex +(`auth: native`, no sidecar) is unaffected by it. + +## What I verified + +- **The render is exactly the intended change, and nothing else moved.** After + `python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` (exit 0, + silent): `git diff --stat apps/` = `apps/wave-0/plex.yaml | 6 ++++++`, and + `git status --porcelain apps/` = a single ` M apps/wave-0/plex.yaml`. Read line by line, the diff + is the `dri` volumeMount and the `dri` hostPath volume, and nothing more. **Specifically checked:** + the same invocation also rewrites `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, which + are git-tracked but excluded from the golden comparison by `RENDER_GOLDEN_NOT_ONBOARDED` in + `ci/render-golden.sh` — a spurious diff there would have landed **ungated**. Neither file shows a + diff; whole-repo `git status --porcelain` lists only `apps/wave-0/plex.yaml` and `catalog/plex.yaml` + plus my new files. +- **The rendered output matches what the contract promises** — `schemas/renderer-contract.md` + `### hardware` says exactly two things (a `/dev/dri` hostPath with `type: DirectoryOrCreate` and a + read-write mount on the app container) and exactly two things appeared. No `securityContext`, no + `nodeSelector`, no resources in the diff. +- **The runbook is projected into the server, not merely committed.** `runbooks/kustomization.yaml` + carries `- plex-quicksync=plex-quicksync.md`, whose shape matches the generator-entry regex the + L0 check parses; the extensionless key is what makes `/runbooks/plex-quicksync` a plain static + lookup. Also indexed in `runbooks/README.md`. +- **The runbook clears the L0 quality floor** that `alert-runbooks-resolvable` applies to + alert-linked pages, even though nothing links to this one (it is advisory for an unreferenced + page): all four required sections (`## What this means here`, `## First commands`, + `## Likely causes, most likely first`, `## When to escalate or stop`), well over the 1500-byte + floor, `dev.sh` used throughout, and `--context k3s-prod` on every command. +- **The gate.** See the run recorded in the PR description for the per-stage table and the real rc. +- **No secret was printed.** Every `Preferences.xml` read in the runbook greps one named key, and the + page states explicitly why: the file holds `PlexOnlineToken`. Nothing was `sops -d`'d. + +## What I did NOT verify + +**Hardware transcoding has NOT been observed working.** Nothing in this branch is evidence that it +does. A green gate here proves the manifest renders and the golden matches — it proves nothing about +runtime behaviour, and `vars.CLUSTER_AVAILABLE` is unset so L2/L3 conformance did not run at all. + +- **The device-cgroup question is OPEN, and it is the largest remaining technical risk.** Whether + containerd on this k3s build permits a **non-privileged** pod to `open()` a bind-mounted character + device is **UNVERIFIED**. It can only be settled by deploying and trying. The container inits as + uid 0, which settles *file* permissions, but uid 0 does **not** bypass the cgroup device gate — and + the two failures look identical from the outside. If it is refused, the fallback is the Intel + Device Plugin (`gpu.intel.com/i915`), which is a `catalog/render.py` change plus a re-render with + **no ServiceDefinition edit**; `privileged: true` is not available (Kyverno + `restrict-host-namespaces` is Enforce). The runbook has the diagnostic sequence. +- **The Plex-side toggle was still unset at the time of writing.** `HardwareAcceleratedCodecs` was + **absent** from `Preferences.xml` — the toggle has never been enabled on this server. Until an + operator ticks it in the UI, this branch changes the pod spec and **nothing about transcoding**. + That is the whole reason the runbook exists. +- **No metric, so no alert, so no automated detection — permanently.** `catalog/plex.yaml` declares + no `metricsEndpoint`, so Plex renders no ServiceMonitor; there is no transcode-mode signal for a + rule to evaluate. A silent degradation to software transcoding (a node losing its iGPU, the + `DirectoryOrCreate` path) will **not** be reported by anything. I deliberately did not add an + alert: with no metric behind it, it could never fire, and this repo has been burned at scale by + exactly that. Closing the gap honestly means an exporter first. +- **I did not re-derive this session's node or in-container measurements.** The three-node + `renderD128`/`i915`/GID-992 facts, the cont-init hook behaviour, the `s6-setuidgid` group + replacement, the `PLEX_PREFERENCE*` absence, the six `setPref` keys and the + `HardwareAcceleratedCodecs` string were established earlier in this session and are cited, not + re-measured, by me. Per CLAUDE.md §2b, treat this file as a record of belief and re-measure against + the cluster before relying on any of it. +- **The image is a floating tag.** Everything known about the hook came from digest + `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), while the Deployment runs + `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. A silent pull can invalidate the + "no `supplementalGroups` needed" conclusion **with no commit in this repo**. Not guarded against — + the runbook tells the reader to re-check the hook after an image bump, which is documentation, not + enforcement. +- **I did not verify the live Gitea gate state** (branch protection, required contexts, runner + online) for this PR. Per §2b that must be queried, not read out of `CLAUDE.md`'s table; I did not + query it. +- **`/runbooks/plex-quicksync` was not fetched.** The L0 check asserts every static precondition for + an HTTP 200; it does not make the request, and I did not either. diff --git a/decisions/2026-09-30-sd-hardware-capability.md b/decisions/2026-09-30-sd-hardware-capability.md new file mode 100644 index 0000000..5157934 --- /dev/null +++ b/decisions/2026-09-30-sd-hardware-capability.md @@ -0,0 +1,204 @@ +--- +date: 2026-09-30 +branch: feat/sd-hardware-capability +title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` mounts /dev/dri and nothing else; the enum is what keeps the mechanism swappable +--- + +## What changed + +A new optional top-level ServiceDefinition property, `hardware`, with one optional member, +`gpu`, whose enum today has exactly one value: `intel-quicksync`. **No catalog entry uses it.** +This PR delivers the capability; consuming it (Plex) is separate work on top. + +- `schemas/servicedefinition.schema.json` — adds `hardware` (object, `additionalProperties: false`, + `minProperties: 1`) with `gpu` (enum `["intel-quicksync"]`). Schema version bumped **1.7.0 → + 1.8.0**, changelog entry prepended to the schema `description` per that file's own convention. +- `catalog/render.py` — two module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`) and the + emission in `r_deployment`. +- `schemas/renderer-contract.md` — new `### hardware` field-mapping section, plus cross-field + invariant **8** in that file's numbered list. +- `schemas/README.md` — its `**Schema version:**` line said `1.0.0`; it had been stale since 1.1.0. + Synced to 1.8.0 and annotated that the schema `description` is the changelog of record. Added one + bullet to "Cross-field rules" for the new enum. That was the whole of the staleness; nothing + ballooned. + +**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container only (never a +sidecar), and it is exactly two things: + +1. a hostPath volume `dri` → `/dev/dri`, `type: DirectoryOrCreate`; +2. a read-write `volumeMount` of `dri` at `/dev/dri` on the app container (deliberately not + `readOnly` — VA-API opens the render node read-write). + +**Mounting the device is sufficient.** It emits **no** `privileged: true` (Kyverno +`restrict-host-namespaces` is Enforce on app namespaces and it is not needed), **no** `nodeSelector` +(placement is a separate concern with its own justification), **no** resource requests (there is no +app-container resource render path), and — the one that is worth spelling out — **no +`podSecurityContext.supplementalGroups`**. + +**Why no `supplementalGroups`, when `/dev/dri/renderD128` is `root:render 0660`.** An earlier +revision of this branch merged the host `render` GID into `supplementalGroups`. **It was measured to +be a no-op and has been removed.** In the running container on prod-2, 2026-09-30: + + PID 1 s6-svscan Uid: 0 Gid: 0 Groups: 0 + pid 246 Plex Media Server Uid: 1000 Gid: 1000 Groups: 100 1000 + +`/etc/services.d/plex/run` ends in `exec s6-setuidgid plex …`, and the child's group list is exactly +the `/etc/group` account-database list — root's gid `0`, which PID 1 holds, has **vanished**. That is +a `setgroups()` that *replaces* rather than appends, so anything Kubernetes puts in +`supplementalGroups` reaches `s6-svscan` and is discarded before the app ever starts. What actually +works is the image's own `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner`, which `find`s +every char device under `/dev/dri`, `stat -c '%g'`s its GID, `groupadd`s a matching group inside the +container when none exists, and `usermod -a -G`s the app user into it — every start, nothing +hardcoded; `cont-init.d` completes before `services.d`, so the database is correct at exec time. + +Shipping an inert `supplementalGroups` would have been worse than shipping nothing: it would read as +the mechanism to whoever debugs this next. The reasoning lives in a comment **at the emission site** +and in `renderer-contract.md`, and `render.py --selftest` now asserts the *absence* of a +`securityContext`, so it cannot be "helpfully" added back without a test failing. A pleasant +corollary: because the hook re-`stat`s the device every start, host GID drift is **self-healing**, +and no GID is pinned anywhere in this repo. + +**The design decision worth defending: the field is an enum of intent, not a device path.** The SD +says *what class of host device the workload needs*; the hostPath and the mount exist only in +`catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource +requests, no hostPath at all) is then a renderer change with zero catalog edits. Both the schema +`description` and the contract section say this explicitly, because the obvious "simplification" — +letting an SD name `/dev/dri` — would push a platform implementation detail into the operator-facing +contract and make that migration a catalog-wide edit. + +**`DirectoryOrCreate` is load-bearing.** A node that comes back without the render device (rebuild, +BIOS change, driver regression, hardware swap) must degrade the workload to software transcoding, +not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so a failed +hostPath mount stalls the entire app tier for the sake of a feature that is not load-bearing. +`/dev` is tmpfs, so the empty directory the kubelet creates on such a node leaves nothing durable +behind. Do not "tighten" it to `Directory`. The reasoning is recorded at the emission site. + +### Rebase note (2026-10-01, Ruth Okonkwo) — resolution only, no redesign + +Rebased onto `main` @ `08deb20` (was cut from `0fbf544`). Re-landed as a single cherry-pick onto +current `main`; the branch ref was force-updated with `--force-with-lease`, so PR #27 is the same +PR with a new head. + +**One conflict, in `catalog/render.py`, purely textual.** PR #28 +(`fix/oauth2-proxy-cookie-collision-fleet`, merged as `6caa5f2`) inserted its own constants blocks +(`OAUTH2_ACCESS_TOKEN_LIFETIME_S` / `OAUTH2_COOKIE_REFRESH`, and `OAUTH2_PROXY_METRICS_PORT`) at the +same insertion point immediately after `KEYCLOAK_ISSUER` that this branch used for +`DRI_VOLUME_NAME` / `DRI_DEVICE_DIR`. The two blocks are unrelated and do not interact, so **both +sides were kept in full**, oauth2-proxy's first and the hardware block after it. Nothing was +dropped and nothing was rewritten: the rebased commit adds the **same 126 `render.py` lines** as the +pre-rebase commit (verified by comparing the added-line sets) and deletes none. In particular the +`supplementalGroups`-is-deliberately-not-emitted and `DirectoryOrCreate`-is-load-bearing comments +survive verbatim. + +Re-verified after the resolution: `--selftest` PASS (rc 0), and a full re-render still produces a +**zero** diff in `apps/` — this PR remains inert, as it must be while no ServiceDefinition opts in. + +## What I verified + +Measured, in this worktree, on this branch: + +- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** New cases in the + existing `check(...)` style: the positive (volume + mount, and *no* `securityContext`, *no* + privileged, *no* nodeSelector), the negative that matters (an SD without `hardware` gains no + volume, no volumeMount and no `securityContext` — mirroring the existing `podSecurityContext` + negative), and a pass-through case (an SD declaring both `hardware` and its own + `podSecurityContext: {fsGroup: 1001, supplementalGroups: [44]}` keeps it **exactly** as written). +- **The self-tests actually bite.** I mutated the guard (`if hw_gpu == "intel-quicksync":` → `if + True:`) in a scratch copy and re-ran: **rc 1**, failing on "an SD without `hardware` must gain no + dri volume, no mount and no securityContext". A test that cannot fail is not a test; this one can. +- **Re-render is a zero diff.** + `python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` then + `git diff --stat apps/` → **empty**, and `git status --porcelain apps/` → **empty** (so the two + not-onboarded by-products, `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, were + rewritten byte-identically and needed no revert). The change is inert until a service opts in. +- **Schema validity and catalog conformance.** `Draft202012Validator.check_schema` passes; all + **12** `catalog/*.yaml` validate with **0** errors. On the new field specifically: `gpu: + intel-quicksync` accepted; `gpu: /dev/dri` rejected (enum); `hardware: {gpu: …, path: …}` rejected + (`additionalProperties: false`); `hardware: {}` rejected (`minProperties: 1`). +- **Hand-checked the emitted YAML** by rendering a throwaway SD carrying `hardware`, a Longhorn + volume and its own `podSecurityContext: {runAsUser: 1000, supplementalGroups: [44]}`. Result: + `volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `volumes` = the PVC then + `{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`; `securityContext` = + `{runAsUser: 1000, supplementalGroups: [44]}` — i.e. **unchanged**, nothing appended. That toy SD + was never written into `catalog/`. +- **GATE: `rc 3` — UNDETERMINED, NOT green.** Run in the pinned toolbox from this worktree with + both CLAUDE.md §3 workarounds (`DEV_KUBECONFIG=/dev/null`, and an `ENGINE` shim bind-mounting the + main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f`; touched + requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Run **four** times on this branch with an identical + result each time, most recently against tree `c132893045b1` — the final commit differs from that + run only by this paragraph. Per stage: + + | stage | result | + |---|---| + | spec-immutability | **SKIPPED** — sibling spec repo not at `/work/../home-network-k3s`. Expected from a worktree; tolerated by §3. | + | decisions-log | pass | + | lint | pass | + | l0 | pass — 19 offline cases, `0 failure(s), 0 skip(s), 0 unwired` | + | render-golden | pass — *"all 10 onboarded service golden(s) reproduce from the canonical render"* | + | l1 | pass — 3 offline checks, `0 failure(s), 0 skip(s), 0 unwired`; coverage 51/51 requirements scaffolded | + | approval-policy | pass (`event='none'`, not a pull_request event — N/A locally; it will evaluate for real on the PR) | + | gitea-settings | **SKIPPED** — `SETTINGS_API_TOKEN` unset. Expected; branch protection and runner registration are therefore UNVERIFIED by this run. | + | conformance (L2/L3) | **SKIPPED** — `CLUSTER_AVAILABLE!=true`. Phase-0 standing skip. | + + The gate's own verdict: *"Stages that did not run: spec-immutability gitea-settings"*. **No stage + failed, and no stage skipped for a reason caused by this change.** rc 3 is reported as rc 3; I am + not calling this green. +- **Blast radius of the schema change:** `hardware` is absent from every catalog entry, so + `apply_defaults` never materializes it and `s.get("hardware")` is `None` on all 12 — which is what + the zero diff above demonstrates rather than asserts. +- **Touched no sensitive path.** Changes are confined to `schemas/`, `catalog/`, `decisions/`; per + `infra/gitea/approval-policy.json` none of those are sensitive. `conformance/**` and `clusters/**` + are untouched. I did not run `--emit-waves` (it writes `clusters/base/`, and the set of files + under `apps/` did not change). + +## What I did NOT verify + +- **Whether containerd's device cgroup permits a non-privileged pod to `open()` a bind-mounted char + device. Still unverified, and it is the single largest technical risk in this workstream.** + containerd installs a per-container **device cgroup** allow-list, and whether + `/dev/dri/renderD128` arriving as a bind-mounted device node is permitted by it on this + k3s/containerd version is **unmeasured**. Settling it requires applying a pod, which is an + operator action, not an agent one. The container's PID 1 running as uid 0 helps on the DAC side + but does **not** bypass the cgroup gate. If it blocks, the mount will succeed, the image's group + hook will do its job, and the `open()` will still fail — and the fallback is the Intel GPU device + plugin, which *sets* the cgroup entry. Note that that fallback is a **renderer-only change**: no + ServiceDefinition moves, because the authoring surface is an intent enum. That is the design + justifying itself, and it is the reason to keep the enum even if the hostPath mechanism loses. +- **That a missing device degrades silently — and nothing will tell you.** This is the accepted cost + of `DirectoryOrCreate`. The workload will software-transcode and **no signal reports it**: media + apps in this catalog declare no `metricsEndpoint`, so no ServiceMonitor exists, and there is no + metric to alert on. I deliberately did **not** propose an alert, because an alert that cannot fire + is worse than none — see the 2026-09-13 correction in `renderer-contract.md` §`observability`, + where twelve monitors discovered zero targets. The only proof of hardware transcode is an operator + seeing it in the application's own dashboard. +- **That "mounting the device is sufficient" generalizes beyond this one image.** It does not, and + I have not verified that it does. The claim was measured for an image that resolves the device + group itself in `cont-init.d`. A future consumer of `hardware.gpu` that runs non-root with no + equivalent hook would still need the group solved — and the fix for that case is a **renderer** + change (per-image handling, or the device plugin), never a new SD field. I scoped the claim in + `renderer-contract.md` rather than letting it read as general; the first such consumer must + re-measure. +- **The feature now depends on a hook inside an image that floats.** The `cont-init.d` behaviour + above was read from digest `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), but the + deployment runs `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. **A silent pull can + change the behaviour this feature depends on, with no commit anywhere in this repo to show for + it** — and the failure would be the silent software-transcode described above. I have not fixed + that here (it is the Plex wiring's tag/digest decision, not the capability's), but it should not + be forgotten. +- **GID drift is NOT on this list any more, deliberately.** An earlier revision pinned + `INTEL_RENDER_GID = 992` and listed "a node rebuild could move it" as an open risk. The hook + re-`stat`s the device on every start, so drift is self-healing and no GID is pinned anywhere. The + risk is retired, not overlooked. +- **Anything on the live cluster.** I ran no `kubectl` and I opened no shell in any container. The + node facts (device present, `root:render` 0660, `i915` bound, no device plugin, no + `gpu.intel.com/*` advertised), the in-container measurements quoted above, and the Kyverno + posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups` + restriction; no PSA labels on app namespaces) are **prior recon handed to me, not re-measured by + me**. Per CLAUDE.md §2b, treat them as second-hand until re-queried. +- **That CI exercises the self-tests. It does not.** `git grep -- --selftest ci/` returns nothing — + no gate stage invokes the renderer's self-test. They are the executable spec and house style, and + I ran them by hand, but a future regression in this code path **will not be caught by the gate**. + Wiring `--selftest` into a gate stage is a real gap and is not part of this PR (`ci/` is a + sensitive path and would drag in a human reviewer for a change unrelated to this one). +- **Runtime behaviour of any kind.** The gate proves the L0/L1 offline subset only; + `vars.CLUSTER_AVAILABLE` is unset, so L2/L3 conformance does not run in CI. diff --git a/infra/ansible/roles/node_health/README.md b/infra/ansible/roles/node_health/README.md index 95f7dd6..88fcb97 100644 --- a/infra/ansible/roles/node_health/README.md +++ b/infra/ansible/roles/node_health/README.md @@ -19,9 +19,9 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0 ## Contract -- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu`, - and — only when it can escalate — `smartctl`. It writes nothing to the node and - **installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is +- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu` + (and `lspci` only if it happens to be installed), and — only when it can escalate — + `smartctl`. It writes nothing to the node and **installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is absent the SMART checks SKIP with that as the stated reason (operator ruling 2026-08-31: package installation is `node_baseline`'s job, never a health check's, never a manual `apt-get`). @@ -50,6 +50,9 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0 | `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite **and whether it survives a reboot** (see [Swap off is not the same as swap disabled](#swap-off-is-not-the-same-as-swap-disabled)), EDAC correctable/uncorrectable counters | | `memory.physical_total` | **is this machine using the RAM it has?** `/proc/zoneinfo` present-vs-managed pages (no root, metal or VM) and `MemTotal` vs the DIMM capacity SMBIOS reports (`dmidecode -t 17`, root, bare metal). Cause-independent by design — see [The EFI memattr check](#the-efi-memattr-check) | | `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) | +| `graphics.render_device` | is there a usable `/dev/dri/renderD*` character device — the path Plex hardware-transcodes through. See [The render-device check](#the-render-device-check) | +| `graphics.drm_driver` | which DRM driver is bound to the display device, against `node_health_graphics_expected_driver` (`i915` here) | +| `graphics.media_stack` | **INFO only**: `vcs*`/`vecs*` media engines, GuC/HuC firmware state, host VAAPI userspace. None of these is ever graded — see below for why | | `thermal.max` | hottest sensor across thermal zones and hwmon chips | | `pressure.cpu` / `.memory` / `.io` | PSI `some avg60` — dev learned to care about CPU PSI specifically | | `network.link`, `network.errors`, `network.resolvers` | link state/speed, NIC error counters, DNS upstream redundancy per address family | @@ -91,6 +94,78 @@ used to blame that state on "this node has not been baselined yet", which sent t back to re-run the role that had already run. +## The render-device check + +Plex transcodes on the iGPU through `/dev/dri/renderD128`. **When that node stops existing, +nothing fails loudly.** Plex falls back to software transcoding by itself; the only symptom +is a hot CPU and stuttering clients. A node rebuild, a BIOS change that disables the onboard +IGD, a kernel that binds a different DRM driver, or a driver regression all produce exactly +that silent degradation. + +The state was measured by hand on all three prod nodes on 2026-09-30 and found working. **A +fact measured once decays silently**, so it is re-measured on every health run instead of +living in a runbook. Nothing is installed, no module is loaded, no group is changed — the +role's read-only contract is unchanged. + +| fact | where it comes from | root? | +|---|---|---| +| `/dev/dri` contents, ownership, mode, char-vs-not | `lstat()` on each entry | no | +| owning group **name** for the render node's gid | `grp.getgrgid()` | no | +| bound DRM driver | `/sys/class/drm/card*/device/driver` symlink target | no | +| is the iGPU on the bus at all | `/sys/bus/pci/devices/*/class` (PCI base class `0x03`) | no | +| media engines (`vcs*`, `vecs*`) | `/sys/class/drm/card*/engine/*` | no | +| module load state | `/proc/modules` | no | +| GuC/HuC | `/sys/module/i915/parameters/enable_guc` + `/proc/cmdline` | **parameter file is mode 0400** | + +### The PCI scan is separate from the DRM scan on purpose + +`/sys/class/drm/card*` exists only once a driver has **bound**. A machine whose iGPU sits on +the bus with nothing attached has no `card0` at all — and that is precisely the failure being +hunted. So the display device is looked for on the PCI bus independently, from sysfs, which +is always there. That is what lets `graphics.render_device` distinguish + +- *no GPU in this machine* → **SKIP** (a VM guest; nothing to grade, the same convention + `thermal.max` uses for "no sensor on this hardware"), from +- *a GPU is on the bus and there is still no render node* → **FAIL**. + +`lspci` is used **only** to decorate the PCI entry with a readable name, and it is treated as +optional because it is: `pciutils` is **not** declared in `node_baseline_packages`, so nothing +in this repo guarantees it. It did turn out to be present on all three prod nodes +(`/usr/bin/lspci`, measured 2026-09-30 — Debian installs `pciutils` by default), but depending +on that would mean depending on an undeclared package. Absent, `description` is `null`, the +numeric IDs are printed instead, and no verdict changes. + +### The render node's GID is reported, never asserted + +The render device was `root:render`, **gid 992**, mode `0660` on all three prod nodes on +2026-09-30. The check does **not** compare against `992`, and must not be changed to. +`render` is a **systemd dynamic allocation** (`systemd-sysusers`), not a Debian static gid +from `base-passwd`, so a rebuilt node can legitimately land on a different number. Plex's own +container entrypoint stats the device and joins whichever group owns it, so drift genuinely +is not a fault. Recording the observed group name and gid in the detail line makes drift +**visible** without making it an error — a hard `!= 992` would be a false alarm waiting for a +reinstall. + +What *is* configurable is the expected DRM driver, `node_health_graphics_expected_driver` +(default `i915`), so the verdict stays out of task logic. `xe` bound instead of `i915` is a +**WARN**, not a FAIL: the device still works, but the two drivers do not share a userspace +media stack and a libva/iHD build that transcodes correctly against one is not automatically +right for the other. On these nodes `xe.ko.xz` exists on disk and is bound to nothing. + +### Two things that are informational and must stay that way + +- **GuC/HuC firmware not loaded** (`i915.enable_guc=0`) is a known, accepted state here. It + costs low-power H.264 encode *quality* only; decode and encode both still run on the media + engines. Grading it would put a permanent WARN on a chosen configuration. +- **No host VAAPI userspace / no `vainfo`** is **not a defect**. Plex ships its own libva and + media driver inside its container, so the host needs neither. Flagging it would invite + someone to `apt install vainfo` on a node to clear a health check — and this role installs + nothing. + +`enable_guc` is one of i915's `0400` module parameters, so an unprivileged pass usually +cannot read it. It reports as an honest `NOT READ — ` rather than a fabricated `0`, +with `/proc/cmdline` as the world-readable second source for an explicit boot override. + ## The EFI memattr check `infra/PROD-PROVISIONING.md` §0b is authoritative; this is only what the check needs. @@ -273,3 +348,10 @@ stays `null` — no claim is made either way. but the wear/endurance context SKIPs rather than guessing. - `ioerr_cnt` counts commands the kernel saw fail, which includes benign rejected or unsupported commands during discovery. Attribution needs `dmesg`, which needs root. +- `graphics.render_device`'s **FAIL path has never fired against a real node** — every prod + node has a working render device, and the role is read-only so one cannot be taken away to + prove it. The failure branches were exercised against synthetic fact sets only. The same + applies to the `xe`-bound WARN: no node here has `xe` bound. +- `graphics.*` proves the render **device** exists and which driver owns it. It does **not** + prove a transcode succeeds — that needs a VAAPI round trip inside Plex's own container, + with Plex's own libva, which is out of scope for a node-level check that installs nothing. diff --git a/infra/ansible/roles/node_health/defaults/main.yml b/infra/ansible/roles/node_health/defaults/main.yml index eae98c9..9b02dbc 100644 --- a/infra/ansible/roles/node_health/defaults/main.yml +++ b/infra/ansible/roles/node_health/defaults/main.yml @@ -35,6 +35,24 @@ node_health_try_privileged: true # --nocheck=standby: never spin up a sleeping disk just to read it. Read-only hygiene. node_health_smartctl_args: "--json=c --all --nocheck=standby" +# --- Graphics / render device --------------------------------------------------------- +# The DRM driver this site EXPECTS to be bound to its display device. Measured on all three +# prod nodes 2026-09-30: `i915` on the CoffeeLake-S GT2 [UHD Graphics 630] iGPU of every HP +# EliteDesk 800 G5 Mini. The newer `xe` module is present on disk as xe.ko.xz but is bound to +# nothing, and that difference matters enough to surface: the two drivers do not share a +# userspace media stack (a libva/iHD build that works against i915 is not automatically the +# right one for xe), so a silent switch would change Plex's transcoding path. A site on AMD +# or NVIDIA sets this to `amdgpu` / `nouveau` / `nvidia` rather than editing task logic. +# +# WHAT IS DELIBERATELY *NOT* A SETTING HERE: the numeric GID of the render node's owning +# group. `render` is a systemd DYNAMIC allocation (systemd-sysusers), not a Debian static +# one, so a rebuilt node may legitimately land on a different number — it was 992 on all +# three prod nodes on 2026-09-30, and an assertion against that literal would be a false +# alarm waiting to happen. Plex's own container entrypoint stats the device and joins +# whichever group owns it, so drift genuinely is not a fault. The observed group name and GID +# are therefore REPORTED in the check detail and never graded. +node_health_graphics_expected_driver: i915 + # --- Thresholds --------------------------------------------------------------------- # All verdicts come from here, so a site can retune without touching task logic. node_health_thresholds: diff --git a/infra/ansible/roles/node_health/files/node_health_collect.py b/infra/ansible/roles/node_health/files/node_health_collect.py index 8eb79e7..d6d6e1a 100644 --- a/infra/ansible/roles/node_health/files/node_health_collect.py +++ b/infra/ansible/roles/node_health/files/node_health_collect.py @@ -35,10 +35,13 @@ # ============================================================================= import glob +import grp import json import os +import pwd import re import shutil +import stat import subprocess import sys import time @@ -756,6 +759,212 @@ def collect_pressure(): return out +# --- Graphics / DRM render device ----------------------------------------------------- +# WHY THIS IS HERE AT ALL. Plex transcodes on the iGPU through /dev/dri/renderD128. If that +# node stops existing — a node rebuild, a BIOS change that disables the IGD, a kernel that +# binds a different DRM driver — nothing breaks loudly: Plex silently falls back to software +# transcoding and the only symptom is a hot CPU. So the render device is measured on every +# health run rather than being a fact someone established once by hand. +# +# EVERYTHING BELOW IS UNPRIVILEGED. /dev/dri is world-traversable, the device nodes are +# stat()able by anyone, /sys/class/drm and /sys/bus/pci/devices are world-readable, and so +# are the module parameters under /sys/module. Nothing here needs root, so nothing here +# belongs in the privileged pass. (GuC/HuC *load* status lives in debugfs, which does need +# root AND debugfs mounted; the module PARAMETER below is the unprivileged stand-in and is +# what actually decides whether the firmware is loaded at all.) +# +# THE PCI SCAN IS DELIBERATELY SEPARATE FROM THE DRM SCAN. /sys/class/drm/card* only exists +# once a driver has BOUND. A machine whose iGPU is on the bus with no driver bound has no +# card0 at all, and that is precisely the failure this is meant to catch — so the display +# device is looked for on the PCI bus independently, from sysfs, which is always there. +# `lspci` is used ONLY to decorate that with a human-readable name, and it is treated as +# optional because it IS optional: pciutils is not declared in node_baseline_packages, so +# nothing in this repo guarantees it. It did turn out to be present on all three prod nodes +# (/usr/bin/lspci, measured 2026-09-30 — Debian installs pciutils by default), but relying on +# that would make the collector depend on an undeclared package. A missing lspci leaves +# `description` null, the numeric IDs are reported instead, and no verdict changes. +DISPLAY_PCI_CLASS_PREFIX = "0x03" # PCI base class 03 = display controller (VGA/3D/other) +# DRM drivers worth reporting the load state of. The bound-driver symlink is authoritative +# (a built-in driver never appears in /proc/modules); this is corroboration only, and it is +# how "xe.ko.xz exists on disk but is bound to nothing" becomes a visible fact. +DRM_MODULES = ("i915", "xe", "amdgpu", "radeon", "nouveau", "nvidia") + + +def _uid_name(uid): + try: + return pwd.getpwuid(uid).pw_name + except (KeyError, OverflowError, TypeError, OSError): + return None + + +def _gid_name(gid): + """Group NAME for a numeric gid, or None when nothing resolves it. + + Reported as an OBSERVED value, never asserted against a literal. The `render` group is + a systemd DYNAMIC allocation (it is created by systemd-sysusers, not fixed by Debian's + static base-passwd list), so its gid can legitimately differ after a rebuild. Plex's own + container init stats the device and joins whatever group owns it, so a moved gid is + drift to notice, not a fault to fail on.""" + try: + return grp.getgrgid(gid).gr_name + except (KeyError, OverflowError, TypeError, OSError): + return None + + +def _module_param(mod, param): + """One /sys/module//parameters/, with WHY it is missing when it is. + + Three outcomes that must not be conflated: a real value; "the module exposes no such + parameter" (not loaded, built in, or a driver that simply has no such knob — xe has no + enable_guc, GuC is mandatory there); and "the file exists but is mode 0400". Most i915 + module parameters are `i915_param_named(..., 0400, ...)` — root-only — so an + unprivileged pass legitimately cannot read them. Returning null with the reason keeps + that an honest UNKNOWN instead of a fabricated 0. read_text() also records the + permission denial in UNREADABLE, so the summary lists it with everything else root + would add.""" + path = "/sys/module/%s/parameters/%s" % (mod, param) + if not os.path.exists(path): + return {"value": None, "reason": "%s exposes no %s parameter (module not loaded, " + "built in, or no such knob on this driver)" % (mod, param)} + value = read_text(path) + if value is None: + return {"value": None, "reason": "%s is not readable without root (i915 module " + "parameters are mode 0400)" % path} + return {"value": value, "reason": None} + + +def _dev_node(path): + """stat() one /dev/dri entry. lstat, so a symlink is reported as the symlink it is.""" + try: + st = os.lstat(path) + except OSError as exc: + UNREADABLE.append({"path": path, "reason": "stat failed: %s" % exc}) + return None + return { + "path": path, + "name": os.path.basename(path), + "char_device": stat.S_ISCHR(st.st_mode), + "mode": "%04o" % stat.S_IMODE(st.st_mode), + "uid": st.st_uid, + "gid": st.st_gid, + "owner": _uid_name(st.st_uid), + "group": _gid_name(st.st_gid), + } + + +def collect_graphics(): + dri_dir = "/dev/dri" + dri_present = os.path.isdir(dri_dir) + devices = [] + if dri_present: + for path in sorted(glob.glob(dri_dir + "/*")): + # /dev/dri/by-path/ is a directory of symlinks, not a device. Skipping + # directories rather than filtering to char devices is deliberate: anything + # in here that is NOT a character special file is an anomaly worth seeing, and + # `char_device: false` is how it becomes visible. + if os.path.isdir(path): + continue + entry = _dev_node(path) + if entry is not None: + devices.append(entry) + render_nodes = [d for d in devices if d["name"].startswith("renderD")] + + # Bound DRM devices. /sys/class/drm also holds one entry per CONNECTOR + # (card0-HDMI-A-1, card0-DP-1 ...) — those are outputs, not devices, and are skipped. + cards = [] + for cdir in sorted(glob.glob("/sys/class/drm/card[0-9]*")): + name = os.path.basename(cdir) + if "-" in name: + continue + driver = None + drvlink = cdir + "/device/driver" + if os.path.islink(drvlink): + driver = os.path.basename(os.path.realpath(drvlink)) + address = None + devlink = cdir + "/device" + if os.path.islink(devlink): + address = os.path.basename(os.path.realpath(devlink)) + # i915/xe publish one directory per hardware engine. vcs* is a video (H.264/HEVC) + # engine and vecs* the video-enhancement engine: those two are what a transcode + # actually runs on, so their presence is the difference between "a GPU is bound" + # and "this GPU can do media work". + engines = sorted( + { + os.path.basename(p) + for p in (glob.glob(cdir + "/engine/*") + glob.glob(cdir + "/device/engine/*")) + if os.path.isdir(p) + } + ) + cards.append( + { + "card": name, + "driver": driver, + "pci_address": address, + "vendor_id": read_text(cdir + "/device/vendor"), + "device_id": read_text(cdir + "/device/device"), + "engines": engines, + "media_engines": [e for e in engines if e.startswith(("vcs", "vecs"))], + } + ) + + # Display-class devices on the PCI bus, driver bound or not. + pci_display = [] + for pdir in sorted(glob.glob("/sys/bus/pci/devices/*")): + class_id = read_text(pdir + "/class") or "" + if not class_id.startswith(DISPLAY_PCI_CLASS_PREFIX): + continue + address = os.path.basename(pdir) + driver = None + if os.path.islink(pdir + "/driver"): + driver = os.path.basename(os.path.realpath(pdir + "/driver")) + description = None + rc, out, _ = run(["lspci", "-nn", "-s", address]) + if rc == 0 and out.strip(): + description = out.strip().splitlines()[0] + pci_display.append( + { + "address": address, + "class_id": class_id or None, + "vendor_id": read_text(pdir + "/vendor"), + "device_id": read_text(pdir + "/device"), + "revision": read_text(pdir + "/revision"), + "driver": driver, + "description": description, + } + ) + + loaded = set() + for line in (read_text("/proc/modules") or "").splitlines(): + fields = line.split() + if fields: + loaded.add(fields[0]) + modules = {m: (m in loaded) for m in DRM_MODULES} + + # GuC/HuC. i915's `enable_guc` is a bitmask: 0 = neither, 1 = GuC submission, 2 = HuC + # firmware, 3 = both, -1 = whatever the driver defaults to on this platform. It is the + # one place the state is visible without root AND without debugfs — but the sysfs file + # is mode 0400, so unprivileged it usually reads back as an honest UNKNOWN. The kernel + # command line is the world-readable second source: an explicit `i915.enable_guc=` there + # is authoritative about what was ASKED for, even when the parameter file cannot be read. + uc = {mod: {"enable_guc": _module_param(mod, "enable_guc")} for mod in ("i915", "xe")} + cmdline_params = [ + token + for token in (read_text("/proc/cmdline") or "").split() + if token.startswith(tuple("%s." % m for m in DRM_MODULES)) + ] + + return { + "dri_present": dri_present, + "devices": devices, + "render_nodes": render_nodes, + "drm_cards": cards, + "pci_display": pci_display, + "modules": modules, + "uc": uc, + "cmdline_params": cmdline_params, + } + + # --- Storage ------------------------------------------------------------------------ PSEUDO_FS = { "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "cgroup2", "pstore", @@ -1095,7 +1304,15 @@ def collect_os(): # we only DETECT, and the role skips the SMART checks with an accurate reason. def collect_tools(): tools = {} - for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu"): + # lspci (pciutils) and vainfo (libva-utils) are BOTH optional and NEITHER is declared in + # node_baseline_packages, so nothing in this repo guarantees either. Measured on prod + # 2026-09-30: lspci IS there (/usr/bin/lspci — Debian ships pciutils by default) and + # vainfo is NOT. lspci only decorates the PCI display entry with a readable name; vainfo + # is not needed on the host at all, because Plex ships its own libva and media driver + # inside its container. They are probed so the report can state their presence or absence + # as a fact instead of leaving the reader to wonder — and so nothing here is ever tempted + # to install them. This role installs nothing. + for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu", "lspci", "vainfo"): tools[tool] = which(tool) return tools @@ -1127,6 +1344,7 @@ def main(): "os": collect_os(), "dmi": collect_dmi(), "cpu": collect_cpu(), + "graphics": collect_graphics(), "memory": collect_memory(), "thermal": collect_thermal(), "pressure": collect_pressure(), diff --git a/infra/ansible/roles/node_health/tasks/assess.yml b/infra/ansible/roles/node_health/tasks/assess.yml index 1e34098..20dcdb5 100644 --- a/infra/ansible/roles/node_health/tasks/assess.yml +++ b/infra/ansible/roles/node_health/tasks/assess.yml @@ -652,6 +652,204 @@ ~ (', lscpu Virtualization=' ~ c.virtualization if c.virtualization else '') ~ (', running as a guest of ' ~ c.hypervisor_vendor if c.is_virtual_guest | bool else '') }} +# --- Graphics / render device ---------------------------------------------------------- +# WHY A HEALTH CHECK AND NOT A NOTE IN A RUNBOOK. Plex transcodes on the iGPU through +# /dev/dri/renderD128. When that node stops existing nothing fails loudly — Plex falls back +# to software transcoding on its own and the only symptom is a hot CPU and stuttering +# clients. A node rebuild, a BIOS setting that disables the onboard IGD, or a kernel that +# binds a different DRM driver all produce exactly that silent degradation. It was measured +# by hand on all three prod nodes on 2026-09-30 and found working; a fact measured once +# decays silently, so it is re-measured on every health run instead. +# +# READ-ONLY, AND UNPRIVILEGED. Every fact behind these three checks comes from the +# unprivileged pass: stat() on /dev/dri, the /sys/class/drm bound-driver symlink, +# /sys/bus/pci/devices class codes, /proc/modules and /proc/cmdline. The role installs +# nothing (no vainfo, no pciutils), loads no module and changes no group — see the role +# README. The ONE fact that is root-gated is i915's enable_guc module parameter (mode 0400), +# and it reports as an honest UNKNOWN with the reason rather than as a 0. +# +# THE THREE VERDICTS ARE SPLIT ON PURPOSE: +# graphics.render_device the OUTCOME — can anything hardware-transcode on this node. +# graphics.drm_driver the CAUSE side — what is bound, and is it what this site expects. +# graphics.media_stack INFO only. GuC/HuC and host VAAPI userspace both belong here and +# NEITHER is ever a failure; see the task's own comment for why. +- name: Assess — locate the display device and its bound driver (once, shared by three checks) + ansible.builtin.set_fact: + node_health_render: + # Display-class PCI devices (base class 0x03), driver bound or not. This is the + # question "is there a GPU in this machine at all", and it is answered from the PCI + # bus rather than from /sys/class/drm because a card* entry only exists once a driver + # has BOUND — so a GPU with nothing bound to it has no card* and would otherwise look + # identical to a machine with no GPU. + display_devices: "{{ g.pci_display }}" + bound_drivers: >- + {{ g.pci_display | map(attribute='driver') | select('string') | unique | list }} + # Only a CHARACTER SPECIAL file is a usable render node. Anything else called renderD* + # is an anomaly, not a device, and must not be counted as one. + render_chardevs: "{{ g.render_nodes | selectattr('char_device') | list }}" + # Video (vcs*) and video-enhancement (vecs*) engines across every bound card — the + # difference between "a GPU is bound" and "this GPU can do media work". + media_engines: >- + {{ g.drm_cards | map(attribute='media_engines') | flatten | unique | sort | list }} + vars: + ansible_become: false + g: "{{ node_health_facts.graphics }}" + +- name: Assess — DRM render node for hardware transcoding + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'graphics.render_device', + 'title': 'DRM render node present for hardware transcoding (/dev/dri/renderD*)', + 'status': render_status, + 'detail': render_detail }] }}" + vars: + ansible_become: false + g: "{{ node_health_facts.graphics }}" + r: "{{ node_health_render }}" + # SKIP, not PASS and not FAIL, when the machine has no display device at all: there is + # then nothing to grade, and the same convention thermal.max already uses for "no sensor + # on this hardware" applies. FAIL is reserved for the state that actually breaks Plex — + # a GPU IS on the bus and there is still no render node. + render_status: >- + {{ 'SKIP' if (r.display_devices | length) == 0 + else ('PASS' if (r.render_chardevs | length) > 0 else 'FAIL') }} + dri_listing: >- + {{ (g.devices | map(attribute='name') | join(', ')) if (g.devices | length) > 0 + else 'nothing' }} + render_detail: >- + {%- if (r.display_devices | length) == 0 -%} + this machine exposes no display-class PCI device (base class 0x03) at all, so there is + no GPU here for a render node to belong to and hardware transcoding is not available on + this node by construction. Normal for a VM guest. Measured from /sys/bus/pci/devices, + not assumed. + {%- elif (r.render_chardevs | length) > 0 -%} + {{ r.render_chardevs | length }} render node(s): + {% for d in r.render_chardevs %}{{ d.path }} (character special, mode {{ d.mode }}, + owner {{ d.owner | default(d.uid, true) }}, group {{ d.group | default('unresolved', true) }} + gid {{ d.gid }}){{ ', ' if not loop.last else '' }}{% endfor %}. + /dev/dri holds: {{ dri_listing }}. + THE OWNING GROUP AND GID ARE REPORTED AS OBSERVED, NEVER ASSERTED against a literal: + `render` is a systemd DYNAMIC allocation rather than a Debian static one, so a rebuilt + node may legitimately land on a different number (all three prod nodes read gid 992 on + 2026-09-30), and Plex's container entrypoint stats the device and joins whichever group + owns it. Drift is therefore visible in this line without being a failure. + {%- else -%} + HARDWARE TRANSCODING IS BROKEN ON THIS NODE. A display-class device IS present on the + PCI bus ({{ r.display_devices | map(attribute='address') | join(', ') }}) but there is + no usable character-special render node: /dev/dri + {{ 'does not exist' if not (g.dri_present | bool) else 'holds ' ~ dri_listing }}. + {{ ('Note that /dev/dri holds ' ~ (g.render_nodes | map(attribute='name') | join(', ')) + ~ ' — present, but NOT character special, which is an anomaly in its own right. ') + if (g.render_nodes | length) > 0 else '' }} + Nothing fails loudly in this state — Plex silently falls back to software transcoding + and the only symptom is CPU load. Check graphics.drm_driver below for whether a DRM + driver bound at all, then whether the onboard IGD is still enabled in BIOS. + {%- endif -%} + +- name: Assess — DRM driver bound to the display device + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'graphics.drm_driver', + 'title': 'DRM driver bound to the display device (expected ' ~ expected ~ ')', + 'status': driver_status, + 'detail': driver_detail }] }}" + vars: + ansible_become: false + g: "{{ node_health_facts.graphics }}" + r: "{{ node_health_render }}" + expected: "{{ node_health_graphics_expected_driver }}" + # WARN rather than FAIL when a DIFFERENT driver is bound (the `xe` case): the device + # still works, but the userspace media stack is not the same one, so it is a change the + # operator must see before a transcode regression surfaces as a user complaint. FAIL is + # for nothing bound at all, which is the cause of a missing render node. + driver_status: >- + {{ 'SKIP' if (r.display_devices | length) == 0 + else ('FAIL' if (r.bound_drivers | length) == 0 + else ('PASS' if expected in r.bound_drivers else 'WARN')) }} + driver_detail: >- + {%- if (r.display_devices | length) == 0 -%} + no display-class PCI device on this machine, so no DRM driver can be bound to one. + Measured from /sys/bus/pci/devices. + {%- else -%} + {% for p in r.display_devices %}{{ p.address }} + {{ p.description | default(((p.vendor_id | default('?', true)) ~ ':' ~ (p.device_id | default('?', true))), true) }} + -> driver {{ p.driver | default('NONE BOUND', true) }}{{ '; ' if not loop.last else '' }}{% endfor %}. + {{ ('DRM cards: ' ~ (g.drm_cards | map(attribute='card') | zip(g.drm_cards | map(attribute='driver')) + | map('join', '=') | join(', ')) ~ '. ') if (g.drm_cards | length) > 0 + else 'No /sys/class/drm/card* entry exists, which is consistent with no driver being bound. ' }} + Module load state (/proc/modules, corroboration only — a built-in driver never appears + there, the bound-driver symlink above is what decides): + {{ g.modules | dict2items | map(attribute='key') | zip(g.modules | dict2items | map(attribute='value') | map('string')) + | map('join', '=') | join(', ') }}. + {%- if (r.bound_drivers | length) == 0 %} + NOTHING IS BOUND to the display device. That is why there is no render node: the DRM + subsystem creates /dev/dri/renderD* only when a driver attaches. Check whether the + module exists for this kernel and whether it was blacklisted or failed to probe. + {%- elif expected in r.bound_drivers %} + This is the expected driver for this site (node_health_graphics_expected_driver). + {%- else %} + EXPECTED `{{ expected }}`, GOT `{{ r.bound_drivers | join(', ') }}`. The device still + works, so this is not an outage — but the two DRM drivers do not share a userspace + media stack, and a libva/iHD build that transcodes correctly against one is not + automatically right for the other. Confirm Plex still hardware-transcodes before + treating this as benign. + {%- endif %} + {%- endif -%} + +# INFO ONLY, AND THAT IS A DELIBERATE RULING, NOT AN OVERSIGHT: +# +# GuC/HuC firmware not loaded (i915.enable_guc=0) — the accepted state on these nodes. It +# costs low-power H.264 encode QUALITY, nothing else; decode and encode both still work +# through the media engines. Grading it would put a permanent WARN on a configuration the +# operator has chosen. +# No host VAAPI userspace / no `vainfo` — NOT a defect. Plex ships its own libva and media +# driver inside its container, so the host needs neither. Flagging it would invite someone +# to `apt install vainfo` on a node to clear a health check, and this role installs nothing. +# +# The media ENGINES are the fact worth watching in this line: vcs* (video) and vecs* +# (video-enhancement) are what a transcode actually runs on. +- name: Assess — media engines, GuC/HuC firmware and host VAAPI userspace (informational) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'graphics.media_stack', + 'title': 'Media engines, GuC/HuC firmware and host VAAPI userspace (informational)', + 'status': ('SKIP' if (node_health_render.display_devices | length) == 0 else 'INFO'), + 'detail': media_detail }] }}" + vars: + ansible_become: false + g: "{{ node_health_facts.graphics }}" + r: "{{ node_health_render }}" + # `.get()` rather than a subscript: uc only carries the two Intel drivers, and a site + # that set the expected driver to amdgpu must get a clean "no such parameters" line + # instead of an undefined-key traceback. + guc: "{{ g.uc.get(node_health_graphics_expected_driver) }}" + guc_text: >- + {{ ('enable_guc=' ~ guc.enable_guc.value) if (guc is mapping and guc.enable_guc.value is not none) + else ('NOT READ — ' ~ (guc.enable_guc.reason | default('no reason recorded', true)) + if guc is mapping + else 'no /sys/module/' ~ node_health_graphics_expected_driver ~ '/parameters at all') }} + media_detail: >- + {%- if (r.display_devices | length) == 0 -%} + no display-class PCI device on this machine, so there is no media engine, no GuC/HuC + firmware and nothing to transcode with. + {%- else -%} + media engines: + {{ (r.media_engines | join(', ')) if (r.media_engines | length) > 0 + else 'NONE exposed under /sys/class/drm/card*/engine — a bound driver with no vcs*/vecs* engine cannot do media work' }} + (all engines: {{ (g.drm_cards | map(attribute='engines') | flatten | unique | sort | join(', ')) | default('none', true) }}). + GuC/HuC: {{ guc_text }}. + {{ ('Kernel cmdline carries ' ~ (g.cmdline_params | join(' '))) if (g.cmdline_params | length) > 0 + else 'No i915/xe parameter on the kernel cmdline' }}. + GuC/HuC being off is a KNOWN, ACCEPTED state here and is never graded: it costs + low-power H.264 encode quality only, and both decode and encode still run on the + engines above. + Host VAAPI userspace: vainfo + {{ ('present at ' ~ node_health_facts.tools.vainfo) if node_health_facts.tools.vainfo + else 'NOT installed on this host' }} + — also never graded, because Plex ships its own libva and media driver inside its + container, so the host needs none. This role installs nothing. + {%- endif -%} + # --- Thermals ----------------------------------------------------------------------- - name: Assess — temperatures ansible.builtin.set_fact: diff --git a/infra/ansible/roles/node_health/templates/summary.txt.j2 b/infra/ansible/roles/node_health/templates/summary.txt.j2 index 41ebb69..7e9f5ab 100644 --- a/infra/ansible/roles/node_health/templates/summary.txt.j2 +++ b/infra/ansible/roles/node_health/templates/summary.txt.j2 @@ -136,6 +136,37 @@ {% endfor %} {% endfor %} +-------------------------------------------------------------------------------- + GRAPHICS / RENDER DEVICE (the Plex hardware-transcoding path) +-------------------------------------------------------------------------------- +{% set gfx = node_health_facts.graphics %} +{% if gfx.pci_display %} +{% for p in gfx.pci_display %} + pci {{ '%-12s' | format(p.address) }} {{ p.description | default(((p.vendor_id | default('?', true)) ~ ':' ~ (p.device_id | default('?', true)) ~ ' class ' ~ (p.class_id | default('?', true))), true) }} + driver {{ p.driver | default('NONE BOUND', true) }} +{% endfor %} +{% else %} + pci no display-class device (PCI base class 0x03) on this machine — no GPU here +{% endif %} +{% for c in gfx.drm_cards %} + drm {{ '%-12s' | format(c.card) }} driver {{ '%-8s' | format(c.driver | default('-', true)) }} at {{ c.pci_address | default('-', true) }} engines {{ c.engines | join(' ') | default('none exposed', true) }} +{% endfor %} +{% if gfx.dri_present %} +{% for d in gfx.devices %} + dev {{ '%-12s' | format(d.name) }} {{ '%-18s' | format('character special' if d.char_device else 'NOT A CHAR DEVICE') }} mode {{ d.mode }} {{ d.owner | default(d.uid, true) }}:{{ d.group | default('(gid unresolved)', true) }} gid {{ d.gid }} +{% endfor %} +{% else %} + dev /dev/dri DOES NOT EXIST — no DRM device node on this node at all +{% endif %} +{# GID is printed, never asserted: `render` is a systemd dynamic allocation, so a rebuild may + legitimately move it and Plex's container init joins whichever group owns the device. See + defaults/main.yml (node_health_graphics_expected_driver) for the full reasoning. #} +{% set guc = gfx.uc.get(node_health_graphics_expected_driver) %} + GuC/HuC {{ ('enable_guc=' ~ guc.enable_guc.value) if (guc and guc.enable_guc.value is not none) else (('NOT READ — ' ~ (guc.enable_guc.reason | default('no reason recorded', true))) if guc else ('no /sys/module/' ~ node_health_graphics_expected_driver ~ '/parameters')) | regex_replace('\s+', ' ') | trim | wordwrap(64) | indent(14) }} + {{ ('kernel cmdline: ' ~ (gfx.cmdline_params | join(' '))) if gfx.cmdline_params else 'no i915/xe parameter on the kernel cmdline' }} + modules {% for m, loaded in gfx.modules.items() %}{{ m }}={{ loaded | string | lower }}{{ ', ' if not loop.last }}{% endfor %} (/proc/modules — corroboration only; the bound-driver symlink above decides) + vainfo {{ node_health_facts.tools.vainfo | default('NOT installed on this host — not a defect: Plex ships its own libva/media driver inside its container', true) }} + -------------------------------------------------------------------------------- PRESSURE STALL (/proc/pressure — % of wall time stalled) -------------------------------------------------------------------------------- diff --git a/runbooks/README.md b/runbooks/README.md index 53436b9..0eaaa5d 100644 --- a/runbooks/README.md +++ b/runbooks/README.md @@ -135,6 +135,22 @@ Each affected runbook says so on its own page and gives the by-hand equivalent. plex.tv/claim — so it is a timed sequence, scripted as `infra/plex/claim-ship.sh`: token → `sops` → PR → gate → merge → **forced** `flux reconcile` → pod roll (~2m45s measured). Includes why a changed Secret does not restart anything, and why an expired token is harmless. +- **`plex-quicksync`** — hardware transcoding on the host iGPU. GitOps puts `/dev/dri` in the pod + (`hardware.gpu: intel-quicksync`); **a human still has to tick Settings → Transcoder → "Use hardware + acceleration when available"**, and no environment variable can do it (the image supports no + `PLEX_PREFERENCE*`, verified). The page names the real preference key (`HardwareAcceleratedCodecs`, + read out of the binary), says which widely-cited key **does not exist**, and is explicit that the + **only** proof is `Transcode (hw)` on the Plex dashboard — Plex renders no ServiceMonitor, so no + alert could ever fire. Also: the device-cgroup failure to expect first and its device-plugin + fallback, and why a `:latest` pull can silently break it. +- **`oauth2-proxy-session`** — a gated UI loads but its saves quietly do nothing, and you are never + re-prompted to log in. Tells the three stacked auth layers apart from the sidecar's own access log + (the four lines that matter, including the upstream `-` that means the proxy answered a request the + app never saw). Covers the cookie collision between per-app sidecars sharing the default + `_oauth2_proxy` name on `.home.keithjackson.dev` — which deletes sessions belonging to apps in + *other* namespaces and is only legible with two logs side by side — the 5-minute access-token + session with no `--cookie-refresh`, and why an XHR answered with a 302 instead of a 401 loses a + POST body silently. - **`crafty-admin-credential`** — you signed into Keycloak and Crafty *still* asks for a password. Expected: Crafty 4 has no OIDC, so its own local login sits behind the oauth2-proxy gate as a second factor. The page tells the two layers apart from the sidecar log, and explains that the bootstrap diff --git a/runbooks/kustomization.yaml b/runbooks/kustomization.yaml index e016017..0cc6e4a 100644 --- a/runbooks/kustomization.yaml +++ b/runbooks/kustomization.yaml @@ -89,11 +89,25 @@ configMapGenerator: # that "is up but does not work" — the gate answers an unauthenticated XHR with a 302, # not a 401, so the failure arrives as an HTTP 200 carrying the wrong body. - oauth2-proxy-auth-failures=oauth2-proxy-auth-failures.md + # Operator procedure, not alert-linked. Reading the per-app oauth2-proxy sidecar log to tell + # Keycloak, the sidecar and the app's own login apart. Cookie collision between two sidecars + # presents as random session loss in ONE app's log and is only legible with both side by side, + # so the page has to be reachable at 3am without going looking for it in git. + # SIBLING PAGE to oauth2-proxy-auth-failures above, and they are deliberately both here: this + # one is the SESSION-LIFECYCLE procedure (why a session died), that one is the DIAGNOSTIC + # TRIAGE (which of the three layers refused, and how to tell a 302-to-200 from a real success). + - oauth2-proxy-session=oauth2-proxy-session.md # Operator procedure, not an alert page (no alert links to it — the # alert-runbooks-resolvable check treats that as advisory). It is served # because the plex.tv claim window is short and is not a thing to go looking # for in git: /runbooks/plex-claim has to resolve the moment it is needed. - plex-claim=plex-claim.md + # Operator procedure, not an alert page, and NO alert is possible for it: plex renders no + # metricsEndpoint, so there is no ServiceMonitor, no transcode-mode metric and nothing a rule + # could evaluate. Served because the feature it completes is INERT until a human ticks a box in + # the Plex UI — the one step GitOps cannot reach — and a page nobody can find is how a shipped + # feature stays switched off for months. + - plex-quicksync=plex-quicksync.md - pvc-capacity=pvc-capacity.md - targets-down=targets-down.md - update-rollback=update-rollback.md diff --git a/runbooks/oauth2-proxy-session.md b/runbooks/oauth2-proxy-session.md new file mode 100644 index 0000000..da09678 --- /dev/null +++ b/runbooks/oauth2-proxy-session.md @@ -0,0 +1,195 @@ +# oauth2-proxy-session — telling the auth layers apart from the sidecar log + +**Not alert-driven.** This is an operator procedure for the per-app oauth2-proxy sidecars +(`jackett`, `pihole`, `qbittorrent`, `radarr`, `sonarr`, `crafty`) and the shared forward-auth +proxy in namespace `oauth2-proxy`. Use it when a gated UI behaves as though you are signed in and +signed out at the same time: the page loads, but its buttons and saves quietly do nothing. + +> The sidecar log contains **email addresses, session UUIDs and full callback URLs including an +> OAuth `code`**. Read it, do not paste it. (Repo rule §6: secrets never enter a transcript.) + +## What this means here + +Every gated app on `home.keithjackson.dev` has **at least two** independent authentications +stacked on it, and the symptom rarely names the one that refused: + +1. **Keycloak** — the `homelab` realm issues the token. Failures here are visible: you see a login + form, or an error page from Keycloak's own hostname. +2. **The oauth2-proxy sidecar** — validates the session cookie and the group claim + (`media-users` / `platform-admins`). **This is the layer that fails silently**, and the rest of + this page is about reading it. +3. **The app's own login**, where it has one — Crafty and Radarr/Sonarr each present a local login + *behind* the Keycloak gate. See `runbooks/crafty-admin-credential.md` for that layer. + +The sidecar's access log distinguishes all three, if you know the four lines to look for. Get it: + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n jackett logs \ + -l app.kubernetes.io/name=jackett -c oauth2-proxy --tail=-1' + +Substitute the namespace and label for the app you are chasing. Always pass `--context k3s-prod`: +`dev.sh` merges both clusters and picks a current-context **per invocation**, and that default has +already changed once. Do not run two `dev.sh` invocations at the same time. + +### The four lines, and what each one means + +| Log line | What actually happened | Which layer | +|---|---|---| +| `[AuthSuccess] Authenticated via OAuth2: Session{... groups:[...]}` | Keycloak issued a token and the proxy accepted it. **The IdP is fine.** | Keycloak ✔ | +| `[oauthproxy.go:1026] No valid authentication in request. Initiating login.` | **No session cookie was presented at all.** Either you never had one, or something *deleted* it — see below. | sidecar | +| `[stored_session.go:94] Error loading cookied session: cookie signature not valid, removing session` | A cookie **of the right name** arrived that this proxy **could not decrypt**. Almost always a *different* proxy's cookie. Note `removing session`: this proxy then **deletes** it. | sidecar, cookie collision | +| an upstream path of `-` instead of `/` in an access-log line | the **proxy answered the request itself**; it never reached the app. Compare `POST - "/api/..."` against `POST / "/api/..."`. | sidecar | + +That last column is the one people miss. The upstream field sits between the method and the quoted +path. `-` means the proxy handled it (a redirect, a 401, its own `/oauth2/*` endpoints); `/` means +it was proxied to the app. Count them: + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n jackett logs \ + -l app.kubernetes.io/name=jackett -c oauth2-proxy --tail=-1 \ + | grep -v "\"/ping\"" | grep -E " (POST|PUT|DELETE) " | cut -c1-140' + +If your writes show `-`, the app never saw them. On 2026-09-30, **4 of the 6 POSTs** in jackett's +log had `-`. + +## First commands + +**1. Did Keycloak ever succeed?** No `AuthSuccess` at all means this is not a cookie problem — +go to `runbooks/keycloak-down.md` or `runbooks/keycloak-client-secrets.md`. + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n logs \ + -l app.kubernetes.io/name= -c oauth2-proxy --tail=-1 | grep -c AuthSuccess' + +**2. How long does a session last?** Subtract `created` from `expires` on an `AuthSuccess` line. +If it is exactly **5m**, the session's lifetime is Keycloak's access-token lifetime and the proxy +is not refreshing it: + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n logs \ + -l app.kubernetes.io/name= -c oauth2-proxy --tail=-1 \ + | grep AuthSuccess | tail -1 | grep -o "created:[^ ]* [^ ]*\|expires:[^ ]* [^ ]*"' + +The same line says `refresh_token:true` — the proxy is *holding* a refresh token. It will only +spend it if `--cookie-refresh` is set; the default is `0`, meaning disabled. + +**3. Is this a cookie collision?** Two checks, and you want both. + +*Names and scopes* — every instance that emits the default name `_oauth2_proxy` on the shared +`.home.keithjackson.dev` domain is in the collision set with every other one: + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get pods -A -o json \ + | grep -E "cookie-name|cookie-domain"' + +An instance with **no** `--cookie-name` argument is using the default. That is the failure — the +flag being absent, not wrong. + +*The smoking gun* — search **every** proxy, not just the one you are debugging, for the removal +line, and put the timestamps next to each other: + + infra/devcontainer/dev.sh run bash -c 'for ns in jackett pihole qbittorrent radarr sonarr crafty; do \ + echo "== $ns"; kubectl --context k3s-prod -n $ns logs -l app.kubernetes.io/name=$ns \ + -c oauth2-proxy --tail=-1 2>/dev/null | grep stored_session; done' + +**4. Do the cookie-secrets actually differ?** They should — but confirm the collision is between +instances that genuinely cannot read each other. Compare **hashes**, never values: + + infra/devcontainer/dev.sh run bash -c 'for ns in jackett pihole qbittorrent radarr sonarr crafty; do \ + printf "%s " "$ns"; kubectl --context k3s-prod -n $ns get secret $ns-oauth2-proxy \ + -o jsonpath="{.data.cookie-secret}" | sha256sum | cut -c1-12; done' + +Never print `.data` or `.stringData` itself. A password has been leaked into a transcript here +before. + +### Do not rank apps by their failure ratio + +Counting `No valid authentication` against `AuthSuccess` per app looks like a health metric and is +not one. **It ranks chattiness, not breakage.** An app whose UI polls or holds a WebSocket +manufactures one failure line per retry, so a single dead cookie in one idle browser tab becomes +scores of log lines, while an app nobody is using scores zero no matter how broken it is. Measured +2026-09-30: Radarr showed **156** unauthenticated requests against **1** `AuthSuccess`, but the top +paths were `/api/v3/command` (62), `/api/v3/queue/details` (46) and `/signalr/messages/negotiate` +(43) — polling, not 156 user-visible failures. + +**Bound the window to the current pod, not to wall-clock.** A ratio measured over 24 h across a pod +that restarted 13 h ago is mostly about the *previous* configuration, and that is exactly how a +deployed fix gets written off as ineffective. The pod's own log has an exact boundary — its process +start — so read it there: + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod -n get pod \ + -l app.kubernetes.io/name= -o jsonpath="{.items[*].status.startTime}{\"\n\"}"' + +then count within that same pod's log. On 2026-09-30 this took Crafty from an apparent 81:1 over 24 h +to **3 failures in its fixed pod's entire 13 h 52 m life**, two of them `/ws` reconnects. + +## Likely causes, most likely first + +1. **Cookie collision between two per-app proxies.** Several instances emit the default + `_oauth2_proxy` on `.home.keithjackson.dev` with different `cookie-secret`s. Whoever + authenticates last overwrites the rest; the losers log `cookie signature not valid, removing + session` and — because `removing session` clears the cookie on *their* `--cookie-domain`, which + is the whole zone — **delete a session belonging to an app in another namespace**. The victim + then logs `No valid authentication` with no cookie at all, seconds after a clean login, while + its own `expires` is still minutes away. **Read both logs side by side; one app's log alone + makes this look like random session loss.** Fix: a unique `--cookie-name` and a host-scoped + `--cookie-domain` on the per-app sidecar. The shared forward-auth proxy keeps the zone-wide + scope — cross-host auth is its job. +2. **Session pinned to a 5-minute access token with no refresh.** `--cookie-refresh` unset means + the proxy never spends the refresh token it holds, so the session expires with the access token. + Set it to about half the measured token lifetime. +3. **An XHR got a 302 instead of a 401.** The app's JavaScript follows the redirect invisibly, the + live Keycloak SSO session satisfies it without a prompt, and the XHR resolves with **HTML where + it expected JSON**. Nothing is logged as an error and the operator is never re-prompted — this + is what "it just doesn't save, and reloading doesn't help" looks like. Worse on a write: a + browser following a 302 re-issues a POST as a GET and **drops the body**, so the change is + discarded silently. oauth2-proxy's built-in AJAX guard only fires on an exact + `Accept: application/json`, which most app JavaScript does not send; make it deterministic with + `--api-route=` (**singular** — `--api-routes` does not exist and the proxy refuses to + start). Keep browser-navigable paths *out* of the regex so a plain page load still signs you in. +4. **Group membership.** An `AuthSuccess` followed by `403` is the `--allowed-group` gate, not a + session problem. Check the `groups:[...]` field on the `AuthSuccess` line. +5. **The app's own second login.** Crafty, Radarr and Sonarr each have one. `AuthSuccess` followed + by the app's own `/login` page is layer 3, not layer 2. + +## When to escalate or stop + +- **Stop before you `kubectl edit`/`patch`/`apply` anything.** Flux reconciles `main`; a + hand-applied flag is drift that the next reconcile reverts or, worse, adopts. Argument changes + go through a branch and a PR. Reading logs is always safe; changing the Deployment is not an + agent action. +- **Do not hand-edit `apps/**/.yaml` — those files are GENERATED.** The five per-app sidecars + (jackett, pihole, qbittorrent, radarr, sonarr) get their argument list from one shared function, + `_oauth2_sidecar_container()` in `catalog/render.py`; `catalog/.yaml` declares only + `auth.mode` and `auth.groups`, and the `auth` schema is `additionalProperties: false`, so there + is **no per-app override hook**. A hand-edit to the rendered file is caught by the merge gate's + `render-golden` stage ("GOLDEN DRIFT ... does not match the canonical render") and would be + erased by the next render anyway. Change the renderer, then regenerate: + + python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation + + **Budget for the blast radius**: because the function is shared, any change to it rewrites the + sidecar block in **all five** rendered apps, and the gate requires all five be regenerated in + the same commit. There is no way to change one app's sidecar arguments alone. Measured + 2026-09-30. Coordinate before starting — this is a single change, not five. +- **Crafty is the exception** and is not rendered from the catalog: its sidecar is hand-written in + `platform/crafty/deployment.yaml`, so it can be, and was, fixed independently. +- **You cannot currently alert on any of this**, and that is worth knowing before you promise + monitoring. Measured 2026-09-30: no per-app sidecar sets `--metrics-address` and none declares a + `metrics` container port, so oauth2-proxy's metrics listener never starts — the only oauth2-proxy + series in Prometheus come from the shared proxy in namespace `oauth2-proxy`, which does set both. + Until the renderer emits them for the per-app sidecars, every diagnosis on this page is + log-reading only. +- **Validate an argument change before you open the PR**, not after. The pinned image will tell + you what it understood, without touching the cluster: + + podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help + + Running it with your full proposed argument list prints a `Cookie settings:` line and an + `API route - Path:` line back at you, and rejects mutually exclusive flags outright + (`--session-cookie-minimal` with `--cookie-refresh`, for one). A flag typo here is a CrashLoop + on a service you have just told someone is fixed. +- **Do not rotate a `cookie-secret` to "clear" a bad session.** It signs every live session out of + that app and does nothing about the collision, which is caused by the *names* matching, not the + secrets differing. +- **Escalate to Keycloak** (`runbooks/keycloak-down.md`, `runbooks/keycloak-client-secrets.md`) if + there is no `AuthSuccess` in any proxy's log — that is an IdP or client-secret problem and no + cookie flag will help. +- **Hand off rather than guess** if you see `cookie signature not valid` on a proxy that already + has a unique `--cookie-name`. That combination should not occur; it means something is writing + that name which is not in this inventory. Record what you measured. diff --git a/runbooks/plex-quicksync.md b/runbooks/plex-quicksync.md new file mode 100644 index 0000000..4ccd4d5 --- /dev/null +++ b/runbooks/plex-quicksync.md @@ -0,0 +1,283 @@ +# plex-quicksync — making Plex actually use Intel Quick Sync, and proving it did + +**Alert:** none, and none is possible — see "How to verify" below. This is an **operator procedure**, +not an incident page. It is indexed here because the feature it completes is **inert until a human +clicks one button**, and a switched-off feature that looks shipped is exactly what this page exists +to prevent. +**Files:** `catalog/plex.yaml` (`hardware.gpu: intel-quicksync`), `apps/wave-0/plex.yaml` (the +rendered golden — the `dri` hostPath volume and the `/dev/dri` mount), `catalog/render.py` (the +mechanism), `schemas/renderer-contract.md` `### hardware` (the contract). +**Related:** `runbooks/plex-claim.md` — the other Plex procedure with a manual last mile. + +## What this means here + +Plex transcodes in **software** by default. On the UHD Graphics 630 iGPU of an i7-9700T, Quick Sync +moves that work off the CPU — which matters on a three-node cluster where the CPU is also running +everything else. Two independent things have to be true before that happens, and **only one of them +is in git**: + +| Half | Who does it | State | +|---|---|---| +| The render device is inside the container at `/dev/dri` | **GitOps.** `hardware.gpu: intel-quicksync` in `catalog/plex.yaml` renders a `dri` hostPath volume (`/dev/dri`, `type: DirectoryOrCreate`) and a read-write `/dev/dri` volumeMount on the app container. Flux applies it. | Done by merging that change. | +| Plex is *told* to use it | **The operator, by hand, in the Plex web UI.** | **NOT done.** `HardwareAcceleratedCodecs` was **absent** from `Preferences.xml` when this page was written — the toggle has never been on. | + +**There is no GitOps path to the second half, and that is verified rather than assumed.** A bounded +grep across the running image's entire init surface — `/etc/cont-init.d`, `/etc/services.d`, +`/etc/fix-attrs.d`, `/plex-common.sh`, `/usr/local/bin` — found **zero** `PLEX_PREFERENCE*` support. +Every `setPref` the image performs is one of six hardcoded keys (`MachineIdentifier`, +`ProcessedMachineIdentifier`, `PlexOnlineToken`, `customConnections`, `allowedNetworks`, +`TranscoderTempDirectory`), none of them the transcode toggle. No environment variable, no ConfigMap +and no `postBuild` substitution can set it. So do not go looking for the clever way to automate this; +there isn't one, and the search has already been done. + +What the device mount does **not** do, so nobody reads more into a green gate than is there: it does +not enable hardware transcoding, it does not prove the device can be *opened* from inside a +non-privileged pod (see "Likely causes" #2), and it renders no `nodeSelector` — Plex can land on any +of the three nodes and all three carry the device. + +## Running the commands in this runbook + +`kubectl` does not work on the operator workstation; the toolchain lives only in the pinned dev +container. Every command below runs as: + + infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get nodes' + +Always pass `--context k3s-prod` explicitly. `dev.sh` merges both clusters into one kubeconfig and +picks a current-context **per invocation**, and that default has already changed once. Do not run two +`dev.sh` invocations at once. + +**Never `cat` `Preferences.xml`.** It contains `PlexOnlineToken`, a plex.tv account credential. Every +read of that file below greps a single named key — do not widen one. + +## First commands + +**1. Is the device mounted into the pod at all?** (Proves Flux applied the change, not that it works.) + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \ + -o jsonpath="{.items[0].spec.containers[0].volumeMounts[*].mountPath}"; echo' +``` + +`/dev/dri` in that list ⇒ the GitOps half landed. Absent ⇒ the running pod predates the merge; compare + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n flux-system get kustomization apps-wave-0 \ + -o jsonpath="{.status.lastAppliedRevision}"; echo' +``` + +against the merge commit before blaming anything else. + +**2. Is the device actually visible inside the container?** + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex exec deploy/plex -- ls -l /dev/dri' +``` + +Expect `renderD128` (usually `card0` too), mode `crw-rw----`, group `render`. An **empty directory** +here is the `DirectoryOrCreate` degradation: the kubelet created an empty `/dev/dri` because the node +had none. That is the designed-safe outcome — the app tier still reconciles — but it means software +transcoding, silently. Find which node the pod is on and ask why that node lost its iGPU: + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex -o wide' +``` + +**3. Can Plex *open* it?** This is the question the manifest cannot answer — see "Likely causes" #2. + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex exec deploy/plex -- \ + bash -c "exec 3<>/dev/dri/renderD128 && echo RW-OPEN-OK || echo RW-OPEN-DENIED"' +``` + +`RW-OPEN-DENIED` (or `Operation not permitted` / `Permission denied`) with the device plainly visible +in step 2 is the **device-cgroup** failure, not a file-permission failure. The two look identical from +the outside and are not the same problem. + +**4. Is the toggle on?** Named key only — never the whole file: + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex exec deploy/plex -- \ + grep -o "HardwareAcceleratedCodecs=\"[01]\"" \ + "/config/Library/Application Support/Plex Media Server/Preferences.xml" || echo ABSENT' +``` + +`ABSENT` ⇒ never enabled (the state this page was written in). `="0"` ⇒ explicitly off. `="1"` ⇒ on, +and the remaining question is whether it *works* — go to "How to verify". + +## The one manual step + +Plex web UI → **Settings → Transcoder → "Use hardware acceleration when available"** → tick → Save. +(It sits under *Show Advanced*.) A transcode already in flight is unaffected; the next one picks it up. + +The underlying preference key is **`HardwareAcceleratedCodecs`**. That name was confirmed by +extracting strings from the 23 MB `Plex Media Server` binary in the running container — **not** from +recall and not from a forum post. Real neighbouring keys from the same binary, so you can recognise +them and not invent them: + +| Key | What it is | +|---|---| +| `HardwareAcceleratedCodecs` | the toggle this page is about | +| `HardwareAcceleratedEncoders` | hardware **encode**, as distinct from decode | +| `HardwareDevicePath` | how Plex is pointed at a *specific* render node — relevant only if a host ever exposes more than one | +| `TranscoderToneMapping`, `TranscoderToneMappingAgorithm` | HDR→SDR tone mapping. **That misspelling of "Algorithm" is upstream's, in the binary. It is not a typo in this page — do not "fix" it** here or anywhere else; the corrected spelling is not a key Plex reads. | +| `TranscoderHEVCEncoding` | HEVC output | + +**`HardwareAcceleratedCodecsEnableHEVC` does not exist.** It is widely cited and it is not in the +binary. Recorded explicitly so nobody spends an evening hunting for it or, worse, writes it into +`Preferences.xml` and concludes hardware transcoding is broken. + +Editing `Preferences.xml` by hand is **not** the procedure. It is app state on the replicated Longhorn +tier, it is in no git commit, and the UI writes the key correctly. Use the UI. + +## How to verify — and why there is no alert + +Start a playback that **forces a transcode** (a client that cannot direct-play the source, or pick a +lower quality explicitly), then read Plex's own **Dashboard → Activity**: + +- **`Transcode (hw)`** on the session ⇒ hardware transcoding. **This is the proof.** +- plain **`Transcode`** ⇒ software. The feature is not working, whatever the manifest says. + +**That dashboard is the *only* proof available, and nothing else will ever tell you.** +`catalog/plex.yaml` declares no `metricsEndpoint`, so Plex renders **no ServiceMonitor at all** — +there is no scrape, no transcode-mode metric, and therefore no alert that could fire. Do not open work +to "alert on software-transcode fallback": with no metric behind it such a rule would be permanently +inert, and this repo has already been burned at scale by rules that evaluated against nothing while +looking like coverage (`runbooks/README.md`, "Alerts you should not trust the silence of"). If that +gap is ever worth closing, the honest order is a real exporter first and a rule second. + +Corroborating, not a substitute — the server log on the config PVC: + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex exec deploy/plex -- \ + sh -c "grep -i -m20 -e hardware -e vaapi -e hwdevice \ + \"/config/Library/Application Support/Plex Media Server/Logs/Plex Media Server.log\""' +``` + +Treat those strings as advisory: they are Plex's, not this repo's, and they move between versions. +Absence of a match is **not** evidence of software transcoding; a VA-API device open is good evidence +of hardware. The dashboard remains the verdict. + +## Likely causes, most likely first + +When the dashboard still says plain `Transcode`: + +1. **The toggle is off.** Overwhelmingly the most likely answer, and the reason this page exists. + First-command #4 returns `ABSENT` or `="0"`. Remedy: the manual step above. Check this **before** + investigating anything technical. + +2. **containerd's device cgroup is refusing the `open()` — UNVERIFIED, and the largest remaining + technical risk in this feature.** Whether a non-privileged pod may `open()` a bind-mounted + character device on this k3s/containerd build **has not been established**; it can only be settled + by deploying and trying. The container inits as uid 0, which settles *file* permissions, but + **uid 0 does not bypass the cgroup device gate.** Symptom: step 2 shows `renderD128` present and + correctly owned, step 3 returns `RW-OPEN-DENIED`. + **The documented fallback is the Intel Device Plugin for Kubernetes** (`gpu.intel.com/i915` as a + resource request, no hostPath). Because the ServiceDefinition field expresses *intent* rather than + a device path, that migration is a **`catalog/render.py` change plus a re-render — with no + ServiceDefinition edit anywhere**; `catalog/plex.yaml` keeps saying exactly what it says today. + **`privileged: true` is not an available escape hatch:** Kyverno `restrict-host-namespaces` is in + **Enforce** mode and rejects it in app namespaces. Do not reach for it. + +3. **The pod landed on a node without the device.** Step 2 shows an empty `/dev/dri`. All three prod + nodes carried `renderD128` with `i915` bound when this was measured (2026-09-30), so this means a + node changed — a rebuild, a BIOS reset, or a driver regression. `DirectoryOrCreate` is why the app + tier still came up; the cost is that the loss is silent. **Fix the node, do not tighten the mount:** + `Directory` would stall `apps-wave-0` (`wait: true`, with `apps-wave-1` depending on it) for a + feature that is not load-bearing. + +4. **The image changed under you.** See "The floating-image caveat". A `:latest` pull can remove the + mechanism this feature leans on with no commit in this repo. + +5. **The session was never transcoding.** Direct Play / Direct Stream never touches the transcoder, so + `Transcode (hw)` cannot appear. Confirm the session actually says `Transcode` before treating it as + a hardware failure. + +6. **The source needs a codec Quick Sync on this generation cannot handle** — e.g. AV1 decode, which + Coffee Lake's UHD 630 has no engine for. Plex falls back to software *per stream*. Symptom: some + sessions show `(hw)` and others do not. Not a fault. + +## The floating-image caveat + +Everything this page states about the image's behaviour — the `cont-init.d` hook that adds the app +user to the device's group, the absence of `PLEX_PREFERENCE*`, the six `setPref` keys, the +`HardwareAcceleratedCodecs` string — was read from digest **`sha256:e0ab27395614…`** +(`plexmediaserver 1.43.4.10903`). + +The deployment runs **`plexinc/pms-docker:latest` with `imagePullPolicy: Always`**. A silent pull can +therefore change the behaviour this feature depends on **with no commit in this repo**, while the +manifest still looks identical. Symptom to watch for: hardware transcoding that worked and quietly +stopped, with no git change to point at. + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \ + -o jsonpath="{.items[0].status.containerStatuses[0].imageID}"; echo' +``` + +If that digest is not the one above, **re-verify the hook before trusting this page**: + +``` +infra/devcontainer/dev.sh run bash -c \ + 'kubectl --context k3s-prod -n plex exec deploy/plex -- ls /etc/cont-init.d/' +``` + +`45-plex-hw-transcode-and-connected-tuner` still present ⇒ the group mechanism is still the one +described here. Gone or renamed ⇒ stop and re-measure. In particular the "no `supplementalGroups` +needed" conclusion in `catalog/plex.yaml` and `schemas/renderer-contract.md` is only true while that +hook exists. + +## When to escalate or stop + +- **Do not add `supplementalGroups` or a `podSecurityContext` to `catalog/plex.yaml`.** It would be + **inert**, and it would mislead the next person. Measured in-container 2026-09-30: the image's + `cont-init.d` hook stats each `/dev/dri` char device and `usermod -a -G`s the `plex` user into its + group *inside* the container, and the service then execs via `s6-setuidgid plex`, which **replaces** + the process group list from `/etc/group` rather than appending — `Plex Media Server` runs with + `Groups: 100 1000`, root's gid `0` (which PID 1 does hold) gone. Anything Kubernetes supplies is + discarded before Plex starts. `render.py --selftest` asserts that absence deliberately. +- **Do not reach for `privileged: true`.** Kyverno `restrict-host-namespaces` is Enforce and will + reject it. If the cgroup gate is the blocker, the answer is the Intel device plugin (cause #2). +- **Do not `kubectl edit`/`patch` the Deployment to add or change the device mount.** Flux reconciles + `main`; a hand-applied change is drift that the next reconcile reverts or, worse, adopts. Mutating + `kubectl` and `flux reconcile` are **operator** actions, and the durable change is a git change + through a branch, a PR and a green gate. +- **Do not edit `Preferences.xml` on the PVC**, never `cat` it, and never `sops -d` anything into a + terminal. It holds `PlexOnlineToken`. +- **Do not delete the `plex-config` PVC to "start clean".** It is the library database, metadata and + watch state (`policy: daily-offcluster`). The transcode toggle is one preference, not a reason to + lose the DB. +- **Stop and get the operator** if the device cannot be opened from the pod (cause #2 confirmed). The + next step is a platform change — the Intel device plugin — with its own PR, not more poking at this + workload. + +## Known gaps, deliberately not fixed here + +Recorded so they are findable, and so nobody assumes they were overlooked: + +- **Transcode scratch is on the wrong medium, and it is invisible.** The image sets + `TranscoderTempDirectory=/transcode`, which is **not a volume** — it is the container's writable + overlay, i.e. the node's `/data`. That is NVMe on prod-2 but **SATA on prod-1 and prod-3**, so + transcode scratch performance changes silently whenever the pod is rescheduled. Both candidate fixes + need a **schema** change and are therefore out of scope here: pin Plex to prod-2 (needs a + `nodeSelector` field the schema does not have), or give `/transcode` a `medium: Memory` emptyDir and + transcode in RAM (~31 GiB per node; needs an emptyDir field the schema does not have). The RAM option + is the better one, because it removes the node asymmetry outright instead of working around it. +- **GuC/HuC firmware is not loaded** (`i915.enable_guc=0`). The cost is low-power H.264 **encode** + quality only — decode, and the feature on this page, are unaffected. Changing it needs a kernel + parameter and a **serialized rolling reboot**, which is an operator action (CLAUDE.md §5), and this + cluster has a documented history of that going badly (a prod node hung on a warm reboot on + 2026-09-03). Not worth coupling to this change. + +--- +Source of truth: `runbooks/plex-quicksync.md`. Index: `/runbooks/` +The node measurements (all three prod nodes) and the in-container image measurements cited here were +taken **2026-09-30** against `plexinc/pms-docker` digest `sha256:e0ab27395614…` +(`plexmediaserver 1.43.4.10903`). Hardware transcoding had **not** been observed working at the time +of writing, and the Plex-side toggle was still unset. diff --git a/schemas/README.md b/schemas/README.md index 6847b69..20bddc9 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -15,7 +15,7 @@ docs/13 exactly. Do not diverge from it here — escalate spec problems, don't f | `servicedefinition.schema.json` | Formal JSON Schema (draft 2020-12) for a ServiceDefinition. Enforces enums, required fields, patterns, and cross-field invariants. | | `renderer-contract.md` | Authoritative field → rendered-resource mapping, with requirement IDs. The contract other workstreams consume. | -**Schema version:** 1.0.0 (in the schema `description`). Treat any change as an interface change: +**Schema version:** 1.8.0 (in the schema `description`, which carries the full per-version changelog — this line had said `1.0.0` since 1.1.0 and is now synced). Treat any change as an interface change: propose on the task board, update dependent renderers/tests in the same change, never silently break a consumer (coordination-protocol §2). @@ -35,6 +35,9 @@ consumer (coordination-protocol §2). - `ingressRoutes[].host` must match `