Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2983378496 | ||
|
|
99560c2d22 | ||
|
|
cdf3e14e27 | ||
|
|
41a20dfd20 | ||
|
|
df1fd1b3d1 | ||
|
|
0fee584e3a | ||
|
|
f716e9d167 | ||
|
|
fa4765bb3e |
No files matched your search
@@ -100,6 +100,7 @@ spec:
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
- --api-route=^/api/
|
||||
- --cookie-name=_oauth2_proxy_jackett
|
||||
- --cookie-domain=jackett.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
|
||||
@@ -143,6 +143,8 @@ spec:
|
||||
- name: media
|
||||
mountPath: /media
|
||||
readOnly: true
|
||||
- name: dri
|
||||
mountPath: /dev/dri
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
@@ -150,6 +152,10 @@ spec:
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: plex-media
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: DirectoryOrCreate
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
@@ -153,6 +153,8 @@ spec:
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --api-route=^/api/
|
||||
- --api-route=^/signalr/
|
||||
- --cookie-name=_oauth2_proxy_radarr
|
||||
- --cookie-domain=radarr.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
|
||||
@@ -51,6 +51,23 @@ auth:
|
||||
# This grants no real privilege: platform-admins already maps to cluster-admin via
|
||||
# k8s-oidc-rbac.yaml (oidc:platform-admins), so a member can already exec into these pods.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
# Answer Jackett's OWN API with 401 instead of a 302 to Keycloak (schema 1.9.0; renders
|
||||
# --api-route, singular and repeatable, onto the oauth2-proxy sidecar).
|
||||
#
|
||||
# MEASURED on this pod's access log, 2026-09-30: on /api/ paths, 33 x 302 against 2 x 401 —
|
||||
# and 4 of 6 POSTs were answered BY THE PROXY and never reached Jackett (access-log upstream
|
||||
# field `-`, not `/`), including both `/api/v2.0/server/update` calls. Jackett's jQuery does
|
||||
# not send the exact `Accept: application/json` oauth2-proxy's AJAX guard requires, so the
|
||||
# 302 was followed invisibly, satisfied silently by the live Keycloak SSO session, and
|
||||
# resolved 200 carrying the dashboard's HTML where the JavaScript expected JSON. A browser
|
||||
# following a 302 also re-issues a POST as a GET and DROPS THE BODY, which is why "adding an
|
||||
# indexer silently does nothing" was the user-visible symptom and nothing logged a failure.
|
||||
#
|
||||
# SCOPE: `/api/` only. Jackett serves its UI at /UI/ and its static assets elsewhere, and a
|
||||
# browser never NAVIGATES to /api/ — so a plain page load is still 302'd into a silent SSO
|
||||
# sign-in, which is the behaviour that must not change.
|
||||
apiRoutes:
|
||||
- ^/api/
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=0.22.0" # stay on the Jackett 0.22.x stable line
|
||||
|
||||
@@ -107,6 +107,13 @@ auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no native OIDC (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- platform-admins # sensitive admin UI: authenticated AND in group (R-IDP-06)
|
||||
# NO `apiRoutes` HERE, DELIBERATELY. PiHole's admin UI serves no /api/ path, so the `^/api/`
|
||||
# that jackett and radarr need would match nothing on a good day — and a BROADER guess would
|
||||
# answer PiHole's own page loads with 401, leaving a logged-out operator on a bare error page
|
||||
# with no sign-in offered. This app is the reason auth.apiRoutes is per-service (schema 1.9.0)
|
||||
# rather than a renderer constant: the measurement that justifies the flag on two apps is
|
||||
# evidence about those two apps. Its oauth2-proxy log holds 0 unauthenticated requests across
|
||||
# the current pod's whole 23h lifetime (measured 2026-10-01), so there is nothing to act on.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=2024.0.0"
|
||||
|
||||
@@ -80,6 +80,58 @@ env:
|
||||
# remote access are UNVERIFIED. Advertising a wrong/internal-only URL breaks remote access for ALL
|
||||
# clients, so per "prefer safety over completeness" it is left unset. FOLLOW-UP: once remote
|
||||
# reachability through the Gateway is confirmed, set ADVERTISE_IP: "https://plex.home.keithjackson.dev:443".
|
||||
# HARDWARE TRANSCODING — the host iGPU's render device (schema 1.8.0 `hardware`; R-PLAT-02).
|
||||
# INTENT ONLY. This says "this workload needs an Intel Quick Sync render device"; it does NOT name a
|
||||
# device path. The renderer currently satisfies it with exactly two things — a `/dev/dri` hostPath
|
||||
# volume (`type: DirectoryOrCreate`) and a read-write `/dev/dri` volumeMount on the app container —
|
||||
# and nothing else: no nodeSelector, no privileged, no resources (schemas/renderer-contract.md
|
||||
# `### hardware`). If that mechanism is later replaced by the Intel GPU device plugin
|
||||
# (`gpu.intel.com/i915`), it is a `catalog/render.py` change and THIS BLOCK DOES NOT MOVE. Do not
|
||||
# "simplify" it into a raw `/dev/...` path; that is the whole point of the enum.
|
||||
#
|
||||
# THE DEVICE IS PRESENT ON EVERY PROD NODE, so nothing had to be installed and Plex can land
|
||||
# anywhere. Measured on all three nodes 2026-09-30: `/dev/dri/renderD128` present, `root:render`,
|
||||
# GID 992, mode 0660, driver `i915` bound (NOT `xe`), UHD Graphics 630 on an i7-9700T, media engines
|
||||
# `vcs0`/`vecs0` live, iGPU enabled in BIOS on each. Plex happens to run on k3s-prod-2 today, but
|
||||
# that is scheduler history rather than a requirement — this field deliberately renders no
|
||||
# nodeSelector, because placement is a separate concern with its own justification.
|
||||
#
|
||||
# NO `podSecurityContext.supplementalGroups`, AND ADDING ONE WOULD BE INERT — do not "fix" this.
|
||||
# The device is `root:render` 0660 and this process is uid/gid 1000, so the obvious move is to push
|
||||
# the host `render` GID in as a supplemental group. It cannot work here. MEASURED IN THE RUNNING
|
||||
# CONTAINER, 2026-09-30: the image's own
|
||||
# `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` stats every char device under
|
||||
# `/dev/dri`, `groupadd`s a matching group INSIDE the container when none exists, and
|
||||
# `usermod -a -G`s the `plex` user into it — every start, nothing hardcoded, which also makes a host
|
||||
# GID renumbering self-healing (no GID is pinned anywhere in this repo). The service then execs via
|
||||
# `s6-setuidgid plex`, and `s6-setuidgid` REPLACES the process group list from `/etc/group` rather
|
||||
# than appending to it: `Plex Media Server` runs with `Groups: 100 1000`, and root's gid `0` — which
|
||||
# PID 1 (`s6-svscan`) does hold — is GONE. Anything Kubernetes supplies therefore reaches s6-svscan
|
||||
# and is discarded before Plex ever starts. Mounting the device is sufficient; the group is the
|
||||
# image's job and it does it correctly. `render.py --selftest` asserts the ABSENCE of a pod
|
||||
# securityContext for this field precisely so nobody adds an inert one back later.
|
||||
#
|
||||
# `DirectoryOrCreate` IS THE SAFE FAILURE MODE, not laziness. A node that comes back without the
|
||||
# device (rebuild, BIOS reset, driver regression, hardware swap) gets an empty directory and Plex
|
||||
# degrades to SOFTWARE transcoding. `Directory` would fail the mount instead — and because
|
||||
# `apps-wave-0` is `wait: true` with `apps-wave-1` `dependsOn` it, that would stall the WHOLE app
|
||||
# tier for a feature that is not load-bearing. The price, stated plainly: a missing device is
|
||||
# SILENT. This SD renders no `metricsEndpoint`, so there is no ServiceMonitor, no transcode-mode
|
||||
# metric, and no alert that could ever fire. The only proof is an operator reading the Plex
|
||||
# dashboard.
|
||||
#
|
||||
# THE LAST MILE IS A MANUAL OPERATOR STEP AND CANNOT LIVE IN THIS REPO. Mounting the device does
|
||||
# not enable hardware transcoding — Plex's own `HardwareAcceleratedCodecs` preference does, and it
|
||||
# is ABSENT from `Preferences.xml` today (the toggle has never been on). NO environment variable can
|
||||
# set it: a grep of the running image's entire init surface (`/etc/cont-init.d`, `/etc/services.d`,
|
||||
# `/etc/fix-attrs.d`, `/plex-common.sh`, `/usr/local/bin`) found ZERO `PLEX_PREFERENCE*` support,
|
||||
# and the six prefs the image does write (`MachineIdentifier`, `ProcessedMachineIdentifier`,
|
||||
# `PlexOnlineToken`, `customConnections`, `allowedNetworks`, `TranscoderTempDirectory`) are all
|
||||
# unrelated. So there is no GitOps path to the toggle; an operator clicks it once in the web UI.
|
||||
# `runbooks/plex-quicksync.md` is that procedure, plus how to verify it took and the
|
||||
# device-cgroup failure mode to expect first.
|
||||
hardware:
|
||||
gpu: intel-quicksync
|
||||
exposure: public # edge port-forward + public DNS; clients + Seerr reach it (docs/09; R-DNS/R-NET)
|
||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||
storage:
|
||||
|
||||
@@ -448,6 +448,12 @@ auth:
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
# NO `apiRoutes` HERE: UNMEASURED, not cleared. qBittorrent's WebUI drives everything through
|
||||
# /api/v2/, so it is plausibly affected in the same way jackett is — but plausibility is what
|
||||
# put a renderer constant on the table in the first place. MEASURED 2026-10-01 on the live pod
|
||||
# (qbittorrent-5547bdb9bd-zq557, 0 restarts, 23h): 13,649 log lines, ZERO unauthenticated
|
||||
# requests, nothing but health probes. No traffic means no evidence either way. Add `^/api/`
|
||||
# when someone opens the WebUI and records the 302/401 split, as was done for jackett.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=5.0.0 <6.0.0" # stay on the qBittorrent 5.x stable line
|
||||
|
||||
@@ -112,6 +112,32 @@ auth:
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
# Answer Radarr's machine endpoints with 401 instead of a 302 to Keycloak (schema 1.9.0;
|
||||
# renders --api-route, singular and repeatable, onto the oauth2-proxy sidecar).
|
||||
#
|
||||
# `^/signalr/` — MEASURED ON THE LIVE POD, 2026-10-01, 23h after the cookie fix deployed
|
||||
# (radarr-7d55cbffb9-42hmh, 0 restarts): 232 unauthenticated requests in the pod's whole
|
||||
# lifetime and ALL 232 are the same call — `POST /signalr/messages/negotiate` from a browser
|
||||
# UA, answered 302 by the proxy with upstream `-`, i.e. Radarr never saw one of them. That is
|
||||
# a left-open tab whose session the cookie rename orphaned, polling SignalR forever, silently
|
||||
# redirected every time and NEVER re-prompting the operator. It is the reported symptom
|
||||
# ("reloading never fixes it and I am never re-prompted") still running in production on a
|
||||
# service nobody was watching. With 401 the UI's connection breaks LOUDLY, the operator
|
||||
# reloads, and `/` is outside both regexes so the reload signs them back in.
|
||||
#
|
||||
# `^/api/` — measured on radarr, but on the PRE-FIX pod and NOT by me: the 2026-09-30
|
||||
# diagnosis (decisions/2026-09-30-jackett-oauth2-proxy-session.md) recorded radarr's 156
|
||||
# unauthenticated requests as `/api/v3/command` (62), `/api/v3/queue/details` (46) and
|
||||
# `/signalr/messages/negotiate` (43). The current pod has logged ZERO /api/ traffic, so I did
|
||||
# not re-measure it; it is included because 108 of 156 measured rejections on this app were
|
||||
# /api/v3 XHRs, which is this app's own evidence rather than an analogy to another app.
|
||||
#
|
||||
# SCOPE: both are JSON/RPC namespaces a browser never NAVIGATES to. Radarr's UI is served at
|
||||
# `/` with assets under /Content/ and /MediaCover/, none of which match — so a plain page load
|
||||
# is still 302'd into a silent SSO sign-in.
|
||||
apiRoutes:
|
||||
- ^/api/
|
||||
- ^/signalr/
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=5.0.0 <6.0.0" # stay on the Radarr v5 stable line
|
||||
|
||||
+344
-15
@@ -163,6 +163,54 @@ OAUTH2_COOKIE_REFRESH = "2m30s"
|
||||
# are added. Do not read this flag as "metrics are working".
|
||||
OAUTH2_PROXY_METRICS_PORT = 44180
|
||||
|
||||
# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------
|
||||
# The ServiceDefinition declares the CAPABILITY (schemas/servicedefinition.schema.json
|
||||
# `hardware.gpu`); everything below is the MECHANISM that currently satisfies it and is
|
||||
# deliberately confined to this file. Swapping to the Intel GPU device plugin
|
||||
# (`gpu.intel.com/i915` resource requests instead of a hostPath) is a change here and
|
||||
# nowhere else -- no catalog entry moves.
|
||||
DRI_VOLUME_NAME = "dri"
|
||||
DRI_DEVICE_DIR = "/dev/dri"
|
||||
|
||||
# --- API routes: 401 instead of 302, PER APP ------------------------------------------
|
||||
# oauth2-proxy answers an unauthenticated request with a 302 to Keycloak. Its built-in AJAX
|
||||
# guard would send a 401 instead, but it fires only on an EXACT `Accept: application/json`,
|
||||
# which these apps' JavaScript does not consistently send. So an unauthenticated XHR gets a
|
||||
# 302, the browser follows it INVISIBLY, the live Keycloak SSO session satisfies it silently,
|
||||
# and the call resolves 200 carrying the app's HTML where the JavaScript expected JSON — no
|
||||
# prompt, no console error, nothing logged as a failure. On a WRITE it is worse: a browser
|
||||
# following a 302 re-issues a POST as a GET and DROPS THE BODY, so the operator's change is
|
||||
# discarded and the UI reports success. MEASURED, twice:
|
||||
# jackett 2026-09-30: 33 x 302 against 2 x 401 on /api/ paths, and 4 of 6 POSTs answered by
|
||||
# the proxy and never reaching the app (access-log upstream field `-`, not `/`).
|
||||
# radarr 2026-10-01, 23h after the cookie fix rolled: 232 x 302, EVERY ONE a
|
||||
# `POST /signalr/messages/negotiate` from the operator's own browser, upstream `-`.
|
||||
# A left-open tab whose cookie the rename orphaned, polling forever, silently
|
||||
# redirected every time, NEVER re-prompting. That is the reported symptom
|
||||
# ("reloading never fixes it and I am never re-prompted") still happening in
|
||||
# production, on a service nobody was looking at.
|
||||
# `--api-route=<regex>` makes matching paths answer 401.
|
||||
#
|
||||
# WHY THIS IS NOT A RENDERER CONSTANT, AND MUST NEVER BECOME ONE. The path shape is a property
|
||||
# of the APP, not of the forward-auth pattern: radarr's broken path is `/signalr/`, jackett's is
|
||||
# `/api/`, and PIHOLE SERVES NO `/api/` AT ALL. One constant would apply one app's path shape to
|
||||
# five, and the failure mode of a wrong one is not "no improvement" but "this service is
|
||||
# permanently broken": a path a BROWSER NAVIGATES to, answered 401, shows a logged-out operator
|
||||
# a bare error page and never offers a sign-in. So the routes are declared per service in
|
||||
# `auth.apiRoutes` (schema 1.9.0) and rendered one `--api-route` per entry — the flag is
|
||||
# repeatable, and it is SINGULAR: `--api-routes` does not exist in v7.8.1 and the proxy refuses
|
||||
# to start with it (verified against the pinned binary's own --help, 2026-10-01:
|
||||
# `--api-route strings return HTTP 401 instead of redirecting to authentication server...`).
|
||||
#
|
||||
# BREADTH RATCHET. Every entry must be anchored at `^/` and continue with literal path
|
||||
# characters. That rejects `^/`, `^/.*`, `.*`, an unanchored `/api/`, and any alternation — the
|
||||
# shapes that turn a page load into a 401. Alternation is refused rather than supported because
|
||||
# the flag is repeatable, so one regex per route costs nothing and stays readable. The same
|
||||
# pattern is in the schema (so ci/l0.sh's schema stage rejects it at the authoring surface) and
|
||||
# enforced HERE (so ci/render-golden.sh fails even if the schema stage is skipped) — a breadth
|
||||
# mistake must not need two independent gates to both be running.
|
||||
OAUTH2_API_ROUTE_PATTERN = re.compile(r"^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Defaults (renderer-contract.md "Default posture") — materialize BEFORE mapping.
|
||||
@@ -188,6 +236,12 @@ def apply_defaults(sd: dict) -> dict:
|
||||
if "mode" not in auth:
|
||||
auth["mode"] = "forward-auth" if s["ingressRoutes"] else "none"
|
||||
auth.setdefault("groups", [])
|
||||
# apiRoutes default posture is EMPTY, deliberately, and that is the whole point of the field
|
||||
# (schema 1.9.0): an app gets --api-route only where someone MEASURED that it serves a machine
|
||||
# API, so a newly-onboarded app renders no --api-route at all rather than inheriting another
|
||||
# app's path shape. See OAUTH2_API_ROUTE_PATTERN above for why a default of `^/api/` would be
|
||||
# wrong rather than merely generous.
|
||||
auth.setdefault("apiRoutes", [])
|
||||
s["auth"] = auth
|
||||
|
||||
update = dict(s.get("update") or {})
|
||||
@@ -513,6 +567,46 @@ def r_deployment(s: dict) -> dict:
|
||||
"persistentVolumeClaim": {"claimName": f"{s['name']}-{vol['name']}"},
|
||||
}
|
||||
)
|
||||
# hardware capability (schema-optional): `hardware.gpu: intel-quicksync` means this
|
||||
# workload needs the host iGPU's render node for hardware video transcode. The SD says
|
||||
# only that; the wiring below is the platform's current mechanism for satisfying it.
|
||||
#
|
||||
# `type: DirectoryOrCreate` is DELIBERATE and load-bearing -- do NOT "tighten" it to
|
||||
# `Directory`. A node that comes back without the render device (rebuild, BIOS change,
|
||||
# driver regression, hardware swap) must degrade the workload to SOFTWARE transcoding,
|
||||
# not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` dependsOn it, so a
|
||||
# failed hostPath mount stalls the entire app tier for the sake of a feature that is not
|
||||
# load-bearing. `DirectoryOrCreate` turns that failure mode into "transcodes in
|
||||
# software" instead of "the app tier does not reconcile". /dev is tmpfs, so the empty
|
||||
# directory the kubelet creates on such a node leaves nothing durable behind.
|
||||
#
|
||||
# The mount is deliberately NOT readOnly: VA-API opens the render node read-write.
|
||||
#
|
||||
# `podSecurityContext.supplementalGroups` is DELIBERATELY NOT SET here, and adding it
|
||||
# would be a regression, not a hardening. MEASURED in the running container on prod-2,
|
||||
# 2026-09-30: the image's own cont-init hook
|
||||
# `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` `find`s every char device
|
||||
# under /dev/dri, `stat -c '%g'` each one's GID, `groupadd`s a matching group INSIDE the
|
||||
# container when none exists, and `usermod -a -G` the app user into it -- every start,
|
||||
# nothing hardcoded. cont-init.d completes before services.d, so the account database
|
||||
# s6 reads at exec time already carries the group. The service is then started with
|
||||
# `exec s6-setuidgid plex ...`, and `s6-setuidgid` REPLACES the process group list from
|
||||
# /etc/group rather than appending to it: `Plex Media Server` runs with
|
||||
# `Uid: 1000 Gid: 1000 Groups: 100 1000` -- root's gid 0, which PID 1 (s6-svscan) does
|
||||
# have, is GONE. Anything Kubernetes puts in supplementalGroups reaches s6-svscan and is
|
||||
# then discarded before the app ever starts. MOUNTING THE DEVICE IS SUFFICIENT; the
|
||||
# group handling is the image's job and it does it correctly. A supplementalGroups entry
|
||||
# here would be inert AND would mislead whoever debugs this next.
|
||||
hw_gpu = (s.get("hardware") or {}).get("gpu")
|
||||
if hw_gpu == "intel-quicksync":
|
||||
volume_mounts.append({"name": DRI_VOLUME_NAME, "mountPath": DRI_DEVICE_DIR})
|
||||
volumes.append(
|
||||
{
|
||||
"name": DRI_VOLUME_NAME,
|
||||
"hostPath": {"path": DRI_DEVICE_DIR, "type": "DirectoryOrCreate"},
|
||||
}
|
||||
)
|
||||
|
||||
if volume_mounts:
|
||||
container["volumeMounts"] = volume_mounts
|
||||
|
||||
@@ -1005,7 +1099,12 @@ def _oauth2_sidecar_container(s: dict) -> dict:
|
||||
|
||||
--cookie-name and --cookie-domain are DERIVED per app (see the block comment on those two
|
||||
args): every instance holds a different cookie-secret, so sharing one cookie name across the
|
||||
zone makes the instances evict each other's sessions."""
|
||||
zone makes the instances evict each other's sessions.
|
||||
|
||||
--api-route is DECLARED per app, from auth.apiRoutes, and is the one flag here that is
|
||||
neither a constant nor derivable: it names the paths THIS app serves as a machine API, and
|
||||
those differ per app (radarr `/signalr/`, jackett `/api/`, pihole none). An app that declares
|
||||
none renders no --api-route at all."""
|
||||
route = s["ingressRoutes"][0]
|
||||
host = route["host"]
|
||||
upstream_port = route["port"]
|
||||
@@ -1025,6 +1124,27 @@ def _oauth2_sidecar_container(s: dict) -> dict:
|
||||
args += [
|
||||
"--skip-provider-button=true",
|
||||
f"--redirect-url=https://{host}/oauth2/callback",
|
||||
]
|
||||
# ── API ROUTES: 401 instead of 302, ON THIS APP'S OWN PATHS ONLY ───────────────────────
|
||||
# One flag per declared route, in declared order. An app that declares NONE renders NO
|
||||
# --api-route, which is the correct and common case — see OAUTH2_API_ROUTE_PATTERN above for
|
||||
# the measurements, for why this cannot be a renderer constant, and for the breadth ratchet.
|
||||
# Validated here as well as in the schema so a breadth mistake is caught by render-golden
|
||||
# even on a run where the schema stage did not execute.
|
||||
for route in s["auth"].get("apiRoutes", []):
|
||||
if not OAUTH2_API_ROUTE_PATTERN.match(str(route)):
|
||||
raise SystemExit(
|
||||
f"render.py: {s['name']}: auth.apiRoutes entry {route!r} is not an anchored path "
|
||||
f"regex. Each entry must start with '^/' and continue with literal path "
|
||||
f"characters (e.g. '^/api/', '^/signalr/', '^/api/v3/.*'). Rejected shapes: '^/' "
|
||||
f"and '^/.*' (match every page load), an unanchored '/api/' (matches mid-path), "
|
||||
f"and alternation (the flag is repeatable — declare one entry per route). A "
|
||||
f"too-broad --api-route answers a PAGE LOAD with 401, so a logged-out operator "
|
||||
f"sees a bare error page and is never offered a sign-in: the service looks "
|
||||
f"permanently broken rather than merely unauthenticated."
|
||||
)
|
||||
args.append(f"--api-route={route}")
|
||||
args += [
|
||||
# ── COOKIE ISOLATION (fleet fix 2026-09-30; crafty's 2026-09-29 fix, generalized) ──────
|
||||
# EVERY oauth2-proxy on this platform — this sidecar, its four siblings, and the SHARED
|
||||
# forward-auth proxy (platform/oauth2-proxy) — is an INDEPENDENT instance holding its OWN
|
||||
@@ -2530,6 +2650,83 @@ def _selftest() -> int:
|
||||
"an SD without podSecurityContext must not gain a pod securityContext",
|
||||
)
|
||||
|
||||
# 6b) hardware.gpu: intel-quicksync -> a /dev/dri hostPath volume and a read-write mount
|
||||
# on the app container, and NOTHING ELSE. The SD declares the CAPABILITY; these
|
||||
# assertions pin the MECHANISM that currently satisfies it -- including what it
|
||||
# deliberately does NOT emit -- so a change to it is a deliberate, visible edit.
|
||||
hw_docs = render(
|
||||
{
|
||||
"name": "hw",
|
||||
"source": {"image": "a:1"},
|
||||
"hardware": {"gpu": "intel-quicksync"},
|
||||
}
|
||||
)
|
||||
hw_pod = next(
|
||||
d[1] for d in hw_docs if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
{
|
||||
"name": "dri",
|
||||
"hostPath": {"path": "/dev/dri", "type": "DirectoryOrCreate"},
|
||||
}
|
||||
in hw_pod.get("volumes", []),
|
||||
"hardware.gpu=intel-quicksync must render a /dev/dri hostPath volume "
|
||||
"with type DirectoryOrCreate (a GPU-less node must degrade, not fail the mount)",
|
||||
)
|
||||
hw_mounts = hw_pod["containers"][0].get("volumeMounts", [])
|
||||
check(
|
||||
{"name": "dri", "mountPath": "/dev/dri"} in hw_mounts,
|
||||
"hardware.gpu=intel-quicksync must mount /dev/dri READ-WRITE on the app container",
|
||||
)
|
||||
# The device mount is the WHOLE feature: no pod securityContext is invented. The image's
|
||||
# own cont-init hook adds the app user to the device's group inside the container, and
|
||||
# s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway (see the
|
||||
# emission site). Asserting the absence is what stops someone adding an inert one back.
|
||||
check(
|
||||
"securityContext" not in hw_pod,
|
||||
"hardware.gpu must NOT invent a pod securityContext (supplementalGroups is inert "
|
||||
"under s6-setuidgid; the image's cont-init hook handles the device group)",
|
||||
)
|
||||
# Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce)
|
||||
# rejects `privileged: true` in an app namespace, and placement is a separate concern
|
||||
# with its own justification -- this field is about the DEVICE only.
|
||||
check(
|
||||
"privileged" not in hw_pod.get("securityContext", {})
|
||||
and "securityContext" not in hw_pod["containers"][0]
|
||||
and "nodeSelector" not in hw_pod,
|
||||
"hardware.gpu must emit no privileged securityContext and no nodeSelector",
|
||||
)
|
||||
# The regression that matters: absence renders NOTHING. Mirrors the podSecurityContext
|
||||
# negative above -- an SD that does not opt in must be byte-identical to before.
|
||||
nohw_pod = next(
|
||||
d[1] for d in render({"name": "nohw", "source": {"image": "a:1"}})
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
"volumes" not in nohw_pod
|
||||
and "volumeMounts" not in nohw_pod["containers"][0]
|
||||
and "securityContext" not in nohw_pod,
|
||||
"an SD without `hardware` must gain no dri volume, no mount and no securityContext",
|
||||
)
|
||||
# An SD that declares BOTH keeps its own podSecurityContext exactly as written -- the
|
||||
# hardware field neither adds to it nor rewrites it.
|
||||
hwm_pod = next(
|
||||
d[1]
|
||||
for d in render(
|
||||
{
|
||||
"name": "hwm",
|
||||
"source": {"image": "a:1"},
|
||||
"hardware": {"gpu": "intel-quicksync"},
|
||||
"podSecurityContext": {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
}
|
||||
)
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
"hardware.gpu must leave the SD's own podSecurityContext untouched",
|
||||
)
|
||||
|
||||
# 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that
|
||||
# fronts the Gateway path (fail-closed), while oidc/native/none apps are unchanged.
|
||||
fa_docs = render(
|
||||
@@ -2540,15 +2737,27 @@ def _selftest() -> int:
|
||||
"auth": {"mode": "forward-auth", "groups": ["media-users"]},
|
||||
}
|
||||
)
|
||||
fa_dep = next(d[1] for d in fa_docs if d[1]["kind"] == "Deployment")
|
||||
fa_containers = fa_dep["spec"]["template"]["spec"]["containers"]
|
||||
fa_dep = next((d[1] for d in fa_docs if d[1]["kind"] == "Deployment"), None)
|
||||
check(fa_dep is not None,
|
||||
"a forward-auth app rendered NO Deployment, so there is no pod to gate at all")
|
||||
fa_containers = (((fa_dep or {}).get("spec") or {}).get("template") or {}) \
|
||||
.get("spec", {}).get("containers") or []
|
||||
check(
|
||||
len(fa_containers) == 2
|
||||
and fa_containers[0]["name"] == "fa"
|
||||
and fa_containers[1]["name"] == "oauth2-proxy",
|
||||
"forward-auth app must render the app container FIRST + an oauth2-proxy sidecar SECOND",
|
||||
f"forward-auth app must render the app container FIRST + an oauth2-proxy sidecar SECOND; "
|
||||
f"rendered {[c.get('name') for c in fa_containers]!r}",
|
||||
)
|
||||
side = fa_containers[1]
|
||||
# EVERY assertion below reads `side`, so an absent sidecar must NOT be allowed to raise here.
|
||||
# The whole point of the None-tolerant lookups in this block is that a missing sidecar (or a
|
||||
# missing flag on it) reports as NAMED ASSERTIONS rather than as a traceback: an exception
|
||||
# escaping _selftest() exits non-zero with the right verdict and ZERO diagnostics, and it
|
||||
# hides every other assertion the same mutation broke. Measured by Desmond Oyelaran while
|
||||
# wiring this selftest into the render-golden stage: deleting the --cookie-name emission gave
|
||||
# rc 1 via an uncaught StopIteration and printed not one `SELFTEST FAIL:` line. Provoked
|
||||
# again here with the sidecar removed entirely, which raised IndexError at this line.
|
||||
side = fa_containers[1] if len(fa_containers) > 1 else {"args": [], "env": [], "ports": []}
|
||||
check(
|
||||
"--upstream=http://127.0.0.1:9117" in side["args"],
|
||||
"sidecar upstream must be the co-located app UI over http on 127.0.0.1:<uiport>",
|
||||
@@ -2617,12 +2826,101 @@ def _selftest() -> int:
|
||||
"sidecar must declare BOTH the http and metrics container ports — an undeclared port "
|
||||
"cannot be named by a Service port or a PodMonitor, so the flag alone is inert",
|
||||
)
|
||||
# API ROUTES (schema 1.9.0). Three properties, and the FIRST is the one that keeps the field
|
||||
# honest: an app that declares nothing must render NO --api-route. `fa` above declares none.
|
||||
check(
|
||||
not any(a.startswith("--api-route") for a in side["args"]),
|
||||
"a forward-auth app that declares no auth.apiRoutes must render NO --api-route. A "
|
||||
"default or a renderer constant would answer PiHole's page loads with 401 — PiHole "
|
||||
"serves no /api/ at all — so the service would look permanently broken to a "
|
||||
"logged-out operator instead of signing them in",
|
||||
)
|
||||
# Same `next(..., None)` discipline as _cookie_name() below, for the same reason: if the
|
||||
# sidecar stops being rendered at all, this must report as a named assertion rather than as a
|
||||
# StopIteration that swallows every other SELFTEST FAIL line in the run.
|
||||
_faapi_dep = next(
|
||||
(
|
||||
d[1]
|
||||
for d in render(
|
||||
{
|
||||
"name": "faapi",
|
||||
"source": {"image": "a:1"},
|
||||
"ingressRoutes": [{"host": "faapi.home.keithjackson.dev", "port": 9119}],
|
||||
"auth": {
|
||||
"mode": "forward-auth",
|
||||
"groups": ["media-users"],
|
||||
"apiRoutes": ["^/api/", "^/signalr/"],
|
||||
},
|
||||
}
|
||||
)
|
||||
if d[1]["kind"] == "Deployment"
|
||||
),
|
||||
None,
|
||||
)
|
||||
fa_api_side = next(
|
||||
(
|
||||
c
|
||||
for c in ((_faapi_dep or {}).get("spec", {}).get("template", {})
|
||||
.get("spec", {}).get("containers") or [])
|
||||
if c.get("name") == "oauth2-proxy"
|
||||
),
|
||||
None,
|
||||
)
|
||||
check(
|
||||
fa_api_side is not None,
|
||||
"a forward-auth app declaring auth.apiRoutes rendered NO oauth2-proxy sidecar, so the "
|
||||
"routes have nowhere to land and the app is not gated at all (R-IDP-02)",
|
||||
)
|
||||
check(
|
||||
fa_api_side is not None
|
||||
and [a for a in fa_api_side["args"] if a.startswith("--api-route")]
|
||||
== ["--api-route=^/api/", "--api-route=^/signalr/"],
|
||||
"a declared auth.apiRoutes must render as one repeatable --api-route per entry, in "
|
||||
"declared order, and NOTHING else. Note the flag is SINGULAR: --api-routes does not "
|
||||
"exist in v7.8.1 and the proxy refuses to start with it",
|
||||
)
|
||||
# BREADTH. The renderer must REFUSE a regex that would match a page load, not merely
|
||||
# discourage one, because the schema stage can be skipped and render-golden cannot.
|
||||
for _bad in ("^/", "^/.*", ".*", "/api/", "^/api/|^/"):
|
||||
try:
|
||||
render(
|
||||
{
|
||||
"name": "fabad",
|
||||
"source": {"image": "a:1"},
|
||||
"ingressRoutes": [{"host": "fabad.home.keithjackson.dev", "port": 9120}],
|
||||
"auth": {
|
||||
"mode": "forward-auth",
|
||||
"groups": ["media-users"],
|
||||
"apiRoutes": [_bad],
|
||||
},
|
||||
}
|
||||
)
|
||||
_refused = False
|
||||
except SystemExit:
|
||||
_refused = True
|
||||
check(
|
||||
_refused,
|
||||
f"the renderer ACCEPTED auth.apiRoutes entry {_bad!r}. A too-broad --api-route "
|
||||
f"answers a plain page load with 401, so a logged-out operator sees a bare error "
|
||||
f"page and is never offered a sign-in — the service looks permanently broken "
|
||||
f"rather than merely unauthenticated, which is a worse failure than the 302 bug "
|
||||
f"this flag exists to fix",
|
||||
)
|
||||
for _good in ("^/api/", "^/signalr/", "^/api/v3/.*", "^/api/v2.0/server/update$"):
|
||||
check(
|
||||
bool(OAUTH2_API_ROUTE_PATTERN.match(_good)),
|
||||
f"the breadth ratchet rejected the legitimate route {_good!r}. A ratchet that "
|
||||
f"blocks real API prefixes is a ratchet someone eventually deletes",
|
||||
)
|
||||
|
||||
# Two forward-auth apps must not be able to collide. Renders a SECOND one and compares, so a
|
||||
# future refactor to a constant (or to a per-service table with a copy-paste slip) fails here
|
||||
# rather than silently in a browser six weeks later.
|
||||
fa2_side = next(
|
||||
c
|
||||
for c in next(
|
||||
# Same None-tolerant shape as the `faapi` fixture above and for the same reason: a missing
|
||||
# sidecar must surface as a named assertion, never as a traceback that suppresses the
|
||||
# collected SELFTEST FAIL lines.
|
||||
_fa2_dep = next(
|
||||
(
|
||||
d[1]
|
||||
for d in render(
|
||||
{
|
||||
@@ -2633,24 +2931,55 @@ def _selftest() -> int:
|
||||
}
|
||||
)
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]["containers"]
|
||||
if c["name"] == "oauth2-proxy"
|
||||
),
|
||||
None,
|
||||
)
|
||||
fa2_side = next(
|
||||
(
|
||||
c
|
||||
for c in ((_fa2_dep or {}).get("spec", {}).get("template", {})
|
||||
.get("spec", {}).get("containers") or [])
|
||||
if c.get("name") == "oauth2-proxy"
|
||||
),
|
||||
{"args": []},
|
||||
)
|
||||
|
||||
# `next(..., None)` — the DEFAULT IS LOAD-BEARING, not defensive style. With a bare
|
||||
# `next(genexpr)` this raises StopIteration the moment a sidecar renders no --cookie-name,
|
||||
# which is precisely the mutation this assertion exists to catch. The traceback then escapes
|
||||
# _selftest() before its collected `SELFTEST FAIL:` lines are printed, so the run exits
|
||||
# non-zero with the RIGHT VERDICT AND NO DIAGNOSTICS — and it masks every other assertion the
|
||||
# same mutation broke. Measured by Desmond Oyelaran while wiring the selftest into the
|
||||
# render-golden stage: deleting the --cookie-name emission gave rc 1 via an uncaught
|
||||
# StopIteration and suppressed all of the messages. A missing flag must report AS AN
|
||||
# ASSERTION, with its name in the output, which is what the None + explicit check below do.
|
||||
def _cookie_name(container):
|
||||
return next(a for a in container["args"] if a.startswith("--cookie-name="))
|
||||
return next((a for a in container["args"] if a.startswith("--cookie-name=")), None)
|
||||
|
||||
_fa_cn, _fa2_cn = _cookie_name(side), _cookie_name(fa2_side)
|
||||
check(
|
||||
_cookie_name(side) != _cookie_name(fa2_side),
|
||||
_fa_cn is not None and _fa2_cn is not None,
|
||||
f"a forward-auth sidecar rendered NO --cookie-name at all (fa={_fa_cn!r}, "
|
||||
f"fa2={_fa2_cn!r}). Omitting the flag is not neutral: oauth2-proxy then claims its "
|
||||
f"built-in default `_oauth2_proxy`, which every other instance on the platform also "
|
||||
f"claims, so they overwrite and ClearSession-delete each other's sessions",
|
||||
)
|
||||
check(
|
||||
_fa_cn != _fa2_cn,
|
||||
"two forward-auth apps rendered IDENTICAL cookie names — they would overwrite each "
|
||||
"other's session on every sign-in (the 2026-09-29 crafty defect, fleet-wide)",
|
||||
)
|
||||
# Index-safe for the same reason as `side` above: an absent sidecar has no env, and an
|
||||
# IndexError here would exit non-zero with no diagnostics at all.
|
||||
_fa_env = (side.get("env") or [{}])[0]
|
||||
check(
|
||||
side["env"][0]["valueFrom"]["secretKeyRef"]["name"] == "fa-oauth2-proxy",
|
||||
(((_fa_env.get("valueFrom") or {}).get("secretKeyRef") or {}).get("name")
|
||||
== "fa-oauth2-proxy"),
|
||||
"sidecar creds must come by name from the per-app <app>-oauth2-proxy Secret (R-IDP-04)",
|
||||
)
|
||||
fa_svc = next(d[1] for d in fa_docs if d[1]["kind"] == "Service")
|
||||
fa_svc_ports = {p["port"] for p in fa_svc["spec"]["ports"]}
|
||||
fa_svc = next((d[1] for d in fa_docs if d[1]["kind"] == "Service"), None)
|
||||
check(fa_svc is not None, "a forward-auth app must render a Service for the Gateway to reach")
|
||||
fa_svc_ports = {p["port"] for p in (((fa_svc or {}).get("spec") or {}).get("ports") or [])}
|
||||
check(
|
||||
fa_svc_ports == {9117, 4180},
|
||||
"forward-auth Service must expose the app port (east-west) AND :4180 (gateway->sidecar)",
|
||||
|
||||
@@ -112,6 +112,14 @@ auth:
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
# NO `apiRoutes` HERE, AND NOT BECAUSE IT NEEDS NONE — because nothing measures that it does.
|
||||
# Sonarr is nearly the same codebase as Radarr, which is exactly what makes `^/api/` +
|
||||
# `^/signalr/` a tempting guess and still a guess. MEASURED 2026-10-01 on the live pod
|
||||
# (sonarr-ddf485775-8ckwt, 0 restarts, 23h): its oauth2-proxy log contains 13,643 lines and
|
||||
# ZERO unauthenticated requests — no /api/, no /signalr/, nothing but health probes. Nobody
|
||||
# has opened this UI since the cookie fix rolled, so the measurement is empty rather than
|
||||
# negative: it is evidence of no TRAFFIC, not evidence of no need. Add the routes when someone
|
||||
# reproduces the 302-on-XHR here, with the numbers.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=4.0.0 <5.0.0" # stay on the Sonarr v4 stable line
|
||||
|
||||
+1
-54
@@ -1,10 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# ci/render-golden.sh — Stage: the renderer is trustworthy.
|
||||
# (1) catalog/render.py passes its OWN output-contract self-test, and
|
||||
# (2) every committed apps/ golden reproduces byte-for-byte from the canonical render.
|
||||
# Two independent assertions about one subject, catalog/render.py. (1) was added
|
||||
# 2026-10-01 because nothing in ci/ or .gitea/ had ever invoked `--selftest`; see the
|
||||
# block above the invocation for why it lives here and not in a conformance check.
|
||||
# ci/render-golden.sh — Stage: render==golden reproducibility gate.
|
||||
#
|
||||
# WHY: the committed apps/ tree is a set of GOLDENS produced by ONE canonical
|
||||
# invocation of catalog/render.py. If a golden is hand-edited, or a catalog
|
||||
@@ -59,54 +54,6 @@ if [ ! -f "$RENDER_PY" ]; then
|
||||
summary "- render-golden: **SKIPPED** (renderer missing)"
|
||||
exit 3
|
||||
fi
|
||||
# --- The renderer's OWN self-test, before anything is compared ---------------
|
||||
# `catalog/render.py --selftest` is ~106 executed assertions on the renderer's output
|
||||
# contract (default-deny covers both directions, the Kyverno namespace hook label, a valid
|
||||
# auth-mode, the oauth2-proxy sidecar's cookie-name/-domain/-refresh flags, the
|
||||
# placeholder-substitution guard, the observability refusals...). Until now NOTHING in ci/
|
||||
# or .gitea/ invoked it — measured by grep on 2026-09-30 and recorded as the top follow-up
|
||||
# of decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md — so every one of those
|
||||
# assertions was dev-time only and a merge gate could not tell a sound renderer from a
|
||||
# gutted one.
|
||||
#
|
||||
# WHY HERE, AND NOT IN A CONFORMANCE CHECK. The lead was
|
||||
# conformance/tests/l0-static/auditability-proofs-producible/check.sh, which drives
|
||||
# `conformance/lib/{record_run,status}.py --selftest` "because nothing else drives them".
|
||||
# That loop is NOT the right hook: that check serves R-SEC-03 and its subject is the
|
||||
# EVIDENCE CHAIN (can a forged or stale run become a proof?). The renderer is not part of
|
||||
# that chain, and hanging it there would make the check's name and its requirement mapping
|
||||
# a lie. This stage's subject already IS catalog/render.py — it exists to assert that the
|
||||
# renderer's output is trustworthy — so the renderer's own contract test belongs in it.
|
||||
#
|
||||
# It runs BEFORE the catalog/empty-catalog exit-3 below on purpose: the self-test is
|
||||
# deterministic and catalog-independent (it renders synthetic ServiceDefinitions in
|
||||
# memory), so an empty catalog must not be able to skip it.
|
||||
#
|
||||
# render-golden and the self-test are INDEPENDENT assertions, which is why both are needed:
|
||||
# render-golden catches golden-vs-renderer drift but is satisfied by ANY renderer whose
|
||||
# output was re-rendered, including one that just lost a contract invariant and had all ten
|
||||
# goldens regenerated in the same commit. The self-test catches exactly that.
|
||||
#
|
||||
# FAILURE IS A HARD FAIL, NOT A SKIP: the renderer is source in this repo, so "could not
|
||||
# run its self-test" has no legitimate environmental cause once python3 and render.py are
|
||||
# both present (both asserted above). ci/gate.sh maps this stage's rc 1 to BLOCKED.
|
||||
log "Stage: catalog/render.py --selftest (the renderer's own output-contract assertions)"
|
||||
selftest_out="$(mktemp)"
|
||||
if "$PYTHON" "$RENDER_PY" --selftest >"$selftest_out" 2>&1; then
|
||||
ok "render.py --selftest passed"
|
||||
summary "- render-selftest: **passed** (\`catalog/render.py --selftest\`)"
|
||||
else
|
||||
err "catalog/render.py --selftest FAILED — the renderer violates its own output contract."
|
||||
err " Every golden under apps/ is produced by this renderer, so the goldens below cannot"
|
||||
err " be trusted even if they reproduce byte-for-byte. Reproduce with:"
|
||||
err " ${PYTHON} catalog/render.py --selftest"
|
||||
sed 's/^/ /' "$selftest_out" >&2 || true
|
||||
summary "- render-selftest: **FAILED** (\`catalog/render.py --selftest\` — renderer contract violated)"
|
||||
rm -f "$selftest_out"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$selftest_out"
|
||||
|
||||
if [ ! -d "$CATALOG_DIR" ] || ! compgen -G "${CATALOG_DIR}/*.yaml" >/dev/null 2>&1; then
|
||||
skip "no catalog ServiceDefinitions under '${CATALOG_DIR}' — nothing to render."
|
||||
summary "- render-golden: **SKIPPED** (empty catalog)"
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — `--api-route` is PER APP, ANCHORED, and DECLARED
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# THE DEFECT THIS CLOSES. oauth2-proxy answers an unauthenticated request with a 302 to
|
||||
# Keycloak. Its built-in AJAX guard would send a 401 instead, but it fires only on an EXACT
|
||||
# `Accept: application/json`, which these apps' JavaScript does not consistently send. So a
|
||||
# browser XHR gets a 302, FOLLOWS IT INVISIBLY, has it satisfied silently by the live Keycloak
|
||||
# SSO session, and resolves 200 carrying the app's HTML where the JavaScript expected JSON — no
|
||||
# prompt, no console error, nothing logged as a failure. On a WRITE it is worse: a browser
|
||||
# following a 302 re-issues a POST as a GET and DROPS THE BODY, so the operator's change is
|
||||
# discarded and the UI reports success.
|
||||
#
|
||||
# MEASURED, twice, on two different apps and two different paths:
|
||||
# jackett 2026-09-30 — on /api/ paths: 33 x 302 against 2 x 401, and 4 of 6 POSTs answered BY
|
||||
# THE PROXY, never reaching Jackett (access-log upstream field `-`, not `/`). That is
|
||||
# the operator's whole report: "GET works, most PUT/POST to add indexers fail,
|
||||
# reloading never fixes it, and I am never re-prompted for login".
|
||||
# radarr 2026-10-01, 23h after the cookie fix rolled (pod radarr-7d55cbffb9-42hmh, 0
|
||||
# restarts) — 232 unauthenticated requests in the pod's whole lifetime and ALL 232 are
|
||||
# the same call: `POST /signalr/messages/negotiate` from a browser UA, 302, upstream
|
||||
# `-`. A left-open tab whose session the cookie rename orphaned, polling SignalR
|
||||
# forever, silently redirected every time, NEVER re-prompting. The reported symptom,
|
||||
# still running in production, on a service nobody was watching.
|
||||
#
|
||||
# WHY IT CANNOT BE A RENDERER CONSTANT — AND WHY THAT NEEDS A GATE. The path shape belongs to
|
||||
# the APP, not to the forward-auth pattern: radarr's broken path is `/signalr/`, jackett's is
|
||||
# `/api/`, and PIHOLE SERVES NO `/api/` AT ALL. One function renders all five sidecars
|
||||
# (catalog/render.py::_oauth2_sidecar_container), so the cheap fix is a constant — and a
|
||||
# constant applies one app's measurement to four apps nobody measured. The gate that already
|
||||
# exists cannot see that mistake: `ci/render-golden.sh` compares the render to the goldens, and
|
||||
# a constant plus regenerated goldens AGREE WITH EACH OTHER while both being wrong. Hence
|
||||
# assertion (2) below, which compares the rendered flag against each service's OWN
|
||||
# ServiceDefinition rather than against the renderer's output.
|
||||
#
|
||||
# WHY BREADTH IS THE DANGEROUS DIRECTION, AND IT IS NOT SYMMETRIC. A MISSING --api-route is a
|
||||
# silent 302 — bad, invisible, recoverable. A TOO BROAD one answers a plain PAGE LOAD with 401:
|
||||
# a logged-out operator gets a bare error page and is NEVER OFFERED A SIGN-IN, so the service
|
||||
# looks permanently broken and no reload recovers it. That is a worse outcome than the defect
|
||||
# the flag fixes, which is why the anchoring rule is asserted on three independent surfaces
|
||||
# (schema items.pattern, the renderer, and this check) and why assertion (5) fails if they drift
|
||||
# apart — if they do, the loosest one wins silently.
|
||||
#
|
||||
# GIVEN every oauth2-proxy sidecar this repo DEPLOYS (a container in any manifest at any depth:
|
||||
# the rendered sidecars in apps/, crafty's hand-authored one in platform/), the
|
||||
# ServiceDefinitions in catalog/ that generate five of them, and the renderer itself,
|
||||
# WHEN each sidecar's `--api-route` values are collected (the flag is REPEATABLE, so all of
|
||||
# them, in order) and each onboarded forward-auth service is paired with its golden,
|
||||
# THEN (1) every rendered --api-route is anchored at `^/` and continues with literal path
|
||||
# characters — `^/`, `^/.*`, `.*`, an unanchored `/api/` and any alternation FAIL;
|
||||
# (2) every onboarded forward-auth service's rendered route list EQUALS its own
|
||||
# ServiceDefinition's `auth.apiRoutes`, in order — which fails both a hand-edited
|
||||
# golden and a renderer constant, neither of which render-golden can see;
|
||||
# (3) BOTH cases are live in this tree: at least one service declares routes and at least
|
||||
# one declares none and renders none (the pihole case), so neither path is an
|
||||
# unexercised seam;
|
||||
# (4) `_oauth2_sidecar_container()` reads `auth.apiRoutes` and contains NO literal path
|
||||
# regex — a constant is the defect, back in the one place that reintroduces it
|
||||
# fleet-wide in a single edit;
|
||||
# (5) the anchoring pattern in the renderer and in the schema are the SAME pattern this
|
||||
# check enforces;
|
||||
# (6) the SHIPPED renderer, run as a subprocess on synthetic ServiceDefinitions, emits
|
||||
# exactly the declared routes in order, emits NOTHING for a service declaring none or
|
||||
# an empty list, and REFUSES all five breadth shapes.
|
||||
#
|
||||
# WHY ALSO A SCHEMA PATTERN, GIVEN THIS CHECK. The schema is the only surface that rejects a
|
||||
# too-broad regex WHERE A HUMAN TYPES IT — `ci/l0.sh` validates `catalog/*.yaml` against it on
|
||||
# every gate run, so the operator is told at the authoring surface rather than finding out from
|
||||
# a 401 on a page load. The renderer's own guard is there because `render-golden` runs even on a
|
||||
# gate invocation where the schema stage skipped. This check is there because neither of those
|
||||
# can see a renderer constant. Three surfaces, three different blind spots.
|
||||
#
|
||||
# WHY THIS IS A SIBLING AND NOT PART OF oauth2-proxy-cookie-names-distinct. That check's subject
|
||||
# is cookie IDENTITY: its inputs are names and domains, its failure mode is mutual session
|
||||
# eviction, its register is the shared proxy's claim on the default name. `--api-route` shares
|
||||
# none of those. Bundling them would make that check's name a lie and couple two unrelated
|
||||
# ratchets, so a future api-route decision could only be expressed by weakening a COOKIE check —
|
||||
# the same discipline that check's own header argues for, applied to this change.
|
||||
#
|
||||
# WHAT THIS DOES NOT ESTABLISH — read with the handoff entry, not instead of it:
|
||||
# - That a 401 FIXES anything in a browser. No browser round-trip has been made, by anyone,
|
||||
# for this flag. The 302/401 split, the dropped POST bodies and the orphaned SignalR poll
|
||||
# are all read from access logs; the repair is reasoned from HTTP semantics.
|
||||
# - That the regexes are RIGHT for these two apps beyond the measured paths. `^/api/` on
|
||||
# radarr is justified by a measurement of radarr taken on its PRE-FIX pod by another
|
||||
# engineer (decisions/2026-09-30-jackett-oauth2-proxy-session.md: 62 x /api/v3/command,
|
||||
# 46 x /api/v3/queue/details); the current pod has logged zero /api/ traffic.
|
||||
# - Anything about sonarr, qbittorrent or pihole. Their logs were read (2026-10-01) and
|
||||
# contain ZERO unauthenticated requests — no traffic, which is not the same as no need.
|
||||
# Assertion (3) therefore proves the "declares none" PATH works; it does not claim those
|
||||
# three apps are correctly configured.
|
||||
# - That the LIVE pods carry what the tree says. git is not the cluster (CLAUDE.md §2b).
|
||||
# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: oauth2-proxy-api-routes-per-app
|
||||
labels:
|
||||
conformance.home.k3s/R-IDP-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-IDP-02"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE — the subject spans apps/wave-0, apps/wave-1, platform/crafty, every
|
||||
# ServiceDefinition in catalog/, the renderer that generates five of the sidecars, and the
|
||||
# ServiceDefinition schema. No single requirement path-maps to that set, and the renderer
|
||||
# maps to no identity requirement at all, so a requirement-scoped selection would skip this
|
||||
# check on precisely the change that can reintroduce the defect fleet-wide in one edit.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
An unauthenticated XHR is answered 401, not 302, on exactly the paths each app was
|
||||
MEASURED to serve as a machine API: every rendered --api-route is anchored at ^/ and
|
||||
continues with literal path characters, every onboarded forward-auth service's rendered
|
||||
routes equal its own auth.apiRoutes in order (so neither a hand-edited golden nor a
|
||||
renderer constant can pass), an app declaring none renders none, the anchoring pattern is
|
||||
the same in the schema and the renderer, and the shipped renderer is proven on synthetic
|
||||
ServiceDefinitions in both directions — including refusing the five breadth shapes that
|
||||
would answer a page load with 401.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-api-route-is-per-app-anchored-and-declared
|
||||
description: >-
|
||||
Inventory every oauth2-proxy sidecar's repeatable --api-route values, assert each is an
|
||||
anchored path regex, cross-check each onboarded forward-auth service's rendered routes
|
||||
against its own ServiceDefinition, require both the declares-routes and the
|
||||
declares-none cases to be live in the tree, assert the renderer derives from
|
||||
auth.apiRoutes and hardcodes no path regex, assert the schema and renderer share this
|
||||
check's anchoring pattern, and prove the shipped renderer on synthetic
|
||||
ServiceDefinitions in both directions.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 STATIC CHECK — `--api-route` is PER APP, ANCHORED, and DECLARED.
|
||||
# See chainsaw-test.yaml's header for WHY this exists and what it measured. In one paragraph:
|
||||
#
|
||||
# oauth2-proxy answers an unauthenticated request with a 302 to Keycloak. A browser XHR follows
|
||||
# that redirect invisibly, the live SSO session satisfies it silently, and the call resolves 200
|
||||
# carrying the app's HTML where its JavaScript expected JSON — and a POST following a 302 is
|
||||
# re-issued as a GET with its BODY DROPPED. `--api-route=<regex>` makes matching paths answer
|
||||
# 401 instead. The flag is right for jackett (`/api/`) and radarr (`/signalr/`, `/api/`) and
|
||||
# WRONG for pihole, which serves no `/api/` at all — so it cannot be a renderer constant, and
|
||||
# the point of this check is that it never becomes one again.
|
||||
#
|
||||
# The breadth direction is the dangerous one and it is NOT symmetric with the bug. A missing
|
||||
# `--api-route` is a silent 302; a TOO BROAD one answers a plain PAGE LOAD with 401, so a
|
||||
# logged-out operator gets a bare error page and is never offered a sign-in — the service looks
|
||||
# permanently broken. That is a worse outcome than the defect, which is why the anchoring rule
|
||||
# is asserted here as well as in the schema.
|
||||
#
|
||||
# Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, which the runner turns
|
||||
# into rc 3 = UNWIRED, never green, because this test declares the whole-repo scope) only if the
|
||||
# tree contains no forward-auth oauth2-proxy sidecar at all — genuinely nothing to audit.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
# Honest SKIP (never green for a scope-declaring test) if the forward-auth pattern has left the
|
||||
# repo. Anchored on `image:` for the same reason as the sibling cookie check: a bare
|
||||
# `oauth2-proxy/oauth2-proxy:` also matches platform/oauth2-proxy/forward-auth-cec.yaml's Envoy
|
||||
# cluster reference (`serviceName: oauth2-proxy/oauth2-proxy:80`), which runs no proxy.
|
||||
if ! grep -rlE 'image:[[:space:]]*[^[:space:]]*oauth2-proxy' \
|
||||
"${ROOT}/apps" "${ROOT}/platform" >/dev/null 2>&1; then
|
||||
echo "check SKIPPED: no oauth2-proxy container image found under apps/ or platform/ — the forward-auth sidecar pattern is gone and there is nothing to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
"""Static audit: --api-route is per-app, anchored, and declared in the ServiceDefinition.
|
||||
|
||||
PyYAML (the same dependency as catalog/render.py and the sibling l0-static checks) rather than
|
||||
line-based parsing: a container's args live in a nested structure, and a line-based reader of
|
||||
one is precisely the approximate parser that silently stops matching.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree
|
||||
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
# THE BREADTH RATCHET, stated once and asserted against three independent surfaces below (the
|
||||
# rendered tree, the authored catalog, the shipped renderer). Each entry must be anchored at
|
||||
# `^/` and continue with LITERAL path characters. This rejects:
|
||||
# `^/` and `^/.*` — match every page load, so the UI answers 401 and never signs in
|
||||
# `/api/` — unanchored; oauth2-proxy would match it mid-path
|
||||
# `.*` — everything
|
||||
# `^/api/|^/` — alternation smuggles breadth past a prefix-only reading
|
||||
# Alternation is REFUSED rather than supported because `--api-route` is repeatable: one entry
|
||||
# per route costs nothing and each entry stays readable. Kept byte-identical to
|
||||
# catalog/render.py's OAUTH2_API_ROUTE_PATTERN and to the schema's items.pattern; assertion (5)
|
||||
# below fails if they drift apart.
|
||||
ANCHORED = re.compile(r"^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$")
|
||||
SCHEMA_PATTERN = r"^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$"
|
||||
|
||||
|
||||
def read(p: Path) -> str:
|
||||
try:
|
||||
return p.read_text()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return ""
|
||||
|
||||
|
||||
def is_test_fixture(p: Path) -> bool:
|
||||
"""conformance/tests/**/{golden,fixtures}/** is assertion DATA, not deployed manifests."""
|
||||
parts = p.parts
|
||||
return "conformance" in parts and ("golden" in parts or "fixtures" in parts)
|
||||
|
||||
|
||||
def iter_containers(node):
|
||||
"""Yield every container mapping under any `containers`/`initContainers` list, at any depth.
|
||||
|
||||
Depth-agnostic on purpose: the same container shape appears under a Deployment, a DaemonSet,
|
||||
a CronJob's jobTemplate or a bare Pod, and a reader that hardcoded one path would silently
|
||||
stop seeing an instance the day one moved.
|
||||
"""
|
||||
if isinstance(node, dict):
|
||||
for key, val in node.items():
|
||||
if key in ("containers", "initContainers") and isinstance(val, list):
|
||||
for c in val:
|
||||
if isinstance(c, dict):
|
||||
yield c
|
||||
else:
|
||||
yield from iter_containers(val)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
yield from iter_containers(item)
|
||||
|
||||
|
||||
def api_routes_of(args: list) -> list:
|
||||
"""Every `--api-route=<v>` / `--api-route <v>` value, in order. The flag is REPEATABLE, so
|
||||
this must collect ALL of them — a single-value reader would audit the first and wave the
|
||||
rest through, which is the shape of reader bug this file's header warns about."""
|
||||
out = []
|
||||
for i, a in enumerate(args):
|
||||
if not isinstance(a, str):
|
||||
continue
|
||||
if a.startswith("--api-route="):
|
||||
out.append(a.split("=", 1)[1])
|
||||
elif a == "--api-route" and i + 1 < len(args) and isinstance(args[i + 1], str):
|
||||
out.append(args[i + 1])
|
||||
return out
|
||||
|
||||
|
||||
# ============================================================ (0) INVENTORY: THE RENDERED TREE
|
||||
# Every oauth2-proxy container this repo DEPLOYS, keyed by file + owner, with its api-routes.
|
||||
rendered: dict[str, list] = {}
|
||||
for p in sorted(ROOT.rglob("*.yaml")):
|
||||
relparts = p.relative_to(ROOT).parent.parts
|
||||
# NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such
|
||||
# as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs,
|
||||
# so a bare rglob audits up to 18 duplicates of every file — which both invents FAILs and,
|
||||
# worse, can satisfy a "something declares X" check from a copy rather than from real source.
|
||||
if any(part.startswith(".") for part in relparts) or is_test_fixture(p) or ".sops." in p.name:
|
||||
continue
|
||||
txt = read(p)
|
||||
if "oauth2-proxy" not in txt:
|
||||
continue
|
||||
rel = str(p.relative_to(ROOT))
|
||||
try:
|
||||
docs = [d for d in yaml.safe_load_all(txt) if isinstance(d, dict)]
|
||||
except yaml.YAMLError as exc:
|
||||
check(False, f"{rel} is not parseable YAML ({exc}) — any oauth2-proxy instance in it "
|
||||
f"went unaudited, so a too-broad --api-route could be hiding there")
|
||||
continue
|
||||
for d in docs:
|
||||
for c in iter_containers(d):
|
||||
image = str(c.get("image") or "")
|
||||
name = str(c.get("name") or "")
|
||||
if "oauth2-proxy" not in image and name != "oauth2-proxy":
|
||||
continue
|
||||
owner = (d.get("metadata") or {}).get("name") or "?"
|
||||
args = list(c.get("args") or []) + list(c.get("command") or [])
|
||||
rendered[f"{rel}:{owner}/{name or 'oauth2-proxy'}"] = api_routes_of(args)
|
||||
|
||||
check(len(rendered) >= 5,
|
||||
f"only {len(rendered)} oauth2-proxy container(s) parsed out of the tree. This repo "
|
||||
f"deploys more than that (five rendered sidecars plus crafty's hand-authored one), so the "
|
||||
f"inventory has stopped seeing instances and every assertion below would pass vacuously "
|
||||
f"while an unanchored --api-route went unaudited")
|
||||
|
||||
# ============================================================ (1) EVERY ROUTE IS ANCHORED
|
||||
for key, routes in sorted(rendered.items()):
|
||||
for r in routes:
|
||||
check(bool(ANCHORED.match(r)),
|
||||
f"{key} declares --api-route={r!r}, which is not an anchored path regex.\n"
|
||||
f" WHY THIS IS WORSE THAN THE BUG IT FIXES: --api-route makes matching paths "
|
||||
f"answer 401 instead of 302ing to Keycloak. A regex that also matches a path a "
|
||||
f"BROWSER NAVIGATES to — `^/`, `^/.*`, `.*`, or an alternation that reaches the "
|
||||
f"UI root — answers a plain PAGE LOAD with 401. A logged-out operator then sees a "
|
||||
f"bare error page and is NEVER OFFERED A SIGN-IN: the service looks permanently "
|
||||
f"broken rather than merely unauthenticated, and no reload can recover it.\n"
|
||||
f" FIX: anchor it at '^/' followed by literal path characters "
|
||||
f"(e.g. '^/api/', '^/signalr/', '^/api/v3/.*'), one entry per route — the flag is "
|
||||
f"repeatable, so alternation is never needed. For a service rendered from catalog/, "
|
||||
f"do NOT hand-edit the golden: declare auth.apiRoutes and re-render "
|
||||
f"(ci/render-golden.sh's header has the canonical command)")
|
||||
|
||||
# ============================================================ (2) DERIVED FROM THE CATALOG
|
||||
# The five rendered sidecars are OUTPUTS. Auditing only them would let someone "fix" a golden by
|
||||
# hand (reverted by the next render) or make the renderer emit a CONSTANT and regenerate every
|
||||
# golden to match — which render-golden cannot see, because render and golden would agree.
|
||||
# So: every onboarded forward-auth service's rendered route list must EQUAL its own
|
||||
# ServiceDefinition's auth.apiRoutes, in order.
|
||||
catalog_dir = ROOT / "catalog"
|
||||
declared: dict[str, list] = {}
|
||||
forward_auth: list[str] = []
|
||||
for sd_path in sorted(catalog_dir.glob("*.yaml")):
|
||||
try:
|
||||
sd = yaml.safe_load(read(sd_path))
|
||||
except yaml.YAMLError as exc:
|
||||
check(False, f"catalog/{sd_path.name} is not parseable YAML ({exc})")
|
||||
continue
|
||||
if not isinstance(sd, dict) or not sd.get("name"):
|
||||
continue
|
||||
auth = sd.get("auth") or {}
|
||||
routes = auth.get("apiRoutes") or []
|
||||
declared[sd["name"]] = list(routes)
|
||||
if auth.get("mode") == "forward-auth" and (sd.get("ingressRoutes") or []):
|
||||
forward_auth.append(sd["name"])
|
||||
for r in routes:
|
||||
check(bool(ANCHORED.match(str(r))),
|
||||
f"catalog/{sd_path.name}: auth.apiRoutes entry {r!r} is not anchored at '^/' "
|
||||
f"followed by literal path characters. See assertion (1) for why breadth here is "
|
||||
f"the dangerous direction. The schema rejects this too (items.pattern); this "
|
||||
f"assertion exists because the schema stage can be SKIPPED and this one cannot")
|
||||
if routes:
|
||||
check(auth.get("mode") == "forward-auth",
|
||||
f"catalog/{sd_path.name} declares auth.apiRoutes with auth.mode="
|
||||
f"{auth.get('mode')!r}. --api-route renders onto the per-app oauth2-proxy sidecar, "
|
||||
f"and no sidecar is rendered for oidc/native/none — so this is a SILENT NO-OP: the "
|
||||
f"operator believes the app's XHRs now answer 401 and nothing changed")
|
||||
|
||||
matched = 0
|
||||
for name in sorted(forward_auth):
|
||||
golden = next((p for p in sorted(ROOT.glob(f"apps/wave-*/{name}.yaml"))), None)
|
||||
if golden is None:
|
||||
continue # catalog service not onboarded to apps/ yet — no golden
|
||||
key = next((k for k in rendered if k.startswith(str(golden.relative_to(ROOT)) + ":")), None)
|
||||
check(key is not None,
|
||||
f"apps/{golden.name} is the golden for forward-auth service {name!r} but no "
|
||||
f"oauth2-proxy container was parsed out of it. Its --api-route configuration is "
|
||||
f"UNAUDITED")
|
||||
if key is None:
|
||||
continue
|
||||
matched += 1
|
||||
check(rendered[key] == declared.get(name, []),
|
||||
f"{name}: the rendered sidecar declares --api-route {rendered[key]!r} but "
|
||||
f"catalog/{name}.yaml declares auth.apiRoutes {declared.get(name, [])!r}.\n"
|
||||
f" These must be IDENTICAL and in order. A mismatch means one of two things, "
|
||||
f"both bad: a golden was hand-edited (the next re-render silently reverts it), or the "
|
||||
f"renderer is emitting --api-route from somewhere other than this service's own "
|
||||
f"ServiceDefinition — i.e. a constant or a per-service table. A constant is the defect "
|
||||
f"this field exists to prevent: it applies ONE app's path shape to all five, and "
|
||||
f"pihole serves no /api/ at all, so it would answer pihole's page loads with 401. "
|
||||
f"render-golden cannot catch that, because render and golden would agree with each "
|
||||
f"other while both being wrong")
|
||||
|
||||
check(matched >= 4,
|
||||
f"only {matched} onboarded forward-auth service(s) were cross-checked against their "
|
||||
f"ServiceDefinition. This repo onboards five, so the catalog<->golden pairing has stopped "
|
||||
f"resolving and assertion (2) is passing vacuously")
|
||||
|
||||
# ============================================================ (3) BOTH CASES EXIST
|
||||
# Once the tree is consistent, (2) is satisfied by any pairing — including one where NOTHING
|
||||
# declares a route (vacuous) or one where EVERYTHING does (so the "renders nothing" path is
|
||||
# never exercised). Both halves are asserted to be live, because the per-app field only earns
|
||||
# its complexity if both outcomes actually occur in this tree.
|
||||
with_routes = [n for n in forward_auth if declared.get(n)]
|
||||
without_routes = [n for n in forward_auth if not declared.get(n)]
|
||||
check(bool(with_routes),
|
||||
"no onboarded forward-auth service declares auth.apiRoutes, so assertion (2) cannot "
|
||||
"distinguish a working renderer from one that emits the flag nowhere. If the flag is "
|
||||
"genuinely not needed anywhere, delete the field rather than leaving an unexercised seam")
|
||||
check(bool(without_routes),
|
||||
"EVERY onboarded forward-auth service declares auth.apiRoutes, so nothing in this tree "
|
||||
"exercises the 'declares none => renders no flag' path. That path is the whole reason the "
|
||||
"field is per-app: pihole serves no /api/, and a renderer-wide default would answer its "
|
||||
"page loads with 401")
|
||||
for name in sorted(without_routes):
|
||||
golden = next((p for p in sorted(ROOT.glob(f"apps/wave-*/{name}.yaml"))), None)
|
||||
if golden is None:
|
||||
continue
|
||||
key = next((k for k in rendered if k.startswith(str(golden.relative_to(ROOT)) + ":")), None)
|
||||
if key is None:
|
||||
continue
|
||||
check(rendered[key] == [],
|
||||
f"{name} declares NO auth.apiRoutes but its rendered sidecar carries "
|
||||
f"--api-route {rendered[key]!r}. An app that declares nothing must render NOTHING")
|
||||
|
||||
# ============================================================ (4) THE RENDERER DERIVES
|
||||
render_py = ROOT / "catalog" / "render.py"
|
||||
check(render_py.is_file(),
|
||||
"catalog/render.py is missing — the generator of five of these sidecars cannot be audited")
|
||||
if render_py.is_file():
|
||||
src = read(render_py)
|
||||
m = re.search(r"def _oauth2_sidecar_container\(.*?\n(?=\n*def |\n*# =)", src, re.S)
|
||||
fn = m.group(0) if m else ""
|
||||
check(bool(fn),
|
||||
"could not locate _oauth2_sidecar_container() in catalog/render.py. It was renamed or "
|
||||
"removed, so assertion (4) cannot run and the generator is unaudited")
|
||||
if fn:
|
||||
check("--api-route=" in fn,
|
||||
"catalog/render.py's _oauth2_sidecar_container() emits no --api-route at all, so "
|
||||
"every forward-auth app's XHRs are answered with a 302 the browser follows "
|
||||
"invisibly — measured as 33x302-vs-2x401 on jackett and 232x302 on radarr, with "
|
||||
"POST bodies dropped. auth.apiRoutes would then be a field with no consumer")
|
||||
check(not re.search(r'--api-route=\^/[A-Za-z0-9]', fn),
|
||||
"catalog/render.py hardcodes a literal path regex into --api-route. Whatever path "
|
||||
"that is, EVERY forward-auth app now renders it: one app's path shape applied to "
|
||||
"five. pihole serves no /api/, so a constant answers its page loads with 401 and "
|
||||
"the service looks permanently broken to a logged-out operator. Render the routes "
|
||||
"declared in each service's own auth.apiRoutes instead")
|
||||
check("apiRoutes" in fn,
|
||||
"catalog/render.py's _oauth2_sidecar_container() does not read auth.apiRoutes, so "
|
||||
"the per-app declaration has no effect on what is rendered")
|
||||
|
||||
# ============================================================ (5) ONE PATTERN, THREE SURFACES
|
||||
# The ratchet is written in three places (this check, the renderer, the schema) because each
|
||||
# guards a surface the others cannot. If they drift apart, the loosest one wins silently.
|
||||
if render_py.is_file():
|
||||
# The renderer writes the pattern as a raw string literal, so the source text contains it
|
||||
# verbatim. A substring test is deliberate: it also catches a rename of the constant.
|
||||
check(SCHEMA_PATTERN in read(render_py),
|
||||
f"catalog/render.py no longer contains the anchoring pattern {SCHEMA_PATTERN!r} that "
|
||||
f"this check and the schema enforce. Three surfaces guard --api-route breadth (the "
|
||||
f"authoring surface via the schema, the render via the renderer, the tree via this "
|
||||
f"check); if they drift apart the loosest wins and the ratchet is decorative")
|
||||
|
||||
# The schema's pattern is read from the PARSED JSON, not from the file text: in the file its
|
||||
# backslashes are JSON-escaped, so a substring test against the raw bytes would compare the
|
||||
# wrong string and pass or fail for the wrong reason.
|
||||
schema_file = ROOT / "schemas" / "servicedefinition.schema.json"
|
||||
schema_pat = None
|
||||
if schema_file.is_file():
|
||||
try:
|
||||
schema_pat = (((((json.loads(read(schema_file)).get("properties") or {})
|
||||
.get("auth") or {}).get("properties") or {})
|
||||
.get("apiRoutes") or {}).get("items") or {}).get("pattern")
|
||||
except (ValueError, AttributeError) as exc:
|
||||
check(False, f"schemas/servicedefinition.schema.json could not be read as JSON ({exc}) — "
|
||||
f"the schema stage that rejects a too-broad --api-route cannot be audited")
|
||||
check(schema_pat == SCHEMA_PATTERN,
|
||||
f"schemas/servicedefinition.schema.json constrains auth.apiRoutes items with "
|
||||
f"{schema_pat!r}, not {SCHEMA_PATTERN!r}. The schema is the only surface that rejects a "
|
||||
f"too-broad regex WHERE A HUMAN TYPES IT (ci/l0.sh validates catalog/*.yaml on every gate "
|
||||
f"run); if it is absent or looser than the renderer's, the operator finds out from a 401 "
|
||||
f"on a page load")
|
||||
|
||||
# ============================================================ (6) PROVE THE SHIPPED RENDERER
|
||||
# Once the tree is compliant, (1)-(3) have nothing left to catch and their passing is evidence
|
||||
# of nothing. This runs the SHIPPED renderer as a SUBPROCESS on synthetic ServiceDefinitions and
|
||||
# asserts the behaviour directly, in BOTH directions, so the guard stays falsifiable.
|
||||
def _sd(name: str, port: int, routes) -> str:
|
||||
auth = {"mode": "forward-auth", "groups": ["platform-admins"]}
|
||||
if routes is not None:
|
||||
auth["apiRoutes"] = routes
|
||||
return yaml.safe_dump({
|
||||
"name": name,
|
||||
"kind": "container",
|
||||
"source": {"image": "example/app:1"},
|
||||
"exposure": "internal",
|
||||
"ingressRoutes": [{"host": f"{name}.home.keithjackson.dev", "port": port}],
|
||||
"auth": auth,
|
||||
}, sort_keys=False)
|
||||
|
||||
|
||||
def _render(name: str, port: int, routes):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
f = Path(td) / f"{name}.yaml"
|
||||
f.write_text(_sd(name, port, routes))
|
||||
return subprocess.run([sys.executable, str(render_py), str(f)],
|
||||
capture_output=True, text=True)
|
||||
|
||||
|
||||
def _sidecar_args(rendered_yaml: str) -> list:
|
||||
for doc in yaml.safe_load_all(rendered_yaml):
|
||||
if not isinstance(doc, dict) or doc.get("kind") != "Deployment":
|
||||
continue
|
||||
for c in iter_containers(doc):
|
||||
if c.get("name") == "oauth2-proxy":
|
||||
return list(c.get("args") or [])
|
||||
return []
|
||||
|
||||
|
||||
if render_py.is_file():
|
||||
# (6a) DECLARED ROUTES RENDER, IN ORDER, AND NOTHING ELSE DOES.
|
||||
proc = _render("apirtdecl", 8121, ["^/api/", "^/signalr/"])
|
||||
check(proc.returncode == 0,
|
||||
f"the shipped catalog/render.py failed on a synthetic forward-auth fixture declaring "
|
||||
f"two apiRoutes: rc={proc.returncode} {proc.stderr.strip()[:300]}")
|
||||
if proc.returncode == 0:
|
||||
got = api_routes_of(_sidecar_args(proc.stdout))
|
||||
check(got == ["^/api/", "^/signalr/"],
|
||||
f"the shipped renderer turned auth.apiRoutes ['^/api/', '^/signalr/'] into "
|
||||
f"--api-route {got!r}. It must emit one repeatable --api-route per entry, in "
|
||||
f"declared order. Note the flag is SINGULAR: --api-routes does not exist in "
|
||||
f"oauth2-proxy v7.8.1 and the proxy REFUSES TO START with it, so a plural would "
|
||||
f"crashloop every sidecar it rendered")
|
||||
|
||||
# (6b) AN APP THAT DECLARES NOTHING RENDERS NOTHING. The pihole case, proven on the
|
||||
# generator rather than inferred from one committed golden.
|
||||
for label, routes in (("absent", None), ("empty", [])):
|
||||
proc = _render(f"apirtnone{label}", 8122, routes)
|
||||
check(proc.returncode == 0,
|
||||
f"the shipped renderer failed on a synthetic forward-auth fixture with apiRoutes "
|
||||
f"{label}: rc={proc.returncode} {proc.stderr.strip()[:300]}")
|
||||
if proc.returncode == 0:
|
||||
got = api_routes_of(_sidecar_args(proc.stdout))
|
||||
check(got == [],
|
||||
f"the shipped renderer emitted --api-route {got!r} for a synthetic service "
|
||||
f"whose auth.apiRoutes is {label}. A newly-onboarded forward-auth app must "
|
||||
f"render NO --api-route: pihole serves no /api/, so a default would answer "
|
||||
f"its page loads with 401 and the service would look permanently broken to a "
|
||||
f"logged-out operator — a worse failure than the 302 bug the flag fixes")
|
||||
|
||||
# (6c) THE RENDERER REFUSES BREADTH. Asserted on the generator because render-golden runs
|
||||
# on every gate invocation while the schema stage can be skipped.
|
||||
for bad in ("^/", "^/.*", ".*", "/api/", "^/api/|^/"):
|
||||
proc = _render("apirtbad", 8123, [bad])
|
||||
check(proc.returncode != 0,
|
||||
f"the shipped renderer ACCEPTED auth.apiRoutes {bad!r} and rendered a sidecar. "
|
||||
f"That regex matches paths a browser NAVIGATES to, so a plain page load answers "
|
||||
f"401 and the operator is never offered a sign-in. The renderer must refuse it, "
|
||||
f"so a breadth mistake fails render-golden even on a run where the schema stage "
|
||||
f"did not execute")
|
||||
|
||||
# ============================================================ VERDICT
|
||||
inv = " | ".join(f"{k} -> {v or '(none)'}" for k, v in sorted(rendered.items()))
|
||||
print(f"oauth2-proxy sidecars audited ({len(rendered)}): {inv}")
|
||||
if FAILURES:
|
||||
print(f"\nCHECK FAIL (oauth2-proxy-api-routes-per-app): {len(FAILURES)} failed, "
|
||||
f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" FAIL {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"check OK: oauth2-proxy-api-routes-per-app — {CHECKS} assertions (R-IDP-02)")
|
||||
PY
|
||||
@@ -0,0 +1,174 @@
|
||||
---
|
||||
date: 2026-09-30
|
||||
branch: feat/plex-quicksync
|
||||
title: Plex opts into `hardware.gpu: intel-quicksync` — the device mount is GitOps, the Plex-side toggle is a documented manual step, and hardware transcoding has NOT been observed working
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
Three things, and the third is the point of the branch.
|
||||
|
||||
1. **`catalog/plex.yaml`** — added the four lines the whole feature reduces to:
|
||||
|
||||
```yaml
|
||||
hardware:
|
||||
gpu: intel-quicksync
|
||||
```
|
||||
|
||||
placed between `env` and `exposure` to match the schema's own property order, with ~48 lines of
|
||||
comment in the file's established style. The comment records: the field is **intent, not
|
||||
mechanism** (a later move to the Intel device plugin is a `render.py` change and this block does
|
||||
not move); that `/dev/dri/renderD128` is present on **all three** prod nodes with `i915` bound;
|
||||
**why a `supplementalGroups` entry would be inert** (the image's cont-init hook does the group
|
||||
work inside the container and `s6-setuidgid` replaces the process group list from `/etc/group`);
|
||||
why `DirectoryOrCreate` is the load-bearing safe failure mode (`apps-wave-0` is `wait: true` with
|
||||
`apps-wave-1` depending on it, so a failed mount would stall the whole app tier for a feature
|
||||
that is not load-bearing); and that the Plex-side toggle is a **separate manual operator step**,
|
||||
pointing at the new runbook.
|
||||
|
||||
2. **`apps/wave-0/plex.yaml`** — re-rendered. Six added lines, nothing else, in the whole `apps/`
|
||||
tree:
|
||||
|
||||
```
|
||||
- name: dri
|
||||
mountPath: /dev/dri
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: DirectoryOrCreate
|
||||
```
|
||||
|
||||
3. **`runbooks/plex-quicksync.md`** (new), registered in `runbooks/kustomization.yaml`'s
|
||||
`configMapGenerator` as `- plex-quicksync=plex-quicksync.md` and indexed in `runbooks/README.md`
|
||||
under "Operator procedures". **The file alone would have been a 404** — the pod serves a
|
||||
ConfigMap, not the git tree, and `conformance/tests/l0-static/alert-runbooks-resolvable` exists
|
||||
precisely because "the file is in git" and "the URL resolves" are different claims. The page
|
||||
covers: the GitOps/manual boundary as a table; the toggle (Settings → Transcoder → "Use hardware
|
||||
acceleration when available", preference key `HardwareAcceleratedCodecs`); the real neighbouring
|
||||
keys and the widely-cited one that **does not exist**; that the only proof is `Transcode (hw)` on
|
||||
the Plex dashboard **and why no alert is possible**; the device-cgroup failure to expect first
|
||||
with a concrete open() diagnostic and the Intel-device-plugin fallback; and the floating-`:latest`
|
||||
caveat with a digest check.
|
||||
|
||||
**Stacked on PR #27 (`feat/sd-hardware-capability`), which is open and unmerged.** This branch was
|
||||
cut from it rather than from `main`, so its one commit (`d9baa2a`, the schema + renderer + contract)
|
||||
is in this PR's range. That is deliberate: merging #27 is not an agent's action (CLAUDE.md §1), and
|
||||
duplicating its changes here would have been worse than showing them. **This must merge after #27.**
|
||||
|
||||
Not done, deliberately: no `--emit-waves` (it writes into `clusters/base/`, a sensitive path that
|
||||
would force a human reviewer, and it is a no-op unless the *set* of files under `apps/` changes — it
|
||||
does not here). No `podSecurityContext`. No `nodeSelector`. No alert rule. No `kubectl apply`.
|
||||
|
||||
### Two follow-ups left unclaimed, with the reasoning
|
||||
|
||||
- **Transcode scratch is on the wrong medium, and it is invisible.** The image sets
|
||||
`TranscoderTempDirectory=/transcode`, which is **not a volume** — it is the container's writable
|
||||
overlay, i.e. the node's `/data`. That is NVMe on prod-2 but **SATA on prod-1 and prod-3**, so
|
||||
transcode scratch performance changes silently on reschedule, with nothing reporting it. Both
|
||||
candidate fixes need a **schema** change and so neither belongs in this PR: pin Plex to prod-2
|
||||
(needs a `nodeSelector` field the schema does not have), or give `/transcode` a `medium: Memory`
|
||||
emptyDir and transcode in RAM (~31 GiB per node; needs an emptyDir field the schema does not
|
||||
have). **The RAM option is the better one** — it removes the node asymmetry outright instead of
|
||||
working around it — which is exactly why it deserves its own PR and its own review rather than
|
||||
being smuggled in behind a four-line catalog change. Documented in the runbook's "Known gaps".
|
||||
- **GuC/HuC firmware is not loaded** (`i915.enable_guc=0`). The cost is low-power H.264 **encode**
|
||||
quality only; decode and this feature are unaffected. Changing it needs a kernel parameter plus a
|
||||
**serialized rolling reboot** — an operator action under CLAUDE.md §5, on a cluster with a
|
||||
documented history of that going badly (a prod node hung on a warm reboot on 2026-09-03). Kept
|
||||
well clear of this PR.
|
||||
|
||||
### Rebase note (2026-10-01, Ruth Okonkwo) — resolution only, no redesign
|
||||
|
||||
Rebased onto `main` @ `08deb20` (was cut from `0fbf544`), restacked on the rebased
|
||||
`feat/sd-hardware-capability` (PR #27) so the stack stays coherent — this branch still carries
|
||||
exactly PR #27's commit plus its own, with no duplicate of anything already in `main`. The branch
|
||||
ref was force-updated with `--force-with-lease`, so PR #32 is the same PR with a new head.
|
||||
|
||||
**The `catalog/render.py` conflict disappeared** once PR #27 was rebased; it was never this
|
||||
branch's own conflict. **One conflict remained, in `runbooks/README.md`, purely textual.** PR #30
|
||||
(`fix/jackett-oauth2-proxy-session`, merged as `08deb20`) appended an `oauth2-proxy-session` index
|
||||
entry under *Operator procedures* at the same point this branch appended its `plex-quicksync` entry.
|
||||
**Both entries were kept, in full.** Ordering follows this file's existing convention, which is
|
||||
grouped/narrative rather than alphabetical (the *Nodes* section runs down/io/memory/disk/cpu):
|
||||
`plex-quicksync` sits next to `plex-claim` as the other Plex page, leaving `oauth2-proxy-session`
|
||||
adjacent to `crafty-admin-credential`, which cross-references the same sidecar gate. The change to
|
||||
`runbooks/README.md` is an 8-line pure insertion, so PR #30's entry is untouched.
|
||||
`runbooks/kustomization.yaml` auto-merged and **both** `configMapGenerator` entries are present.
|
||||
|
||||
Re-verified after the resolution, on the rebased stack: `--selftest` PASS (rc 0); a full re-render
|
||||
reproduces every committed golden byte-identically; and the **only** `apps/` diff against the new
|
||||
`main` is `apps/wave-0/plex.yaml`, confined to the expected six lines (the `dri` mount and its
|
||||
`DirectoryOrCreate` hostPath volume). No other service's golden moved — PR #28 added ~176 lines of
|
||||
oauth2-proxy sidecar rendering, and this confirms by measurement rather than assumption that Plex
|
||||
(`auth: native`, no sidecar) is unaffected by it.
|
||||
|
||||
## What I verified
|
||||
|
||||
- **The render is exactly the intended change, and nothing else moved.** After
|
||||
`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` (exit 0,
|
||||
silent): `git diff --stat apps/` = `apps/wave-0/plex.yaml | 6 ++++++`, and
|
||||
`git status --porcelain apps/` = a single ` M apps/wave-0/plex.yaml`. Read line by line, the diff
|
||||
is the `dri` volumeMount and the `dri` hostPath volume, and nothing more. **Specifically checked:**
|
||||
the same invocation also rewrites `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, which
|
||||
are git-tracked but excluded from the golden comparison by `RENDER_GOLDEN_NOT_ONBOARDED` in
|
||||
`ci/render-golden.sh` — a spurious diff there would have landed **ungated**. Neither file shows a
|
||||
diff; whole-repo `git status --porcelain` lists only `apps/wave-0/plex.yaml` and `catalog/plex.yaml`
|
||||
plus my new files.
|
||||
- **The rendered output matches what the contract promises** — `schemas/renderer-contract.md`
|
||||
`### hardware` says exactly two things (a `/dev/dri` hostPath with `type: DirectoryOrCreate` and a
|
||||
read-write mount on the app container) and exactly two things appeared. No `securityContext`, no
|
||||
`nodeSelector`, no resources in the diff.
|
||||
- **The runbook is projected into the server, not merely committed.** `runbooks/kustomization.yaml`
|
||||
carries `- plex-quicksync=plex-quicksync.md`, whose shape matches the generator-entry regex the
|
||||
L0 check parses; the extensionless key is what makes `/runbooks/plex-quicksync` a plain static
|
||||
lookup. Also indexed in `runbooks/README.md`.
|
||||
- **The runbook clears the L0 quality floor** that `alert-runbooks-resolvable` applies to
|
||||
alert-linked pages, even though nothing links to this one (it is advisory for an unreferenced
|
||||
page): all four required sections (`## What this means here`, `## First commands`,
|
||||
`## Likely causes, most likely first`, `## When to escalate or stop`), well over the 1500-byte
|
||||
floor, `dev.sh` used throughout, and `--context k3s-prod` on every command.
|
||||
- **The gate.** See the run recorded in the PR description for the per-stage table and the real rc.
|
||||
- **No secret was printed.** Every `Preferences.xml` read in the runbook greps one named key, and the
|
||||
page states explicitly why: the file holds `PlexOnlineToken`. Nothing was `sops -d`'d.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
**Hardware transcoding has NOT been observed working.** Nothing in this branch is evidence that it
|
||||
does. A green gate here proves the manifest renders and the golden matches — it proves nothing about
|
||||
runtime behaviour, and `vars.CLUSTER_AVAILABLE` is unset so L2/L3 conformance did not run at all.
|
||||
|
||||
- **The device-cgroup question is OPEN, and it is the largest remaining technical risk.** Whether
|
||||
containerd on this k3s build permits a **non-privileged** pod to `open()` a bind-mounted character
|
||||
device is **UNVERIFIED**. It can only be settled by deploying and trying. The container inits as
|
||||
uid 0, which settles *file* permissions, but uid 0 does **not** bypass the cgroup device gate — and
|
||||
the two failures look identical from the outside. If it is refused, the fallback is the Intel
|
||||
Device Plugin (`gpu.intel.com/i915`), which is a `catalog/render.py` change plus a re-render with
|
||||
**no ServiceDefinition edit**; `privileged: true` is not available (Kyverno
|
||||
`restrict-host-namespaces` is Enforce). The runbook has the diagnostic sequence.
|
||||
- **The Plex-side toggle was still unset at the time of writing.** `HardwareAcceleratedCodecs` was
|
||||
**absent** from `Preferences.xml` — the toggle has never been enabled on this server. Until an
|
||||
operator ticks it in the UI, this branch changes the pod spec and **nothing about transcoding**.
|
||||
That is the whole reason the runbook exists.
|
||||
- **No metric, so no alert, so no automated detection — permanently.** `catalog/plex.yaml` declares
|
||||
no `metricsEndpoint`, so Plex renders no ServiceMonitor; there is no transcode-mode signal for a
|
||||
rule to evaluate. A silent degradation to software transcoding (a node losing its iGPU, the
|
||||
`DirectoryOrCreate` path) will **not** be reported by anything. I deliberately did not add an
|
||||
alert: with no metric behind it, it could never fire, and this repo has been burned at scale by
|
||||
exactly that. Closing the gap honestly means an exporter first.
|
||||
- **I did not re-derive this session's node or in-container measurements.** The three-node
|
||||
`renderD128`/`i915`/GID-992 facts, the cont-init hook behaviour, the `s6-setuidgid` group
|
||||
replacement, the `PLEX_PREFERENCE*` absence, the six `setPref` keys and the
|
||||
`HardwareAcceleratedCodecs` string were established earlier in this session and are cited, not
|
||||
re-measured, by me. Per CLAUDE.md §2b, treat this file as a record of belief and re-measure against
|
||||
the cluster before relying on any of it.
|
||||
- **The image is a floating tag.** Everything known about the hook came from digest
|
||||
`sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), while the Deployment runs
|
||||
`plexinc/pms-docker:latest` with `imagePullPolicy: Always`. A silent pull can invalidate the
|
||||
"no `supplementalGroups` needed" conclusion **with no commit in this repo**. Not guarded against —
|
||||
the runbook tells the reader to re-check the hook after an image bump, which is documentation, not
|
||||
enforcement.
|
||||
- **I did not verify the live Gitea gate state** (branch protection, required contexts, runner
|
||||
online) for this PR. Per §2b that must be queried, not read out of `CLAUDE.md`'s table; I did not
|
||||
query it.
|
||||
- **`/runbooks/plex-quicksync` was not fetched.** The L0 check asserts every static precondition for
|
||||
an HTTP 200; it does not make the request, and I did not either.
|
||||
@@ -0,0 +1,204 @@
|
||||
---
|
||||
date: 2026-09-30
|
||||
branch: feat/sd-hardware-capability
|
||||
title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` mounts /dev/dri and nothing else; the enum is what keeps the mechanism swappable
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
A new optional top-level ServiceDefinition property, `hardware`, with one optional member,
|
||||
`gpu`, whose enum today has exactly one value: `intel-quicksync`. **No catalog entry uses it.**
|
||||
This PR delivers the capability; consuming it (Plex) is separate work on top.
|
||||
|
||||
- `schemas/servicedefinition.schema.json` — adds `hardware` (object, `additionalProperties: false`,
|
||||
`minProperties: 1`) with `gpu` (enum `["intel-quicksync"]`). Schema version bumped **1.7.0 →
|
||||
1.8.0**, changelog entry prepended to the schema `description` per that file's own convention.
|
||||
- `catalog/render.py` — two module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`) and the
|
||||
emission in `r_deployment`.
|
||||
- `schemas/renderer-contract.md` — new `### hardware` field-mapping section, plus cross-field
|
||||
invariant **8** in that file's numbered list.
|
||||
- `schemas/README.md` — its `**Schema version:**` line said `1.0.0`; it had been stale since 1.1.0.
|
||||
Synced to 1.8.0 and annotated that the schema `description` is the changelog of record. Added one
|
||||
bullet to "Cross-field rules" for the new enum. That was the whole of the staleness; nothing
|
||||
ballooned.
|
||||
|
||||
**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container only (never a
|
||||
sidecar), and it is exactly two things:
|
||||
|
||||
1. a hostPath volume `dri` → `/dev/dri`, `type: DirectoryOrCreate`;
|
||||
2. a read-write `volumeMount` of `dri` at `/dev/dri` on the app container (deliberately not
|
||||
`readOnly` — VA-API opens the render node read-write).
|
||||
|
||||
**Mounting the device is sufficient.** It emits **no** `privileged: true` (Kyverno
|
||||
`restrict-host-namespaces` is Enforce on app namespaces and it is not needed), **no** `nodeSelector`
|
||||
(placement is a separate concern with its own justification), **no** resource requests (there is no
|
||||
app-container resource render path), and — the one that is worth spelling out — **no
|
||||
`podSecurityContext.supplementalGroups`**.
|
||||
|
||||
**Why no `supplementalGroups`, when `/dev/dri/renderD128` is `root:render 0660`.** An earlier
|
||||
revision of this branch merged the host `render` GID into `supplementalGroups`. **It was measured to
|
||||
be a no-op and has been removed.** In the running container on prod-2, 2026-09-30:
|
||||
|
||||
PID 1 s6-svscan Uid: 0 Gid: 0 Groups: 0
|
||||
pid 246 Plex Media Server Uid: 1000 Gid: 1000 Groups: 100 1000
|
||||
|
||||
`/etc/services.d/plex/run` ends in `exec s6-setuidgid plex …`, and the child's group list is exactly
|
||||
the `/etc/group` account-database list — root's gid `0`, which PID 1 holds, has **vanished**. That is
|
||||
a `setgroups()` that *replaces* rather than appends, so anything Kubernetes puts in
|
||||
`supplementalGroups` reaches `s6-svscan` and is discarded before the app ever starts. What actually
|
||||
works is the image's own `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner`, which `find`s
|
||||
every char device under `/dev/dri`, `stat -c '%g'`s its GID, `groupadd`s a matching group inside the
|
||||
container when none exists, and `usermod -a -G`s the app user into it — every start, nothing
|
||||
hardcoded; `cont-init.d` completes before `services.d`, so the database is correct at exec time.
|
||||
|
||||
Shipping an inert `supplementalGroups` would have been worse than shipping nothing: it would read as
|
||||
the mechanism to whoever debugs this next. The reasoning lives in a comment **at the emission site**
|
||||
and in `renderer-contract.md`, and `render.py --selftest` now asserts the *absence* of a
|
||||
`securityContext`, so it cannot be "helpfully" added back without a test failing. A pleasant
|
||||
corollary: because the hook re-`stat`s the device every start, host GID drift is **self-healing**,
|
||||
and no GID is pinned anywhere in this repo.
|
||||
|
||||
**The design decision worth defending: the field is an enum of intent, not a device path.** The SD
|
||||
says *what class of host device the workload needs*; the hostPath and the mount exist only in
|
||||
`catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource
|
||||
requests, no hostPath at all) is then a renderer change with zero catalog edits. Both the schema
|
||||
`description` and the contract section say this explicitly, because the obvious "simplification" —
|
||||
letting an SD name `/dev/dri` — would push a platform implementation detail into the operator-facing
|
||||
contract and make that migration a catalog-wide edit.
|
||||
|
||||
**`DirectoryOrCreate` is load-bearing.** A node that comes back without the render device (rebuild,
|
||||
BIOS change, driver regression, hardware swap) must degrade the workload to software transcoding,
|
||||
not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so a failed
|
||||
hostPath mount stalls the entire app tier for the sake of a feature that is not load-bearing.
|
||||
`/dev` is tmpfs, so the empty directory the kubelet creates on such a node leaves nothing durable
|
||||
behind. Do not "tighten" it to `Directory`. The reasoning is recorded at the emission site.
|
||||
|
||||
### Rebase note (2026-10-01, Ruth Okonkwo) — resolution only, no redesign
|
||||
|
||||
Rebased onto `main` @ `08deb20` (was cut from `0fbf544`). Re-landed as a single cherry-pick onto
|
||||
current `main`; the branch ref was force-updated with `--force-with-lease`, so PR #27 is the same
|
||||
PR with a new head.
|
||||
|
||||
**One conflict, in `catalog/render.py`, purely textual.** PR #28
|
||||
(`fix/oauth2-proxy-cookie-collision-fleet`, merged as `6caa5f2`) inserted its own constants blocks
|
||||
(`OAUTH2_ACCESS_TOKEN_LIFETIME_S` / `OAUTH2_COOKIE_REFRESH`, and `OAUTH2_PROXY_METRICS_PORT`) at the
|
||||
same insertion point immediately after `KEYCLOAK_ISSUER` that this branch used for
|
||||
`DRI_VOLUME_NAME` / `DRI_DEVICE_DIR`. The two blocks are unrelated and do not interact, so **both
|
||||
sides were kept in full**, oauth2-proxy's first and the hardware block after it. Nothing was
|
||||
dropped and nothing was rewritten: the rebased commit adds the **same 126 `render.py` lines** as the
|
||||
pre-rebase commit (verified by comparing the added-line sets) and deletes none. In particular the
|
||||
`supplementalGroups`-is-deliberately-not-emitted and `DirectoryOrCreate`-is-load-bearing comments
|
||||
survive verbatim.
|
||||
|
||||
Re-verified after the resolution: `--selftest` PASS (rc 0), and a full re-render still produces a
|
||||
**zero** diff in `apps/` — this PR remains inert, as it must be while no ServiceDefinition opts in.
|
||||
|
||||
## What I verified
|
||||
|
||||
Measured, in this worktree, on this branch:
|
||||
|
||||
- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** New cases in the
|
||||
existing `check(...)` style: the positive (volume + mount, and *no* `securityContext`, *no*
|
||||
privileged, *no* nodeSelector), the negative that matters (an SD without `hardware` gains no
|
||||
volume, no volumeMount and no `securityContext` — mirroring the existing `podSecurityContext`
|
||||
negative), and a pass-through case (an SD declaring both `hardware` and its own
|
||||
`podSecurityContext: {fsGroup: 1001, supplementalGroups: [44]}` keeps it **exactly** as written).
|
||||
- **The self-tests actually bite.** I mutated the guard (`if hw_gpu == "intel-quicksync":` → `if
|
||||
True:`) in a scratch copy and re-ran: **rc 1**, failing on "an SD without `hardware` must gain no
|
||||
dri volume, no mount and no securityContext". A test that cannot fail is not a test; this one can.
|
||||
- **Re-render is a zero diff.**
|
||||
`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` then
|
||||
`git diff --stat apps/` → **empty**, and `git status --porcelain apps/` → **empty** (so the two
|
||||
not-onboarded by-products, `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, were
|
||||
rewritten byte-identically and needed no revert). The change is inert until a service opts in.
|
||||
- **Schema validity and catalog conformance.** `Draft202012Validator.check_schema` passes; all
|
||||
**12** `catalog/*.yaml` validate with **0** errors. On the new field specifically: `gpu:
|
||||
intel-quicksync` accepted; `gpu: /dev/dri` rejected (enum); `hardware: {gpu: …, path: …}` rejected
|
||||
(`additionalProperties: false`); `hardware: {}` rejected (`minProperties: 1`).
|
||||
- **Hand-checked the emitted YAML** by rendering a throwaway SD carrying `hardware`, a Longhorn
|
||||
volume and its own `podSecurityContext: {runAsUser: 1000, supplementalGroups: [44]}`. Result:
|
||||
`volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `volumes` = the PVC then
|
||||
`{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`; `securityContext` =
|
||||
`{runAsUser: 1000, supplementalGroups: [44]}` — i.e. **unchanged**, nothing appended. That toy SD
|
||||
was never written into `catalog/`.
|
||||
- **GATE: `rc 3` — UNDETERMINED, NOT green.** Run in the pinned toolbox from this worktree with
|
||||
both CLAUDE.md §3 workarounds (`DEV_KUBECONFIG=/dev/null`, and an `ENGINE` shim bind-mounting the
|
||||
main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f`; touched
|
||||
requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Run **four** times on this branch with an identical
|
||||
result each time, most recently against tree `c132893045b1` — the final commit differs from that
|
||||
run only by this paragraph. Per stage:
|
||||
|
||||
| stage | result |
|
||||
|---|---|
|
||||
| spec-immutability | **SKIPPED** — sibling spec repo not at `/work/../home-network-k3s`. Expected from a worktree; tolerated by §3. |
|
||||
| decisions-log | pass |
|
||||
| lint | pass |
|
||||
| l0 | pass — 19 offline cases, `0 failure(s), 0 skip(s), 0 unwired` |
|
||||
| render-golden | pass — *"all 10 onboarded service golden(s) reproduce from the canonical render"* |
|
||||
| l1 | pass — 3 offline checks, `0 failure(s), 0 skip(s), 0 unwired`; coverage 51/51 requirements scaffolded |
|
||||
| approval-policy | pass (`event='none'`, not a pull_request event — N/A locally; it will evaluate for real on the PR) |
|
||||
| gitea-settings | **SKIPPED** — `SETTINGS_API_TOKEN` unset. Expected; branch protection and runner registration are therefore UNVERIFIED by this run. |
|
||||
| conformance (L2/L3) | **SKIPPED** — `CLUSTER_AVAILABLE!=true`. Phase-0 standing skip. |
|
||||
|
||||
The gate's own verdict: *"Stages that did not run: spec-immutability gitea-settings"*. **No stage
|
||||
failed, and no stage skipped for a reason caused by this change.** rc 3 is reported as rc 3; I am
|
||||
not calling this green.
|
||||
- **Blast radius of the schema change:** `hardware` is absent from every catalog entry, so
|
||||
`apply_defaults` never materializes it and `s.get("hardware")` is `None` on all 12 — which is what
|
||||
the zero diff above demonstrates rather than asserts.
|
||||
- **Touched no sensitive path.** Changes are confined to `schemas/`, `catalog/`, `decisions/`; per
|
||||
`infra/gitea/approval-policy.json` none of those are sensitive. `conformance/**` and `clusters/**`
|
||||
are untouched. I did not run `--emit-waves` (it writes `clusters/base/`, and the set of files
|
||||
under `apps/` did not change).
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **Whether containerd's device cgroup permits a non-privileged pod to `open()` a bind-mounted char
|
||||
device. Still unverified, and it is the single largest technical risk in this workstream.**
|
||||
containerd installs a per-container **device cgroup** allow-list, and whether
|
||||
`/dev/dri/renderD128` arriving as a bind-mounted device node is permitted by it on this
|
||||
k3s/containerd version is **unmeasured**. Settling it requires applying a pod, which is an
|
||||
operator action, not an agent one. The container's PID 1 running as uid 0 helps on the DAC side
|
||||
but does **not** bypass the cgroup gate. If it blocks, the mount will succeed, the image's group
|
||||
hook will do its job, and the `open()` will still fail — and the fallback is the Intel GPU device
|
||||
plugin, which *sets* the cgroup entry. Note that that fallback is a **renderer-only change**: no
|
||||
ServiceDefinition moves, because the authoring surface is an intent enum. That is the design
|
||||
justifying itself, and it is the reason to keep the enum even if the hostPath mechanism loses.
|
||||
- **That a missing device degrades silently — and nothing will tell you.** This is the accepted cost
|
||||
of `DirectoryOrCreate`. The workload will software-transcode and **no signal reports it**: media
|
||||
apps in this catalog declare no `metricsEndpoint`, so no ServiceMonitor exists, and there is no
|
||||
metric to alert on. I deliberately did **not** propose an alert, because an alert that cannot fire
|
||||
is worse than none — see the 2026-09-13 correction in `renderer-contract.md` §`observability`,
|
||||
where twelve monitors discovered zero targets. The only proof of hardware transcode is an operator
|
||||
seeing it in the application's own dashboard.
|
||||
- **That "mounting the device is sufficient" generalizes beyond this one image.** It does not, and
|
||||
I have not verified that it does. The claim was measured for an image that resolves the device
|
||||
group itself in `cont-init.d`. A future consumer of `hardware.gpu` that runs non-root with no
|
||||
equivalent hook would still need the group solved — and the fix for that case is a **renderer**
|
||||
change (per-image handling, or the device plugin), never a new SD field. I scoped the claim in
|
||||
`renderer-contract.md` rather than letting it read as general; the first such consumer must
|
||||
re-measure.
|
||||
- **The feature now depends on a hook inside an image that floats.** The `cont-init.d` behaviour
|
||||
above was read from digest `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), but the
|
||||
deployment runs `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. **A silent pull can
|
||||
change the behaviour this feature depends on, with no commit anywhere in this repo to show for
|
||||
it** — and the failure would be the silent software-transcode described above. I have not fixed
|
||||
that here (it is the Plex wiring's tag/digest decision, not the capability's), but it should not
|
||||
be forgotten.
|
||||
- **GID drift is NOT on this list any more, deliberately.** An earlier revision pinned
|
||||
`INTEL_RENDER_GID = 992` and listed "a node rebuild could move it" as an open risk. The hook
|
||||
re-`stat`s the device on every start, so drift is self-healing and no GID is pinned anywhere. The
|
||||
risk is retired, not overlooked.
|
||||
- **Anything on the live cluster.** I ran no `kubectl` and I opened no shell in any container. The
|
||||
node facts (device present, `root:render` 0660, `i915` bound, no device plugin, no
|
||||
`gpu.intel.com/*` advertised), the in-container measurements quoted above, and the Kyverno
|
||||
posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups`
|
||||
restriction; no PSA labels on app namespaces) are **prior recon handed to me, not re-measured by
|
||||
me**. Per CLAUDE.md §2b, treat them as second-hand until re-queried.
|
||||
- **That CI exercises the self-tests. It does not.** `git grep -- --selftest ci/` returns nothing —
|
||||
no gate stage invokes the renderer's self-test. They are the executable spec and house style, and
|
||||
I ran them by hand, but a future regression in this code path **will not be caught by the gate**.
|
||||
Wiring `--selftest` into a gate stage is a real gap and is not part of this PR (`ci/` is a
|
||||
sensitive path and would drag in a human reviewer for a change unrelated to this one).
|
||||
- **Runtime behaviour of any kind.** The gate proves the L0/L1 offline subset only;
|
||||
`vars.CLUSTER_AVAILABLE` is unset, so L2/L3 conformance does not run in CI.
|
||||
@@ -0,0 +1,482 @@
|
||||
---
|
||||
date: 2026-10-01
|
||||
branch: feat/oauth2-proxy-api-route-per-app
|
||||
title: The config model had no way to say "this app's API lives here" — so --api-route could not ship. Now it can, per app.
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
Second half of PR #28 (`decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md`), which
|
||||
fixed the cookie collision, the 5-minute session and the missing metrics listener in one renderer
|
||||
function and **deliberately declined `--api-route`** for a reason I have preserved rather than
|
||||
worked around: that flag was measured on **one** app, and `PiHole serves no /api/ at all`, so a
|
||||
renderer-wide constant would apply one app's path shape to five. The `auth` block is
|
||||
`additionalProperties: false`, so there was no per-app override hook. **Creating the hook was the
|
||||
work; setting the flag is the easy part.**
|
||||
|
||||
**Three surfaces, in order of what they protect:**
|
||||
|
||||
| Surface | Change | What it guards that the others cannot |
|
||||
|---|---|---|
|
||||
| `schemas/servicedefinition.schema.json` (**1.8.0 → 1.9.0**) | new optional `auth.apiRoutes`: array of anchored path regexes, `uniqueItems`, default `[]` | rejects a too-broad regex **where a human types it** — `ci/l0.sh` validates `catalog/*.yaml` against this schema on every gate run |
|
||||
| `catalog/render.py` | emits one `--api-route=<r>` per declared entry, in order; **raises** on an unanchored entry | `ci/render-golden.sh` runs on every gate invocation even when the schema stage skips |
|
||||
| `conformance/tests/l0-static/oauth2-proxy-api-routes-per-app/` (new) | cross-checks each rendered sidecar against its own ServiceDefinition | catches a **renderer constant**, which neither of the above can see |
|
||||
|
||||
**The `auth` block stays `additionalProperties: false`.** One named, documented key was added; the
|
||||
door is not opened. That property is the reason the gap existed, and it is worth keeping — it is
|
||||
what made "there is no override hook" a *fact* rather than a *guess* for the two engineers who hit
|
||||
it before me.
|
||||
|
||||
**Rebased mid-flight.** `origin/main` advanced from `08deb20` to `cdf3e14` while I was working
|
||||
(PR #27 and PR #32, the `hardware.gpu: intel-quicksync` capability). **That change also took schema
|
||||
`1.8.0`**, so mine is renumbered to **1.9.0** and chained behind it; both version entries are
|
||||
present in the description, and `catalog/render.py` had a two-sided conflict in the constants block
|
||||
which is resolved by keeping both (the `DRI_*` constants and `OAUTH2_API_ROUTE_PATTERN` are
|
||||
unrelated). Re-rendering after the rebase still changes **only** jackett and radarr — plex's new
|
||||
golden reproduces byte-identically, so the two changes do not interact.
|
||||
|
||||
**Shape:** a **list** of regexes, not a single string, because Radarr needs two
|
||||
(`^/api/` and `^/signalr/`) and `--api-route` is natively repeatable. A single string would have
|
||||
forced alternation into the value, and alternation is exactly how breadth gets smuggled past a
|
||||
prefix-only reading of a regex.
|
||||
|
||||
**The anchoring ratchet.** Each entry must match `^\^/[A-Za-z0-9._~/-]+(\.\*|\$)?$` — anchored at
|
||||
`^/`, then literal path characters, with an optional trailing `.*` or `$`. The same pattern is in
|
||||
the schema, in the renderer, and in the new check, and assertion (5) of the check **fails if they
|
||||
drift apart**, because if they do the loosest one wins silently. Rejected shapes, all five
|
||||
provoked: `^/`, `^/.*`, `.*`, `/api/` (unanchored), `^/api/|^/` (alternation). Alternation is
|
||||
**refused rather than supported**: the flag is repeatable, so one entry per route costs nothing.
|
||||
|
||||
**Why breadth is the dangerous direction, and it is not symmetric.** A *missing* `--api-route` is a
|
||||
silent 302 — bad, invisible, recoverable. A *too broad* one answers a plain **page load** with 401:
|
||||
a logged-out operator gets a bare error page and is **never offered a sign-in**, so the service
|
||||
looks permanently broken and no reload recovers it. That is a worse outcome than the defect the
|
||||
flag fixes, which is why it is gated on three surfaces rather than documented in one.
|
||||
|
||||
### Also folded in: the selftest could exit 1 with ZERO diagnostics
|
||||
|
||||
Not my change originally — Desmond Oyelaran found it while wiring `catalog/render.py --selftest`
|
||||
into the `render-golden` stage (his PR #33, which touches only `ci/render-golden.sh` and is **not**
|
||||
in my base; `origin/main` at `cdf3e14` does not yet drive the selftest from any gate stage). The
|
||||
coordinator asked me to fold the fix into this PR because the defect is in the file I own.
|
||||
|
||||
`_selftest()` reached `_cookie_name()` through a bare `next(genexpr)` with **no default**. Delete
|
||||
the `--cookie-name` emission — exactly the mutation that assertion exists to catch — and it raises
|
||||
`StopIteration`, which escapes `_selftest()` **before its collected `SELFTEST FAIL:` lines are
|
||||
printed**. Right verdict, no diagnostics, and it masks every other assertion the same mutation
|
||||
broke. Fixed with `next(..., None)` plus an explicit `check()` so a missing flag reports **as a
|
||||
named assertion**.
|
||||
|
||||
Provoking it turned up **three more instances of the same failure class** in the same block, which
|
||||
is why this is not literally a one-line change: `side = fa_containers[1]`,
|
||||
`side["env"][0][...]`, and the `fa_svc` lookup all raise rather than assert when the sidecar is
|
||||
absent entirely. All four are now None-tolerant, and the two fixtures I added for `--api-route`
|
||||
were written the same way from the start. Measured, on throwaway copies:
|
||||
|
||||
| Mutation | Before | After |
|
||||
|---|---|---|
|
||||
| `--cookie-name` emission deleted (origin/main's renderer) | rc 1, **0** `SELFTEST FAIL` lines, 1 `StopIteration` | rc 1, **3** `SELFTEST FAIL` lines, 0 tracebacks |
|
||||
| the oauth2-proxy sidecar not appended at all | rc 1, **0** lines, `IndexError` | rc 1, **20** `SELFTEST FAIL` lines, 0 tracebacks |
|
||||
|
||||
Note the consequence of his stage change for anyone reading this later: once PR #33 lands,
|
||||
`.gitea/workflows/gate.yml:138` runs `./ci/gate.sh || [ "$?" -eq 3 ]`, so **rc 1 is not tolerated
|
||||
in CI the way rc 3 is** — a selftest failure will block. Which is the right outcome, and it is why
|
||||
a selftest that prints nothing was worth fixing before that lands rather than after.
|
||||
|
||||
### Declined, with reasons: the metrics-ingress NetworkPolicy
|
||||
|
||||
The coordinator asked for my judgement on bundling Ines Dubois's finding — that #28's metrics
|
||||
listener genuinely serves (she measured `http=200`, 10,208 bytes,
|
||||
`oauth2_proxy_requests_total{code="302"} 2` via port-forward) but the scrape will be **dropped**,
|
||||
because namespace `jackett`'s default-deny has **no `ingress` key at all** and
|
||||
`jackett-allow-gateway-ingress` admits `fromEntities: [ingress]` only. The fix belongs in each
|
||||
app's own rendered policy, i.e. in my file and my goldens. **I declined to bundle it**, and asked
|
||||
for it as the immediate next PR. Five reasons, in the order that decided it:
|
||||
|
||||
1. **The real reason is that it is a MODEL question, not a policy to bolt on.** Schema 1.6.0
|
||||
already owns this: `observability.metricsEndpoint` renders the Service port **and** the
|
||||
monitoring-namespace ingress allow. What the model cannot currently express is a **second**
|
||||
metrics endpoint on the same pod belonging to a *sidecar the renderer injected*, rather than to
|
||||
the workload the operator declared. Keying a bespoke policy off the pod label
|
||||
`homelab.keithjackson.dev/auth-mode: forward-auth` works today, but it hardcodes "forward-auth
|
||||
implies a metrics port on 44180" into a policy instead of **deriving** it from the sidecar
|
||||
declaration that is already the single source of that port — the exact "list, don't derive"
|
||||
anti-pattern `oauth2-proxy-cookie-names-distinct` assertion (5) exists to forbid, and which I
|
||||
would be introducing one directory away from the check that forbids it. That deserves its own
|
||||
review, and it is squarely the kind of decision I was brought in to make deliberately.
|
||||
2. **Different subject, and it is a trust-boundary change.** Mine is the auth *response code* on
|
||||
API paths (R-IDP-02). That is an ingress allow (R-NET-01/02, R-OBS-01) admitting a new
|
||||
namespace to a new port. Bundling a security-relevant ingress rule into a PR whose headline is
|
||||
`--api-route` is how such a rule gets waved through on the strength of the other half — and this
|
||||
PR already needs a human for a *different* sensitive-path reason, so the second subject would be
|
||||
reviewed in the shadow of the first.
|
||||
3. **The conflict argument does not actually apply to my diff.** It assumes I am regenerating five
|
||||
goldens; **I am regenerating two** (jackett, radarr), because the flag renders only where
|
||||
declared. A metrics-ingress policy renders a *different document* — a CiliumNetworkPolicy — in
|
||||
all five. Different hunks, in files git merges independently; and goldens regenerate
|
||||
deterministically from the renderer, so the follow-up rebases onto `main` cleanly. The N-1
|
||||
rebase problem `decisions/README.md` records was two branches appending to **the same hunk**,
|
||||
which this is not. I checked my own diff before answering rather than accepting the premise.
|
||||
4. **Every fact in the finding is hers and I have measured none of it** — not the 200/10,208 bytes,
|
||||
not the `wget: download timed out` on `:44180` *and* `:4180`, not the Flux sibling ordering that
|
||||
makes `oauth2-proxy` and `apps-wave-0` unordered. Shipping a security-adjacent rule on wholly
|
||||
inherited evidence, as a footnote in someone else's PR, is precisely what CLAUDE.md §2b exists
|
||||
to stop. In its own PR I would measure it first.
|
||||
5. **Sequencing: the PodMonitor is in HER PR #29, not mine.** If I ship the policy and her
|
||||
PodMonitor is reworked or delayed, I have opened an ingress allow for a scraper that selects
|
||||
nothing — a hole with no consumer. The policy is only useful paired with the PodMonitor, which
|
||||
argues for it landing adjacent to that, not here. It also drags in a
|
||||
`KNOWN_EXTERNAL_MONITORS` entry in `conformance/tests/l0-static/servicemonitor-ports-resolve/`,
|
||||
a second conformance interaction on top of the new check this PR already adds.
|
||||
|
||||
Diff size is the tiebreaker rather than the argument: **1,245 insertions across 13 files**,
|
||||
including a ~400-line new conformance check. I would rather hand a reviewer one subject they can
|
||||
actually falsify. **Queued, not dropped** — and if the next person takes it, the model note in (1)
|
||||
is the part worth arguing about, not the YAML.
|
||||
|
||||
### Which apps got which routes, and the measurement behind each
|
||||
|
||||
| App | `auth.apiRoutes` | Evidence |
|
||||
|---|---|---|
|
||||
| **jackett** | `^/api/` | 2026-09-30, its own access log: **33 × 302 against 2 × 401** on `/api/` paths, and **4 of 6 POSTs answered by the proxy and never reaching Jackett** (upstream field `-`, not `/`), including both `/api/v2.0/server/update` calls |
|
||||
| **radarr** | `^/api/`, `^/signalr/` | `^/signalr/`: measured **by me, on the live post-fix pod** — see below. `^/api/`: measured on radarr by the PR #30 author, on the **pre-fix** pod |
|
||||
| **pihole** | *(none — deliberate)* | serves no `/api/`; **0** unauthenticated requests in its current pod's 23 h |
|
||||
| **sonarr** | *(none — not measured)* | 13,643 log lines, **0** unauthenticated requests. No traffic, not a negative result |
|
||||
| **qbittorrent** | *(none — not measured)* | 13,649 log lines, **0** unauthenticated requests. Same |
|
||||
|
||||
**`^/signalr/` on radarr is the live proof this change was needed**, and it is the strongest
|
||||
evidence in this entry. 23 h after PR #28 deployed, on `radarr-7d55cbffb9-42hmh` (0 restarts), the
|
||||
sidecar logged **232 unauthenticated requests across the pod's entire lifetime, and all 232 are the
|
||||
same call**: `POST /signalr/messages/negotiate`, browser UA, **302**, upstream `-` (Radarr never saw
|
||||
one of them). That is a **left-open browser tab** whose session the cookie rename orphaned, polling
|
||||
SignalR forever, silently redirected every time, and **never re-prompting the operator**. It is the
|
||||
reported symptom — "reloading never fixes it and I am never re-prompted" — still happening in
|
||||
production, on a service nobody was looking at. With 401 the UI's connection breaks **loudly**; the
|
||||
operator reloads; `/` matches neither regex, so the reload signs them back in.
|
||||
|
||||
**`^/api/` on radarr: included, and here is the honest basis.** The current pod has logged **zero**
|
||||
`/api/` traffic, so I did **not** re-measure it. It rests on a measurement of **radarr** recorded in
|
||||
the merged PR #30 handoff — its 156 unauthenticated requests were `/api/v3/command` (62),
|
||||
`/api/v3/queue/details` (46) and `/signalr/messages/negotiate` (43), i.e. **108 of 156 were
|
||||
`/api/v3` XHRs**. That is this app's own evidence, by another engineer, not an analogy to another
|
||||
app. I include it because `/api/v3/` is a pure JSON namespace a browser never navigates to (Radarr
|
||||
serves its UI at `/`, assets under `/Content/` and `/MediaCover/`), so the breadth risk is nil and
|
||||
the measured benefit is 70% of that app's rejection volume.
|
||||
|
||||
**Sonarr and qBittorrent were measured and deliberately left alone.** Sonarr is nearly the same
|
||||
codebase as Radarr, which is exactly what makes `^/api/` + `^/signalr/` a tempting guess and still a
|
||||
guess. I read both pods' logs: **zero** unauthenticated requests, nothing but health probes, because
|
||||
nobody has opened either UI since the cookie fix rolled. **That is evidence of no traffic, not
|
||||
evidence of no need.** Each catalog file now carries a comment saying so, with the numbers and what
|
||||
to measure before adding the routes.
|
||||
|
||||
### Goldens and fixtures regenerated
|
||||
|
||||
Rendered with the canonical command
|
||||
(`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation`).
|
||||
**Two goldens changed, not five** — and that is the per-app property proving itself on the real
|
||||
tree rather than in a fixture:
|
||||
|
||||
apps/wave-0/jackett.yaml + --api-route=^/api/
|
||||
apps/wave-1/radarr.yaml + --api-route=^/api/
|
||||
+ --api-route=^/signalr/
|
||||
|
||||
`apps/wave-0/pihole.yaml`, `apps/wave-0/qbittorrent.yaml` and `apps/wave-1/sonarr.yaml` are
|
||||
**byte-identical to `main`**. **No L1 fixture needed regenerating**: all three
|
||||
`conformance/tests/l1-rendering/*/fixtures/svcdef*.yaml` declare `auth.mode: forward-auth` with no
|
||||
`apiRoutes`, so they render no flag and their goldens are unchanged. PR #28 touched 5 goldens and 3
|
||||
L1 fixtures because it changed flags on *every* sidecar; this one changes flags only where a
|
||||
measurement says so, which is the whole point.
|
||||
|
||||
### The L0 decision (and it is a decision, not a default)
|
||||
|
||||
**Not** in `oauth2-proxy-cookie-names-distinct`. That check's subject is cookie **identity** — its
|
||||
inputs are names and domains, its failure mode is mutual session eviction, its register is the
|
||||
shared proxy's claim on the default name. `--api-route` shares none of those. Bundling them would
|
||||
make that check's name a lie and couple two unrelated ratchets, so a future api-route decision could
|
||||
only be expressed by weakening a *cookie* check. That is PR #28's own stated discipline, applied to
|
||||
my change rather than quietly dropped.
|
||||
|
||||
**A sibling check, `oauth2-proxy-api-routes-per-app`** (whole-repo scope, so it runs on every gate
|
||||
invocation via `ci/l0.sh`'s `--include-scope` union — no bespoke stage, which
|
||||
`whole-repo-invariants-wired` would fail). **Nowhere was the wrong answer for exactly one reason:**
|
||||
the existing gates cannot see a renderer constant. `render-golden` compares the render to the
|
||||
goldens, so a constant **plus regenerated goldens agree with each other while both being wrong** —
|
||||
I provoked that and it regenerates with **rc 0**. The new check's assertion (2) compares each
|
||||
rendered sidecar against its **own ServiceDefinition**, which is the only reading that fails it.
|
||||
|
||||
## What I verified
|
||||
|
||||
- **`--api-route` is SINGULAR, read off the PINNED binary rather than taken from the brief.**
|
||||
`podman run --rm quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 --help`:
|
||||
`--api-route strings return HTTP 401 instead of redirecting to authentication server if token
|
||||
is not valid. Format: path_regex`. There is **no `--api-routes`** in the help output. The PM asked
|
||||
me to verify rather than trust, and that is the result — the singular form is confirmed and the
|
||||
help text also confirms the *semantics* I am relying on (401 instead of a redirect).
|
||||
- **The premise, in-cluster, read-only, through the pinned toolbox, `--context k3s-prod` named
|
||||
explicitly** (CLAUDE.md §2b and §5). PR #28 **is** deployed: all five sidecars carry
|
||||
`--cookie-name=_oauth2_proxy_<app>`, `--cookie-domain=<app>.home.keithjackson.dev`,
|
||||
`--cookie-refresh=2m30s` and `--metrics-address=0.0.0.0:44180`. Pods, all **0 restarts**:
|
||||
`jackett-6877fcd9f5-ht9d9`, `pihole-…`, `qbittorrent-5547bdb9bd-zq557`,
|
||||
`radarr-7d55cbffb9-42hmh`, `sonarr-ddf485775-8ckwt` (started 2026-09-30T21:41–21:42Z).
|
||||
- **The full post-fix log census, parsed rather than grepped** (oauth2-proxy's access-log format,
|
||||
with `/ping` health probes excluded), over each pod's whole life:
|
||||
|
||||
app lines unauth AuthSuccess removing session cookie sig invalid
|
||||
jackett 13,649 0 0 0 0
|
||||
pihole 13,649 0 0 0 0
|
||||
qbittorrent 13,649 0 0 0 0
|
||||
radarr 14,099 232 0 0 0
|
||||
sonarr 13,643 0 0 0 0
|
||||
|
||||
Three findings, kept separate:
|
||||
1. **The collision fix works.** Jackett was 39 unauthenticated before; it is **0** in 13,612
|
||||
non-probe lines. The eviction fingerprint (`removing session` / `cookie signature not valid`)
|
||||
is **0 on all five**, where radarr previously had 8 and 8.
|
||||
2. **Radarr's 232 are one request.** 100% `POST /signalr/messages/negotiate`, 100% **302**, 100%
|
||||
upstream `-`, browser UA (`Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537…`). **Zero `/api/`
|
||||
traffic of any kind** on this pod.
|
||||
3. **0 `AuthSuccess` anywhere**, so nobody has signed in since the restart and `--cookie-refresh`
|
||||
is **still** unexercised — unchanged from PR #28's own caveat.
|
||||
- **Only two goldens change, and three are byte-identical to `main`.** `git status` after the
|
||||
canonical render lists exactly `apps/wave-0/jackett.yaml` and `apps/wave-1/radarr.yaml`.
|
||||
`render-golden` is green.
|
||||
- **`catalog/render.py --selftest`: PASS**, with new assertions that (a) the `fa` fixture, which
|
||||
declares nothing, renders **no** `--api-route`; (b) a fixture declaring two routes renders exactly
|
||||
those two, in order; (c) the renderer **refuses** all five breadth shapes; (d) four legitimate
|
||||
routes (`^/api/`, `^/signalr/`, `^/api/v3/.*`, `^/api/v2.0/server/update$`) are **accepted** — a
|
||||
ratchet that blocks real API prefixes is a ratchet someone eventually deletes.
|
||||
- **The new check: rc 0, 43 assertions, 6 sidecars audited**, with its inventory printed so the
|
||||
reviewer can see what it read:
|
||||
`jackett -> ['^/api/'] | pihole -> (none) | qbittorrent -> (none) | radarr -> ['^/api/',
|
||||
'^/signalr/'] | sonarr -> (none) | crafty -> (none)`.
|
||||
- **The check was tested in BOTH directions, on throwaway copies outside the repo**, never by
|
||||
mutating the branch. Each mutation fired its own assertion class and nothing else:
|
||||
- **Pre-change tree** (`git archive origin/main` + the test dropped in): **rc 1, 11 failures** —
|
||||
no service declares routes, the renderer emits none, the renderer does not read `auth.apiRoutes`,
|
||||
the pattern is absent from both renderer and schema, and the shipped renderer accepts all five
|
||||
breadth shapes.
|
||||
- **pihole's golden hand-edited** to carry a route it never declared: **rc 1, 2 failures** (the
|
||||
catalog↔golden mismatch, and "declares nothing must render nothing").
|
||||
- **radarr's golden widened** to `--api-route=^/`: **rc 1, 2 failures** (unanchored, plus the
|
||||
mismatch).
|
||||
- **Renderer turned into a constant and EVERY golden regenerated to match: the regen exits 0** —
|
||||
i.e. `render-golden` is **happy** — and the new check exits **1 with 15 failures**, naming
|
||||
pihole, qbittorrent, radarr and sonarr individually. **This is the case that justifies the
|
||||
check's existence** and I provoked it rather than argued it.
|
||||
- **Schema `items.pattern` deleted**: **rc 1, 1 failure** — the three-surface drift guard.
|
||||
- **Unmutated control: rc 0.**
|
||||
- **The selftest diagnostics fix was PROVOKED, not reasoned.** Both mutations above were run
|
||||
against throwaway copies of the renderer outside the repo (`origin/main`'s `render.py` for the
|
||||
"before" column, mine for the "after"), never by mutating the branch. The pre-fix StopIteration
|
||||
and the post-fix 3-and-20 named assertions are quoted from those runs. `--selftest` on the
|
||||
unmutated branch: **PASS**. Goldens after the fix: **unchanged** (it touches only `_selftest()`).
|
||||
- **The check is selected by the whole-repo union.** `./conformance/run.sh --layer l0 --requirements
|
||||
r-nosuchreq-99 --include-scope whole-repo --print-selection` lists
|
||||
`tests/l0-static/oauth2-proxy-api-routes-per-app` among **21** tests (it was 20), i.e. it runs
|
||||
even when the touched-requirement filter matches nothing.
|
||||
- **THE SCHEMA GUARD IS PROVEN WITH THE PINNED VALIDATOR `ci/l0.sh` ACTUALLY DRIVES**, not by
|
||||
reading the pattern. `./schemas/validate.sh` (`check-jsonschema, version 0.29.4`) against seven
|
||||
throwaway ServiceDefinitions written outside the repo:
|
||||
|
||||
good ["^/api/", "^/signalr/"] rc 0
|
||||
bad-root ["^/"] rc 1 apiRoutes ... does not match
|
||||
bad-dotstar ["^/.*"] rc 1 apiRoutes ... does not match
|
||||
bad-unanchored ["/api/"] rc 1 apiRoutes ... does not match
|
||||
bad-alt ["^/api/|^/"] rc 1 apiRoutes ... does not match
|
||||
bad-everything [".*"] rc 1 apiRoutes ... does not match
|
||||
bad-mode ["^/api/"] on mode: oidc rc 1 const 'forward-auth'
|
||||
|
||||
So a too-broad regex is rejected **where a human types it**, on every gate run, by the real tool
|
||||
— and the shape invariant fires too.
|
||||
- **THE RENDERER'S REFUSAL REACHES `render-golden` AS A READABLE FAILURE, NOT A TRACEBACK.** With
|
||||
`^/signalr/` swapped for `^/` in a *copy* of the catalog (outside the repo), both invocation
|
||||
shapes exit **rc 1 with 0 tracebacks** — the single-SD form and `--all <dir> -o <dir>
|
||||
--wave-layout --no-image-automation`, which is the one `ci/render-golden.sh` drives:
|
||||
|
||||
render.py: radarr: auth.apiRoutes entry '^/' is not an anchored path regex. Each entry
|
||||
must start with '^/' and continue with literal path characters (e.g. '^/api/',
|
||||
'^/signalr/', '^/api/v3/.*'). ... A too-broad --api-route answers a PAGE LOAD with 401, so
|
||||
a logged-out operator sees a bare error page and is never offered a sign-in: the service
|
||||
looks permanently broken rather than merely unauthenticated.
|
||||
|
||||
So the breadth guard is gated by `render-golden` even on a run where the schema stage skipped,
|
||||
and the operator is told *why* rather than handed a stack trace.
|
||||
- **The schema's new shape invariant.** A non-empty `auth.apiRoutes` requires
|
||||
`auth.mode: forward-auth`, because the flag renders onto the per-app sidecar and no sidecar exists
|
||||
for `oidc`/`native`/`none` — declaring it elsewhere would be a **silent no-op**, which is the
|
||||
failure class this repo keeps paying for.
|
||||
- **Gate:** toolbox `./ci/gate.sh` with the two-mount ENGINE shim — exit code and stage list in the
|
||||
next section, because it is not a pass and that belongs there.
|
||||
- **No mutating cluster access.** `kubectl get/logs/jsonpath` only, `--context k3s-prod` named on
|
||||
every call. No `apply`/`edit`/`patch`, no `flux reconcile`. No Secret `.data` or `.stringData` was
|
||||
read or printed; I did not need to touch a Secret at all this session. The one container I ran
|
||||
locally was the pinned oauth2-proxy image, to read its `--help`.
|
||||
- **`~/.git-credentials` was never read.** PR creation used `git credential fill`, which does not
|
||||
echo the value.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **NO BROWSER ROUND-TRIP. BY ANYONE. FOR THIS FLAG.** This is the most important line here. The
|
||||
302/401 split, the dropped POST bodies and the orphaned SignalR poll are all read from access
|
||||
logs; that a **401 repairs** the operator's experience is reasoned from HTTP semantics, not
|
||||
watched. Nobody has added a Jackett indexer or recovered a Radarr tab through this
|
||||
configuration. The human test below is the first real evidence this change will ever have.
|
||||
- **Local gate: rc 3 (UNDETERMINED), not 0.** Green: `spec-immutability`, `decisions-log`, `lint`,
|
||||
`l0`, `render-golden`, `l1`. **Skipped:** `gitea-settings` (no `SETTINGS_API_TOKEN` locally) and
|
||||
**L2/L3 conformance** (`CLUSTER_AVAILABLE` unset). Nothing FAILED; nothing runtime was certified
|
||||
either. **rc 3 is UNDETERMINED, never green.**
|
||||
- **MY LOCAL `approval-policy: green` WAS MEANINGLESS, AND I ALMOST REPORTED IT AS EVIDENCE.**
|
||||
`ci/approval-policy.sh:47` reads `GITHUB_EVENT_NAME`, which is unset outside CI, so the stage
|
||||
short-circuits at line 50: `ok "not a pull_request event (event='none') — approval policy N/A"`.
|
||||
**A clean local gate therefore CANNOT catch a sensitive-path problem — structurally, not by
|
||||
accident.** Flagged to me by the coordinator from Ines Dubois's round, and I verified it by
|
||||
re-running the gate with `GITHUB_EVENT_NAME=pull_request PR_NUMBER=34`: the stage then
|
||||
**evaluates** and fails closed on the missing token —
|
||||
|
||||
[fail] no SETTINGS_API_TOKEN — cannot read this PR's changed files or its reviews.
|
||||
[fail] stage approval-policy: FAIL
|
||||
[fail] approval-policy: a sensitive change lacks a non-author approval -> blocks merge
|
||||
[fail] MERGE GATE: BLOCKED (rc 1, not 3)
|
||||
|
||||
— while **every other stage stays green** (`spec-immutability`, `decisions-log`, `lint`, `l0`,
|
||||
`render-golden`, `l1`). That is the useful result: it bounds the CI failure to this one stage
|
||||
rather than leaving "something is red" as the whole report. Fail-closed is correct here and
|
||||
deliberate: unlike `gitea-settings`, where a missing credential is a lock-out, a skip of this
|
||||
stage would be a bypass (CLAUDE.md §1).
|
||||
- **CI, read from each commit's `/statuses` rather than from a tick.** Both contexts report, and
|
||||
**only one of them runs**. Identical on `99560c2d`, `df1b25b6` and `f1d16f44`:
|
||||
|
||||
gate / gate (pull_request) state=failure desc='Failing after ~1m43-1m45s'
|
||||
gate / conformance (pull_request) state=success desc='Has been skipped'
|
||||
|
||||
**That second line is §2b's trap in the wild**: a *skipped* Gitea job reports
|
||||
`state=success`. The conformance job did not execute, so its green tick certifies nothing —
|
||||
read the description, not the colour. The duration is given as a range on purpose: it differs
|
||||
by a second or two per run, and pinning one number inside the file whose commit triggers the
|
||||
next run is a self-reference, not a measurement. The load-bearing fields are the **state** and
|
||||
the **description**, and those are stable across all three.
|
||||
|
||||
Worth recording that the **same context changed its answer over time**: on the first push it
|
||||
briefly read `state=pending desc='Blocked by required conditions'` — still waiting on the gate
|
||||
job it depends on — and resolved to `success` only once that job settled. So sampling a status
|
||||
once can show either a pending **or** a green tick for a job that never executed.
|
||||
**I could NOT read the CI job log to confirm the failing stage from CI's
|
||||
own output**: this Gitea's `/api/v1/repos/{owner}/{repo}/actions/runs` endpoint answers
|
||||
**404** (only `/actions/tasks` exists, which lists tasks without logs), and
|
||||
`/actions/jobs/{task_id}/logs` answers `404 Not found` for the task id. So my attribution of the
|
||||
CI failure to `approval-policy` rests on the local event-set reproduction above plus the
|
||||
sensitive-path rule, **not** on CI's own log. That is weaker evidence than PR #28 had and I am
|
||||
naming it rather than borrowing its wording.
|
||||
- **Branch protection and the runner, RE-MEASURED this session against the live API** (§2b),
|
||||
not inherited from CLAUDE.md §2's table:
|
||||
|
||||
GET /repos/Home-Network/home-network-k3s-impl/branches/main
|
||||
protected True
|
||||
user_can_push False
|
||||
enable_status_check True
|
||||
status_check_contexts ['gate / gate (pull_request)']
|
||||
required_approvals 1
|
||||
GET /admin/actions/runners
|
||||
id=1 name=incluster-dind status=online
|
||||
|
||||
So `required_approvals=1` applies to **every** PR here regardless of path — this needs a human
|
||||
even without the sensitive-path rule. I did **not** provoke a rejected push to `main` this
|
||||
session; `user_can_push: False` is the API's word, which is exactly the weaker form of evidence
|
||||
CLAUDE.md §2 warns about, and I did not strengthen it.
|
||||
- **`^/api/` ON RADARR IS NOT MY MEASUREMENT.** It is PR #30's, taken on the **pre-fix** pod. The
|
||||
current pod has logged **zero** `/api/` traffic in 23 h, so I could not reproduce it. I judge the
|
||||
breadth risk nil (`/api/v3/` is pure JSON; the UI is at `/`), but if someone wants a strictly
|
||||
self-measured configuration, drop `^/api/` and keep `^/signalr/` — the 232 are entirely SignalR.
|
||||
- **SONARR AND QBITTORRENT ARE UNMEASURED, NOT CLEARED.** Zero traffic in 23 h is zero evidence.
|
||||
qBittorrent's WebUI drives everything through `/api/v2/` and is *plausibly* affected the same way
|
||||
Jackett is; Sonarr is nearly Radarr's codebase. **Both are left alone on purpose** — plausibility
|
||||
is what put a renderer constant on the table in the first place. Expect to add `^/api/` to
|
||||
qBittorrent and `^/api/` + `^/signalr/` to Sonarr once someone opens those UIs and records the
|
||||
split. **This PR does not fix them.**
|
||||
- **PiHole is asserted to serve no `/api/` on inherited evidence, not mine.** Its log shows 0
|
||||
unauthenticated requests, which says nothing about what paths it serves. The claim comes from
|
||||
PR #28/#30. Note the pinned `update.range` is `>=2024.0.0`, and PiHole v6 *does* introduce a REST
|
||||
API at `/api/` — so if PiHole is ever upgraded across that line, this needs re-measuring. It is
|
||||
still correct to declare nothing today, because declaring an unmeasured route is the mistake.
|
||||
- **`--cookie-refresh=2m30s` has STILL never been exercised** — 0 `AuthSuccess` on all five pods.
|
||||
PR #28 named this as the fix most likely to need a second pass and it remains unobserved. My
|
||||
change does not touch it, but a sign-in during the human test below will exercise both at once,
|
||||
so a failure there must be attributed carefully.
|
||||
- **The regexes are not validated against the apps' real route tables.** I assert `/api/v3/` and
|
||||
`/signalr/` are JSON/RPC namespaces a browser never navigates to from Radarr's and Jackett's
|
||||
observed traffic and from how those UIs are built — **not** from reading their routing code or
|
||||
enumerating their endpoints. A path I have not thought of that matches `^/api/` and *is*
|
||||
browser-navigated would 401.
|
||||
- **`OPTIONS` preflight and non-browser clients are unconsidered.** `--skip-auth-preflight` exists
|
||||
and is not set; nothing in these logs shows a CORS preflight, and Sonarr/Radarr talk to Jackett
|
||||
**directly on `:9117`**, bypassing the proxy entirely — so I do not expect impact. Unverified.
|
||||
- **The live pods do not carry this change and nothing re-measures that they will.** git is not the
|
||||
cluster (§2b). **Merging is what deploys**, and it will roll **jackett** and **radarr** only
|
||||
(`strategy: Recreate` on RWO Longhorn volumes, so a brief gap per app, and radarr/sonarr lose
|
||||
direct `:9117` access to Jackett while its pod restarts). Existing sessions survive — no cookie
|
||||
flag changed — so unlike PR #28 nobody is signed out.
|
||||
- **No L2/L3 assertion reads any of this.** There is no runtime check that a `/api/` request to a
|
||||
gated app answers 401, and I did not write one; `CLUSTER_AVAILABLE` is unset so it could not run
|
||||
in CI anyway. The whole guard is static.
|
||||
- **`catalog/render.py --selftest` is UNGATED IN MY BASE, and I did not wire it.** Desmond's PR #33
|
||||
does that (`ci/render-golden.sh`) and is in review, not merged — `origin/main` at `cdf3e14`
|
||||
contains no stage that drives it, which I checked rather than assumed (`grep -n selftest
|
||||
ci/render-golden.sh` → no match). **So until #33 lands, my new selftest assertions are
|
||||
dev-time only**, and the *gated* half of my breadth ratchet is the schema `pattern` (via
|
||||
`ci/l0.sh`'s schema stage), the renderer's own `raise` (via `render-golden`), and the new L0
|
||||
check. I deliberately did not wire the selftest myself: #33 already does it, and duplicating it
|
||||
here would collide with his branch in the one file he owns. Note the two PRs are **independent
|
||||
but complementary** — his makes the stage real, mine makes its output readable when it fires.
|
||||
- **I did NOT verify PR #33's own claims.** The 106-assertion count, the
|
||||
selftest→stage→`gate.sh`→`BLOCKED` propagation chain, and the `--cookie-refresh 2m30s → 10m`
|
||||
mutation being caught by the selftest but not by `render-golden` are all his measurements,
|
||||
relayed to me by the coordinator. I reproduced only the StopIteration defect itself, because that
|
||||
is the part I was asked to fix. If his branch is reworked, nothing in mine depends on it.
|
||||
- **The new check does not read the spec repo** and asserts nothing about `../home-network-k3s`
|
||||
(R-TEST-06, read-only). `schemas/renderer-contract.md` is updated in **this** repo only; the
|
||||
canonical spec `docs/13-service-catalog.md` is **not** updated, because it is immutable from here.
|
||||
Someone with spec write access should mirror `auth.apiRoutes` into docs/13 and docs/15.
|
||||
- **THE OAUTH2-PROXY SIDECARS' METRICS ARE STILL UNSCRAPABLE, AND THIS PR DOES NOT FIX IT.** See
|
||||
the declined-bundle section above. #28 shipped `--metrics-address` as a prerequisite and said
|
||||
plainly that it was not scraped; Ines has now measured that the listener **serves** and that the
|
||||
scrape is **dropped by each app namespace's default-deny**. Nothing in my change touches that.
|
||||
Anyone reading this entry as "the fleet gates are observable now" is wrong, and the next PR is
|
||||
where that changes.
|
||||
- **Alternation is refused, and that is a judgement I may have to revisit.** No app in this catalog
|
||||
needs it, and the flag is repeatable — but if some future app genuinely needs a regex the ratchet
|
||||
rejects, the fix is to widen the pattern **with a reason** on all three surfaces at once, not to
|
||||
delete the ratchet on one.
|
||||
- **CI status: two contexts, and only one of them ran.** Quoted in the PR body and in the report to
|
||||
the PM. `approval-policy` **fails by design** until a non-author approving review exists, because
|
||||
the diff touches `conformance/**`, which `infra/gitea/approval-policy.json` lists as sensitive
|
||||
("the tests and the harness; changing them changes what green MEANS"). That is CLAUDE.md §1
|
||||
working, not a defect. `required_approvals=1` applies to every PR here regardless of path, so this
|
||||
needs a human either way.
|
||||
|
||||
### The human test — and how to tell a too-broad regex from a working one
|
||||
|
||||
After merge, wait for **new** pod names at 2/2 in `jackett` and `radarr`.
|
||||
|
||||
1. **The breadth test FIRST, before anything else, from a signed-out browser** (private window).
|
||||
Open `https://radarr.home.keithjackson.dev/` and `https://jackett.home.keithjackson.dev/UI/Dashboard`.
|
||||
**Each must bounce through Keycloak and land on the app's UI.** If either shows a bare
|
||||
`401 Unauthorized` instead of a sign-in, **the regex is too broad — stop and revert.** That is
|
||||
the one failure this change can introduce, and it is the only test that can see it. A working
|
||||
regex is invisible on a page load; a broken one is total and immediate.
|
||||
2. **Jackett, the original report:** signed in, *+ Add indexer* → pick one → *Configure* → *Okay*.
|
||||
**It must save.** Previously 4 of 6 POSTs never reached Jackett at all.
|
||||
3. **Radarr, the live symptom:** open Radarr and leave the tab open. DevTools → Network, filter
|
||||
`signalr`. Today those negotiate calls are **302** and the UI never complains. They must now be
|
||||
**200** while signed in — and if the session is ever lost, **401**, with the UI showing a visible
|
||||
connection error instead of polling silently forever.
|
||||
4. **The loud-failure test:** in DevTools, delete the `_oauth2_proxy_radarr` cookie and let the tab
|
||||
poll. The negotiate call must answer **401** and Radarr must *show* a problem. Then **reload** —
|
||||
`/` matches neither regex, so the reload must sign you back in silently. That round trip is the
|
||||
entire fix: fail loudly on the API, recover on the page load.
|
||||
5. **The six-minute test (PR #28's open item, still open):** stay on a dashboard for >6 minutes,
|
||||
then save something. That exercises `--cookie-refresh=2m30s` past the 300 s access-token cliff.
|
||||
If it fails, suspect the realm's refresh/SSO-idle timeouts — not this change.
|
||||
6. **If something still fails, it must now fail LOUDLY.** A silent success-looking no-op means the
|
||||
change did not take: check the running pod's args for `--api-route` before assuming anything
|
||||
else.
|
||||
@@ -1,205 +0,0 @@
|
||||
---
|
||||
date: 2026-10-01
|
||||
branch: fix/render-selftest-runs-in-gate
|
||||
title: catalog/render.py's 106 assertions were dev-time only; the gate now runs them and blocks on them
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
Single subject, one file: **`ci/render-golden.sh` now invokes `catalog/render.py --selftest`**
|
||||
and exits 1 when it fails. Nothing else. No renderer change, no new conformance test, no new
|
||||
gate stage.
|
||||
|
||||
This closes the top follow-up of `decisions/2026-09-30-oauth2-proxy-cookie-collision-fleet.md`:
|
||||
|
||||
> Nothing in `ci/` or `.gitea/` runs `catalog/render.py --selftest` — I grepped for it. So the
|
||||
> twenty-odd assertions in that selftest, including the six I just added, are dev-time only: a
|
||||
> future edit that drops either flag *and* re-renders the goldens would pass the entire merge
|
||||
> gate.
|
||||
|
||||
Re-measured here and confirmed: before this commit, `grep -rl -- --selftest` over `ci/` and
|
||||
`.gitea/` returned **nothing** for the renderer. The selftest executes **106** assertions
|
||||
(`check()` invocations counted at runtime via `sys.settrace`, without editing `render.py`; 109
|
||||
static call sites, three of them on failure-only branches). All 106 were unexercised by the
|
||||
merge gate.
|
||||
|
||||
### The lead was WRONG, and here is why
|
||||
|
||||
The brief named `conformance/tests/l0-static/auditability-proofs-producible/check.sh:291` as the
|
||||
one-line fix. That line is real and it does drive selftests:
|
||||
|
||||
for tool in ("record_run.py", "status.py"):
|
||||
r = run([sys.executable, str(LIB / tool), "--selftest"])
|
||||
|
||||
But it is **the wrong hook**. That loop is section (d) of a check that serves **R-SEC-03**, and
|
||||
its subject is the *evidence chain* — can a self-skipped run, or a stale ledger row, become an
|
||||
audit proof? `record_run.py` and `status.py` are **in** that chain; `catalog/render.py` is not.
|
||||
Hanging the renderer there would have made the check's name, its `intent` annotation and its
|
||||
requirement mapping (`R-SEC-03`) all untrue, and a future decision about the renderer could then
|
||||
only be expressed by weakening a *security-auditability* check. That is the precise coupling the
|
||||
cookie-collision entry refused for its own check, and it applies here.
|
||||
|
||||
The second option that entry offered — a new scope-declaring L0 test — is the supported mechanism
|
||||
for a **whole-repo invariant** (`ci/l0.sh` Stage 3b's `--include-scope whole-repo` union, ratcheted
|
||||
by `tests/l0-static/whole-repo-invariants-wired`). It is the right shape for an invariant
|
||||
quantified over the tracked tree. It is the wrong shape for *"a script in this repo must pass its
|
||||
own unit test"*: it would need to declare requirements it does not serve, and it would be two new
|
||||
files of several hundred lines by this tree's conventions for what is a six-line invocation.
|
||||
|
||||
**The honest home is `ci/render-golden.sh`** — the gate stage whose subject *already is*
|
||||
`catalog/render.py`. It already resolves `$PYTHON` and `$RENDER_PY`, already reports rc 3 when
|
||||
either is missing, already exits 1 on renderer error, and `ci/gate.sh` already maps it through
|
||||
`for s in lint l0 render-golden l1` where a FAIL is BLOCKED and a SKIP is UNDETERMINED. The
|
||||
stage's two assertions are independent and both are needed:
|
||||
|
||||
- `render-golden` catches golden-vs-renderer drift, but is satisfied by **any** renderer whose
|
||||
output was re-rendered — including one that just lost a contract invariant and had all ten
|
||||
goldens regenerated in the same commit. That is exactly the hole named in the quote above.
|
||||
- `--selftest` catches that, and is blind to golden drift.
|
||||
|
||||
The file header and the stage name were updated so neither is a lie: the stage is now "the
|
||||
renderer is trustworthy", with two numbered assertions, and the step summary emits its own
|
||||
`- render-selftest:` line so the gate output still NAMES what ran.
|
||||
|
||||
The selftest runs **before** the empty-catalog `exit 3` on purpose: it renders synthetic
|
||||
ServiceDefinitions in memory and is catalog-independent, so an empty catalog must not be able to
|
||||
skip it.
|
||||
|
||||
`catalog/render.py` itself was **not touched** (owned by a concurrent branch this session), and
|
||||
neither was any file under `conformance/`.
|
||||
|
||||
## What I verified
|
||||
|
||||
- **The selftest runs and passes: rc 0, `render.py selftest: PASS`, 106 assertions executed.**
|
||||
Counted at runtime, not from the source: a `sys.settrace` hook counting `check()` calls, in a
|
||||
throwaway script outside the repo, with no edit to `render.py`.
|
||||
- **The stage passes with it wired in**, host run of the single stage:
|
||||
|
||||
[gate] Stage: render==golden reproducibility (canonical render vs committed apps/)
|
||||
[gate] Stage: catalog/render.py --selftest (the renderer's own output-contract assertions)
|
||||
[ ok ] render.py --selftest passed
|
||||
[ ok ] render==golden: all 10 onboarded service golden(s) reproduce from the canonical render
|
||||
|
||||
rc **0**. The golden comparison still reports all 10 goldens, so the new invocation did not
|
||||
displace what the stage already asserted.
|
||||
- **IT CAN FAIL — provoked twice, against a mutated COPY of the renderer, never against the
|
||||
tree.** `ci/render-golden.sh` already honours `RENDER_PY`, so each mutant lived in
|
||||
`/tmp/claude-1000/desmond-selftest/` and the repo's `catalog/render.py` was never modified —
|
||||
which matters, because that file belongs to a concurrent branch this session.
|
||||
- **Mutation 1 — the regression this selftest was written for.** Deleted the
|
||||
`f"--cookie-name=_oauth2_proxy_{s['name']}"` emission from `_oauth2_sidecar_container()`,
|
||||
i.e. reverted the fleet-wide cookie-collision fix of PR #28. Stage rc **1**. Note the shape
|
||||
of the failure, because it is a real (minor) weakness and is listed as a follow-up: the
|
||||
selftest died with an uncaught `StopIteration` out of `next(a for a in container["args"] if
|
||||
a.startswith("--cookie-name="))` at `render.py:2641`, rather than printing its collected
|
||||
`SELFTEST FAIL:` lines. It is correctly non-zero and the stage correctly blocks, but the
|
||||
operator gets a traceback instead of the assertion message.
|
||||
- **Mutation 2 — a clean, named assertion failure.** `OAUTH2_COOKIE_REFRESH = "2m30s"` ->
|
||||
`"10m"`, widening the refresh past the measured 300s access-token lifetime. Stage rc **1**:
|
||||
|
||||
[fail] catalog/render.py --selftest FAILED — the renderer violates its own output contract.
|
||||
[fail] Every golden under apps/ is produced by this renderer, so the goldens below cannot
|
||||
[fail] be trusted even if they reproduce byte-for-byte. Reproduce with:
|
||||
[fail] python3 catalog/render.py --selftest
|
||||
SELFTEST FAIL: --cookie-refresh=10m parses to 600.0s, which is not inside (0, 300s). A
|
||||
refresh at or past the access-token lifetime never renews in time, which is the same
|
||||
logout it was added to stop
|
||||
|
||||
This is the mutation that proves the wiring's value most directly: mutation 2 would **not**
|
||||
be caught by `render-golden`'s diff if the goldens were regenerated in the same commit, and
|
||||
it is one of the two flags the cookie-collision entry explicitly recorded as **ungated**.
|
||||
- **THE NON-ZERO PROPAGATES ALL THE WAY TO `gate.sh`'s rc — provoked, in the pinned toolbox, not
|
||||
reasoned about.** `RENDER_PY=/desmond-selftest/render-mutant2.py ./ci/gate.sh`:
|
||||
|
||||
[ ok ] stage spec-immutability: pass
|
||||
[ ok ] stage decisions-log: pass
|
||||
[ ok ] stage lint: pass
|
||||
[ ok ] stage l0: pass
|
||||
[gate] Stage: catalog/render.py --selftest (the renderer's own output-contract assertions)
|
||||
[fail] catalog/render.py --selftest FAILED — the renderer violates its own output contract.
|
||||
SELFTEST FAIL: --cookie-refresh=10m parses to 600.0s, which is not inside (0, 300s). ...
|
||||
[fail] stage render-golden: FAIL
|
||||
[ ok ] stage l1: pass
|
||||
...
|
||||
[fail] render-golden: FAILED -> blocks merge
|
||||
[fail] MERGE GATE: BLOCKED
|
||||
|
||||
`gate.sh` rc **1**. Three links in the chain, each measured: selftest rc 1 -> stage rc 1 ->
|
||||
gate rc 1 / `MERGE GATE: BLOCKED`. This mattered enough to provoke rather than assume: this
|
||||
repo has a documented habit of exit codes getting flattened, and
|
||||
`.gitea/workflows/gate.yml:138` runs `./ci/gate.sh || [ "$?" -eq 3 ]`, which tolerates rc 3 —
|
||||
so a stage that degraded a failure to a skip would be swallowed by CI. rc 1 is not tolerated
|
||||
there, and rc 1 is what this produces.
|
||||
- **`spec-immutability` genuinely RAN and passed, it did not skip.** The two-mount worktree shim
|
||||
plus `-e SPEC_REPO=/home-network-k3s` (the fix recorded in the 2026-09-30 cookie entry) works:
|
||||
`stage spec-immutability: pass`, not SKIPPED. R-TEST-06 is therefore actually covered by these
|
||||
runs.
|
||||
- **No collision with PR #29.** The brief warned that `feat/oauth2-proxy-auth-failure-visible`
|
||||
had added checks under `conformance/tests/l0-static/`. Measured —
|
||||
`git diff origin/main...origin/feat/oauth2-proxy-auth-failure-visible --name-only` — it touches
|
||||
**no `conformance/` path at all**: `decisions/`, `platform/oauth2-proxy/` (a promtool test, an
|
||||
alert rule, its kustomization), `runbooks/`. Disjoint from this branch either way, since this
|
||||
branch touches only `ci/render-golden.sh` and this file.
|
||||
- **Priya's files were not touched.** `catalog/render.py`, `schemas/servicedefinition.schema.json`,
|
||||
`apps/wave-*` and `conformance/tests/l1-rendering/` are all untouched by this branch — confirmed
|
||||
by `git diff --name-only`, which lists exactly two paths.
|
||||
- **`catalog/render.py --selftest` was the LAST unwired selftest in the tree.** Measured:
|
||||
`grep -rl -- --selftest` across the repo finds selftest-capable artifacts in exactly four
|
||||
places, and after this commit every one is driven by the gate — `conformance/lib/record_run.py`
|
||||
and `conformance/lib/status.py` by `auditability-proofs-producible/check.sh:291`, the Keycloak
|
||||
client-secret reconciler (embedded in `platform/keycloak/client-secret-reconciler.yaml`) by
|
||||
`keycloak-client-secret-reconciler/check.sh:633`, and `catalog/render.py` by this change.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **Local gate: rc 3 (UNDETERMINED), NOT 0. This is not green and must not be reported as
|
||||
green.** Measured in the pinned toolbox on `eeb6211`:
|
||||
|
||||
[ ok ] spec-immutability: green
|
||||
[ ok ] approval-policy: green
|
||||
[ ok ] decisions-log: green
|
||||
[ ok ] lint: green
|
||||
[ ok ] l0: green
|
||||
[ ok ] render-golden: green
|
||||
[ ok ] l1: green
|
||||
[fail] MERGE GATE: UNDETERMINED — greenness was NOT ESTABLISHED.
|
||||
[fail] Stages that did not run: gitea-settings
|
||||
|
||||
Nothing FAILED. The single cause is `gitea-settings` **SKIPPED** — `SETTINGS_API_TOKEN` is not
|
||||
set in my environment, so branch protection and runner registration were not read; CI has that
|
||||
token and the stage is expected to run there. **L2/L3 `conformance`** also skipped
|
||||
(`CLUSTER_AVAILABLE` unset) but is a *declared* gap that `gate.sh` treats as not-applicable
|
||||
rather than undetermined, so it is not what produced the 3. Touched requirements
|
||||
**R-GITOPS-01 R-TEST-02 R-TEST-03 R-TEST-06** are therefore **UNVERIFIED at L2/L3**, all four.
|
||||
The stage this branch changes, `render-golden`, is green **and** was independently proven able
|
||||
to turn the whole gate red (above).
|
||||
- **`approval-policy` passed LOCALLY only because there is no pull_request event**
|
||||
(`not a pull_request event (event='none') — approval policy N/A`). That is not evidence that it
|
||||
passes. This diff touches `ci/**`, a sensitive path, so in CI the stage is expected to **FAIL**
|
||||
until a non-author approving review exists. That is §1 working, fail-closed, not a defect.
|
||||
- **The selftest's own quality is unchanged and ungated.** I wired its *invocation*; I did not
|
||||
review its 106 assertions, and nothing ratchets them. Deleting the body of `_selftest()` would
|
||||
still pass this stage, because an empty selftest returns 0. The repo already has the right
|
||||
answer for that — `conformance/tests/l0-static/keycloak-client-secret-reconciler/mutations.py`
|
||||
injects regressions one at a time and requires `--selftest` to catch every one — and applying
|
||||
that pattern to `catalog/render.py` is the follow-up, deliberately not done here: it is a
|
||||
second subject, it touches `conformance/`, and it would need to live beside the renderer whose
|
||||
file another branch owns this session. **Until it exists, "the selftest runs" is proven and
|
||||
"the selftest is falsifiable" rests on my two hand-run mutations, which are not in the gate.**
|
||||
- **Mutation 1's traceback is not fixed.** `render.py:2641`'s `next(...)` has no default, so the
|
||||
absence of `--cookie-name` raises `StopIteration` and the selftest's collected `SELFTEST FAIL:`
|
||||
lines are never printed. Correct verdict, poor diagnostics, and it suppresses the other
|
||||
assertions that mutation also broke. The one-line fix (`next(..., None)` plus a `check()`) is in
|
||||
`catalog/render.py`, which is another branch's file this session, so it is a follow-up and not
|
||||
a thing I touched.
|
||||
- **Nothing about the live cluster.** No `kubectl` of any kind was run on this branch — not even
|
||||
read-only; this change is offline CI wiring and needed none. No `apply`/`edit`/`patch`, no
|
||||
`flux reconcile`. Merging this deploys nothing: it changes only what the gate refuses.
|
||||
- **CI's own run of this branch.** Quoted in the PR body from the commit's `/statuses`, with
|
||||
descriptions read rather than colours. A *skipped* Gitea job reports `success` with
|
||||
"Has been skipped" and certifies nothing.
|
||||
- **`vars.CLUSTER_AVAILABLE` is still unset in CI**, so even a fully green PR proves the L0/L1
|
||||
offline subset and nothing about runtime behaviour.
|
||||
- **I did not re-measure branch protection, required approvals, or the runner.** Those are
|
||||
CLAUDE.md §2 claims and §2b says re-measure before relying on them; this branch relies on none
|
||||
of them beyond the PR flow itself, and I read the gate's own output rather than the API.
|
||||
@@ -115,6 +115,14 @@ Each affected runbook says so on its own page and gives the by-hand equivalent.
|
||||
plex.tv/claim — so it is a timed sequence, scripted as `infra/plex/claim-ship.sh`:
|
||||
token → `sops` → PR → gate → merge → **forced** `flux reconcile` → pod roll (~2m45s measured).
|
||||
Includes why a changed Secret does not restart anything, and why an expired token is harmless.
|
||||
- **`plex-quicksync`** — hardware transcoding on the host iGPU. GitOps puts `/dev/dri` in the pod
|
||||
(`hardware.gpu: intel-quicksync`); **a human still has to tick Settings → Transcoder → "Use hardware
|
||||
acceleration when available"**, and no environment variable can do it (the image supports no
|
||||
`PLEX_PREFERENCE*`, verified). The page names the real preference key (`HardwareAcceleratedCodecs`,
|
||||
read out of the binary), says which widely-cited key **does not exist**, and is explicit that the
|
||||
**only** proof is `Transcode (hw)` on the Plex dashboard — Plex renders no ServiceMonitor, so no
|
||||
alert could ever fire. Also: the device-cgroup failure to expect first and its device-plugin
|
||||
fallback, and why a `:latest` pull can silently break it.
|
||||
- **`oauth2-proxy-session`** — a gated UI loads but its saves quietly do nothing, and you are never
|
||||
re-prompted to log in. Tells the three stacked auth layers apart from the sidecar's own access log
|
||||
(the four lines that matter, including the upstream `-` that means the proxy answered a request the
|
||||
|
||||
@@ -95,6 +95,12 @@ configMapGenerator:
|
||||
# because the plex.tv claim window is short and is not a thing to go looking
|
||||
# for in git: /runbooks/plex-claim has to resolve the moment it is needed.
|
||||
- plex-claim=plex-claim.md
|
||||
# Operator procedure, not an alert page, and NO alert is possible for it: plex renders no
|
||||
# metricsEndpoint, so there is no ServiceMonitor, no transcode-mode metric and nothing a rule
|
||||
# could evaluate. Served because the feature it completes is INERT until a human ticks a box in
|
||||
# the Plex UI — the one step GitOps cannot reach — and a page nobody can find is how a shipped
|
||||
# feature stays switched off for months.
|
||||
- plex-quicksync=plex-quicksync.md
|
||||
- pvc-capacity=pvc-capacity.md
|
||||
- targets-down=targets-down.md
|
||||
- update-rollback=update-rollback.md
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
# plex-quicksync — making Plex actually use Intel Quick Sync, and proving it did
|
||||
|
||||
**Alert:** none, and none is possible — see "How to verify" below. This is an **operator procedure**,
|
||||
not an incident page. It is indexed here because the feature it completes is **inert until a human
|
||||
clicks one button**, and a switched-off feature that looks shipped is exactly what this page exists
|
||||
to prevent.
|
||||
**Files:** `catalog/plex.yaml` (`hardware.gpu: intel-quicksync`), `apps/wave-0/plex.yaml` (the
|
||||
rendered golden — the `dri` hostPath volume and the `/dev/dri` mount), `catalog/render.py` (the
|
||||
mechanism), `schemas/renderer-contract.md` `### hardware` (the contract).
|
||||
**Related:** `runbooks/plex-claim.md` — the other Plex procedure with a manual last mile.
|
||||
|
||||
## What this means here
|
||||
|
||||
Plex transcodes in **software** by default. On the UHD Graphics 630 iGPU of an i7-9700T, Quick Sync
|
||||
moves that work off the CPU — which matters on a three-node cluster where the CPU is also running
|
||||
everything else. Two independent things have to be true before that happens, and **only one of them
|
||||
is in git**:
|
||||
|
||||
| Half | Who does it | State |
|
||||
|---|---|---|
|
||||
| The render device is inside the container at `/dev/dri` | **GitOps.** `hardware.gpu: intel-quicksync` in `catalog/plex.yaml` renders a `dri` hostPath volume (`/dev/dri`, `type: DirectoryOrCreate`) and a read-write `/dev/dri` volumeMount on the app container. Flux applies it. | Done by merging that change. |
|
||||
| Plex is *told* to use it | **The operator, by hand, in the Plex web UI.** | **NOT done.** `HardwareAcceleratedCodecs` was **absent** from `Preferences.xml` when this page was written — the toggle has never been on. |
|
||||
|
||||
**There is no GitOps path to the second half, and that is verified rather than assumed.** A bounded
|
||||
grep across the running image's entire init surface — `/etc/cont-init.d`, `/etc/services.d`,
|
||||
`/etc/fix-attrs.d`, `/plex-common.sh`, `/usr/local/bin` — found **zero** `PLEX_PREFERENCE*` support.
|
||||
Every `setPref` the image performs is one of six hardcoded keys (`MachineIdentifier`,
|
||||
`ProcessedMachineIdentifier`, `PlexOnlineToken`, `customConnections`, `allowedNetworks`,
|
||||
`TranscoderTempDirectory`), none of them the transcode toggle. No environment variable, no ConfigMap
|
||||
and no `postBuild` substitution can set it. So do not go looking for the clever way to automate this;
|
||||
there isn't one, and the search has already been done.
|
||||
|
||||
What the device mount does **not** do, so nobody reads more into a green gate than is there: it does
|
||||
not enable hardware transcoding, it does not prove the device can be *opened* from inside a
|
||||
non-privileged pod (see "Likely causes" #2), and it renders no `nodeSelector` — Plex can land on any
|
||||
of the three nodes and all three carry the device.
|
||||
|
||||
## Running the commands in this runbook
|
||||
|
||||
`kubectl` does not work on the operator workstation; the toolchain lives only in the pinned dev
|
||||
container. Every command below runs as:
|
||||
|
||||
infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get nodes'
|
||||
|
||||
Always pass `--context k3s-prod` explicitly. `dev.sh` merges both clusters into one kubeconfig and
|
||||
picks a current-context **per invocation**, and that default has already changed once. Do not run two
|
||||
`dev.sh` invocations at once.
|
||||
|
||||
**Never `cat` `Preferences.xml`.** It contains `PlexOnlineToken`, a plex.tv account credential. Every
|
||||
read of that file below greps a single named key — do not widen one.
|
||||
|
||||
## First commands
|
||||
|
||||
**1. Is the device mounted into the pod at all?** (Proves Flux applied the change, not that it works.)
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \
|
||||
-o jsonpath="{.items[0].spec.containers[0].volumeMounts[*].mountPath}"; echo'
|
||||
```
|
||||
|
||||
`/dev/dri` in that list ⇒ the GitOps half landed. Absent ⇒ the running pod predates the merge; compare
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n flux-system get kustomization apps-wave-0 \
|
||||
-o jsonpath="{.status.lastAppliedRevision}"; echo'
|
||||
```
|
||||
|
||||
against the merge commit before blaming anything else.
|
||||
|
||||
**2. Is the device actually visible inside the container?**
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex exec deploy/plex -- ls -l /dev/dri'
|
||||
```
|
||||
|
||||
Expect `renderD128` (usually `card0` too), mode `crw-rw----`, group `render`. An **empty directory**
|
||||
here is the `DirectoryOrCreate` degradation: the kubelet created an empty `/dev/dri` because the node
|
||||
had none. That is the designed-safe outcome — the app tier still reconciles — but it means software
|
||||
transcoding, silently. Find which node the pod is on and ask why that node lost its iGPU:
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex -o wide'
|
||||
```
|
||||
|
||||
**3. Can Plex *open* it?** This is the question the manifest cannot answer — see "Likely causes" #2.
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex exec deploy/plex -- \
|
||||
bash -c "exec 3<>/dev/dri/renderD128 && echo RW-OPEN-OK || echo RW-OPEN-DENIED"'
|
||||
```
|
||||
|
||||
`RW-OPEN-DENIED` (or `Operation not permitted` / `Permission denied`) with the device plainly visible
|
||||
in step 2 is the **device-cgroup** failure, not a file-permission failure. The two look identical from
|
||||
the outside and are not the same problem.
|
||||
|
||||
**4. Is the toggle on?** Named key only — never the whole file:
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex exec deploy/plex -- \
|
||||
grep -o "HardwareAcceleratedCodecs=\"[01]\"" \
|
||||
"/config/Library/Application Support/Plex Media Server/Preferences.xml" || echo ABSENT'
|
||||
```
|
||||
|
||||
`ABSENT` ⇒ never enabled (the state this page was written in). `="0"` ⇒ explicitly off. `="1"` ⇒ on,
|
||||
and the remaining question is whether it *works* — go to "How to verify".
|
||||
|
||||
## The one manual step
|
||||
|
||||
Plex web UI → **Settings → Transcoder → "Use hardware acceleration when available"** → tick → Save.
|
||||
(It sits under *Show Advanced*.) A transcode already in flight is unaffected; the next one picks it up.
|
||||
|
||||
The underlying preference key is **`HardwareAcceleratedCodecs`**. That name was confirmed by
|
||||
extracting strings from the 23 MB `Plex Media Server` binary in the running container — **not** from
|
||||
recall and not from a forum post. Real neighbouring keys from the same binary, so you can recognise
|
||||
them and not invent them:
|
||||
|
||||
| Key | What it is |
|
||||
|---|---|
|
||||
| `HardwareAcceleratedCodecs` | the toggle this page is about |
|
||||
| `HardwareAcceleratedEncoders` | hardware **encode**, as distinct from decode |
|
||||
| `HardwareDevicePath` | how Plex is pointed at a *specific* render node — relevant only if a host ever exposes more than one |
|
||||
| `TranscoderToneMapping`, `TranscoderToneMappingAgorithm` | HDR→SDR tone mapping. **That misspelling of "Algorithm" is upstream's, in the binary. It is not a typo in this page — do not "fix" it** here or anywhere else; the corrected spelling is not a key Plex reads. |
|
||||
| `TranscoderHEVCEncoding` | HEVC output |
|
||||
|
||||
**`HardwareAcceleratedCodecsEnableHEVC` does not exist.** It is widely cited and it is not in the
|
||||
binary. Recorded explicitly so nobody spends an evening hunting for it or, worse, writes it into
|
||||
`Preferences.xml` and concludes hardware transcoding is broken.
|
||||
|
||||
Editing `Preferences.xml` by hand is **not** the procedure. It is app state on the replicated Longhorn
|
||||
tier, it is in no git commit, and the UI writes the key correctly. Use the UI.
|
||||
|
||||
## How to verify — and why there is no alert
|
||||
|
||||
Start a playback that **forces a transcode** (a client that cannot direct-play the source, or pick a
|
||||
lower quality explicitly), then read Plex's own **Dashboard → Activity**:
|
||||
|
||||
- **`Transcode (hw)`** on the session ⇒ hardware transcoding. **This is the proof.**
|
||||
- plain **`Transcode`** ⇒ software. The feature is not working, whatever the manifest says.
|
||||
|
||||
**That dashboard is the *only* proof available, and nothing else will ever tell you.**
|
||||
`catalog/plex.yaml` declares no `metricsEndpoint`, so Plex renders **no ServiceMonitor at all** —
|
||||
there is no scrape, no transcode-mode metric, and therefore no alert that could fire. Do not open work
|
||||
to "alert on software-transcode fallback": with no metric behind it such a rule would be permanently
|
||||
inert, and this repo has already been burned at scale by rules that evaluated against nothing while
|
||||
looking like coverage (`runbooks/README.md`, "Alerts you should not trust the silence of"). If that
|
||||
gap is ever worth closing, the honest order is a real exporter first and a rule second.
|
||||
|
||||
Corroborating, not a substitute — the server log on the config PVC:
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex exec deploy/plex -- \
|
||||
sh -c "grep -i -m20 -e hardware -e vaapi -e hwdevice \
|
||||
\"/config/Library/Application Support/Plex Media Server/Logs/Plex Media Server.log\""'
|
||||
```
|
||||
|
||||
Treat those strings as advisory: they are Plex's, not this repo's, and they move between versions.
|
||||
Absence of a match is **not** evidence of software transcoding; a VA-API device open is good evidence
|
||||
of hardware. The dashboard remains the verdict.
|
||||
|
||||
## Likely causes, most likely first
|
||||
|
||||
When the dashboard still says plain `Transcode`:
|
||||
|
||||
1. **The toggle is off.** Overwhelmingly the most likely answer, and the reason this page exists.
|
||||
First-command #4 returns `ABSENT` or `="0"`. Remedy: the manual step above. Check this **before**
|
||||
investigating anything technical.
|
||||
|
||||
2. **containerd's device cgroup is refusing the `open()` — UNVERIFIED, and the largest remaining
|
||||
technical risk in this feature.** Whether a non-privileged pod may `open()` a bind-mounted
|
||||
character device on this k3s/containerd build **has not been established**; it can only be settled
|
||||
by deploying and trying. The container inits as uid 0, which settles *file* permissions, but
|
||||
**uid 0 does not bypass the cgroup device gate.** Symptom: step 2 shows `renderD128` present and
|
||||
correctly owned, step 3 returns `RW-OPEN-DENIED`.
|
||||
**The documented fallback is the Intel Device Plugin for Kubernetes** (`gpu.intel.com/i915` as a
|
||||
resource request, no hostPath). Because the ServiceDefinition field expresses *intent* rather than
|
||||
a device path, that migration is a **`catalog/render.py` change plus a re-render — with no
|
||||
ServiceDefinition edit anywhere**; `catalog/plex.yaml` keeps saying exactly what it says today.
|
||||
**`privileged: true` is not an available escape hatch:** Kyverno `restrict-host-namespaces` is in
|
||||
**Enforce** mode and rejects it in app namespaces. Do not reach for it.
|
||||
|
||||
3. **The pod landed on a node without the device.** Step 2 shows an empty `/dev/dri`. All three prod
|
||||
nodes carried `renderD128` with `i915` bound when this was measured (2026-09-30), so this means a
|
||||
node changed — a rebuild, a BIOS reset, or a driver regression. `DirectoryOrCreate` is why the app
|
||||
tier still came up; the cost is that the loss is silent. **Fix the node, do not tighten the mount:**
|
||||
`Directory` would stall `apps-wave-0` (`wait: true`, with `apps-wave-1` depending on it) for a
|
||||
feature that is not load-bearing.
|
||||
|
||||
4. **The image changed under you.** See "The floating-image caveat". A `:latest` pull can remove the
|
||||
mechanism this feature leans on with no commit in this repo.
|
||||
|
||||
5. **The session was never transcoding.** Direct Play / Direct Stream never touches the transcoder, so
|
||||
`Transcode (hw)` cannot appear. Confirm the session actually says `Transcode` before treating it as
|
||||
a hardware failure.
|
||||
|
||||
6. **The source needs a codec Quick Sync on this generation cannot handle** — e.g. AV1 decode, which
|
||||
Coffee Lake's UHD 630 has no engine for. Plex falls back to software *per stream*. Symptom: some
|
||||
sessions show `(hw)` and others do not. Not a fault.
|
||||
|
||||
## The floating-image caveat
|
||||
|
||||
Everything this page states about the image's behaviour — the `cont-init.d` hook that adds the app
|
||||
user to the device's group, the absence of `PLEX_PREFERENCE*`, the six `setPref` keys, the
|
||||
`HardwareAcceleratedCodecs` string — was read from digest **`sha256:e0ab27395614…`**
|
||||
(`plexmediaserver 1.43.4.10903`).
|
||||
|
||||
The deployment runs **`plexinc/pms-docker:latest` with `imagePullPolicy: Always`**. A silent pull can
|
||||
therefore change the behaviour this feature depends on **with no commit in this repo**, while the
|
||||
manifest still looks identical. Symptom to watch for: hardware transcoding that worked and quietly
|
||||
stopped, with no git change to point at.
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \
|
||||
-o jsonpath="{.items[0].status.containerStatuses[0].imageID}"; echo'
|
||||
```
|
||||
|
||||
If that digest is not the one above, **re-verify the hook before trusting this page**:
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'kubectl --context k3s-prod -n plex exec deploy/plex -- ls /etc/cont-init.d/'
|
||||
```
|
||||
|
||||
`45-plex-hw-transcode-and-connected-tuner` still present ⇒ the group mechanism is still the one
|
||||
described here. Gone or renamed ⇒ stop and re-measure. In particular the "no `supplementalGroups`
|
||||
needed" conclusion in `catalog/plex.yaml` and `schemas/renderer-contract.md` is only true while that
|
||||
hook exists.
|
||||
|
||||
## When to escalate or stop
|
||||
|
||||
- **Do not add `supplementalGroups` or a `podSecurityContext` to `catalog/plex.yaml`.** It would be
|
||||
**inert**, and it would mislead the next person. Measured in-container 2026-09-30: the image's
|
||||
`cont-init.d` hook stats each `/dev/dri` char device and `usermod -a -G`s the `plex` user into its
|
||||
group *inside* the container, and the service then execs via `s6-setuidgid plex`, which **replaces**
|
||||
the process group list from `/etc/group` rather than appending — `Plex Media Server` runs with
|
||||
`Groups: 100 1000`, root's gid `0` (which PID 1 does hold) gone. Anything Kubernetes supplies is
|
||||
discarded before Plex starts. `render.py --selftest` asserts that absence deliberately.
|
||||
- **Do not reach for `privileged: true`.** Kyverno `restrict-host-namespaces` is Enforce and will
|
||||
reject it. If the cgroup gate is the blocker, the answer is the Intel device plugin (cause #2).
|
||||
- **Do not `kubectl edit`/`patch` the Deployment to add or change the device mount.** Flux reconciles
|
||||
`main`; a hand-applied change is drift that the next reconcile reverts or, worse, adopts. Mutating
|
||||
`kubectl` and `flux reconcile` are **operator** actions, and the durable change is a git change
|
||||
through a branch, a PR and a green gate.
|
||||
- **Do not edit `Preferences.xml` on the PVC**, never `cat` it, and never `sops -d` anything into a
|
||||
terminal. It holds `PlexOnlineToken`.
|
||||
- **Do not delete the `plex-config` PVC to "start clean".** It is the library database, metadata and
|
||||
watch state (`policy: daily-offcluster`). The transcode toggle is one preference, not a reason to
|
||||
lose the DB.
|
||||
- **Stop and get the operator** if the device cannot be opened from the pod (cause #2 confirmed). The
|
||||
next step is a platform change — the Intel device plugin — with its own PR, not more poking at this
|
||||
workload.
|
||||
|
||||
## Known gaps, deliberately not fixed here
|
||||
|
||||
Recorded so they are findable, and so nobody assumes they were overlooked:
|
||||
|
||||
- **Transcode scratch is on the wrong medium, and it is invisible.** The image sets
|
||||
`TranscoderTempDirectory=/transcode`, which is **not a volume** — it is the container's writable
|
||||
overlay, i.e. the node's `/data`. That is NVMe on prod-2 but **SATA on prod-1 and prod-3**, so
|
||||
transcode scratch performance changes silently whenever the pod is rescheduled. Both candidate fixes
|
||||
need a **schema** change and are therefore out of scope here: pin Plex to prod-2 (needs a
|
||||
`nodeSelector` field the schema does not have), or give `/transcode` a `medium: Memory` emptyDir and
|
||||
transcode in RAM (~31 GiB per node; needs an emptyDir field the schema does not have). The RAM option
|
||||
is the better one, because it removes the node asymmetry outright instead of working around it.
|
||||
- **GuC/HuC firmware is not loaded** (`i915.enable_guc=0`). The cost is low-power H.264 **encode**
|
||||
quality only — decode, and the feature on this page, are unaffected. Changing it needs a kernel
|
||||
parameter and a **serialized rolling reboot**, which is an operator action (CLAUDE.md §5), and this
|
||||
cluster has a documented history of that going badly (a prod node hung on a warm reboot on
|
||||
2026-09-03). Not worth coupling to this change.
|
||||
|
||||
---
|
||||
Source of truth: `runbooks/plex-quicksync.md`. Index: `/runbooks/`
|
||||
The node measurements (all three prod nodes) and the in-container image measurements cited here were
|
||||
taken **2026-09-30** against `plexinc/pms-docker` digest `sha256:e0ab27395614…`
|
||||
(`plexmediaserver 1.43.4.10903`). Hardware transcoding had **not** been observed working at the time
|
||||
of writing, and the Plex-side toggle was still unset.
|
||||
+4
-1
@@ -15,7 +15,7 @@ docs/13 exactly. Do not diverge from it here — escalate spec problems, don't f
|
||||
| `servicedefinition.schema.json` | Formal JSON Schema (draft 2020-12) for a ServiceDefinition. Enforces enums, required fields, patterns, and cross-field invariants. |
|
||||
| `renderer-contract.md` | Authoritative field → rendered-resource mapping, with requirement IDs. The contract other workstreams consume. |
|
||||
|
||||
**Schema version:** 1.0.0 (in the schema `description`). Treat any change as an interface change:
|
||||
**Schema version:** 1.8.0 (in the schema `description`, which carries the full per-version changelog — this line had said `1.0.0` since 1.1.0 and is now synced). Treat any change as an interface change:
|
||||
propose on the task board, update dependent renderers/tests in the same change, never silently break a
|
||||
consumer (coordination-protocol §2).
|
||||
|
||||
@@ -35,6 +35,9 @@ consumer (coordination-protocol §2).
|
||||
- `ingressRoutes[].host` must match `<label>.home.keithjackson.dev`.
|
||||
- `storage[].class ∈ {longhorn, bulk}`; `exposure ∈ {internal, public}`; `egress ∈ {internet, vpn}`;
|
||||
`auth.mode ∈ {oidc, forward-auth, native, none}`.
|
||||
- `hardware.gpu ∈ {intel-quicksync}` — a host **capability class**, never a device path. The enum
|
||||
is the contract: the mechanism that satisfies it lives only in `catalog/render.py` (see
|
||||
`renderer-contract.md` §`hardware`).
|
||||
- `additionalProperties: false` everywhere — unknown keys are rejected (catches typos in review/CI).
|
||||
|
||||
## Validate examples against the schema — L1, no cluster (R-TEST-02)
|
||||
|
||||
@@ -23,6 +23,7 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
| `dependencies` | `[]` → reaches **only DNS** (default-deny) | R-NET-01 |
|
||||
| `inClusterClients` | `[]` → no direct east-west client; only the Gateway may reach an exposed port | R-NET-01/02 |
|
||||
| `auth.mode` | `forward-auth` for any exposed UI | R-IDP-02 |
|
||||
| `auth.apiRoutes` | `[]` → the sidecar renders **no** `--api-route`; every unauthenticated request is 302'd to Keycloak | R-IDP-02 |
|
||||
| `update.channel` | `stable` | R-UPD-01 |
|
||||
| `update.autoRollbackOnFailedHealth` | `true` | R-UPD-03 |
|
||||
| `observability.metrics` | `true` | R-OBS-01 |
|
||||
@@ -160,6 +161,64 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
- **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication),
|
||||
**R-STATE-04** (bulk tier). (docs/04)
|
||||
|
||||
### `hardware` (`gpu`) — host device capability
|
||||
- **The field is INTENT, never MECHANISM.** `hardware.gpu` is an **enum of capability classes**
|
||||
(today: `intel-quicksync`), not a device path, not a hostPath, not a resource request. The
|
||||
ServiceDefinition says *what class of host device this workload needs*; how the platform hands it
|
||||
over is `catalog/render.py`'s business and lives nowhere else. **Do not "simplify" this into a raw
|
||||
`/dev/...` path.** The entire payoff is that migrating from the hostPath mechanism below to the
|
||||
Intel GPU device plugin (`resources.limits: gpu.intel.com/i915`) is a renderer change with **zero**
|
||||
catalog edits and zero re-review of per-service device wiring. Extend the enum for a genuinely
|
||||
different capability class (e.g. `nvidia-nvenc`); never to name a second route to the same one.
|
||||
- `gpu: intel-quicksync` → on the **app container only** (never a sidecar), exactly two things:
|
||||
1. a **hostPath volume** `dri` → `/dev/dri`, **`type: DirectoryOrCreate`**;
|
||||
2. a **read-write volumeMount** of `dri` at `/dev/dri` on the app container (VA-API opens the
|
||||
render node read-write, so it is deliberately not `readOnly`).
|
||||
|
||||
**Mounting the device is sufficient. That is the whole mapping.**
|
||||
- **`podSecurityContext.supplementalGroups` is deliberately NOT rendered, and adding it would be a
|
||||
regression rather than a hardening.** `/dev/dri/renderD128` is `root:render` `0660`, so the obvious
|
||||
move is to push the host `render` GID in — and it does not work. **Measured in the running
|
||||
container on prod-2, 2026-09-30:** the image's own
|
||||
`/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` hook `find`s every char device under
|
||||
`/dev/dri`, `stat -c '%g'` each one's GID, `groupadd`s a matching group *inside the container* when
|
||||
none exists, and `usermod -a -G` the app user into it — every start, nothing hardcoded.
|
||||
`cont-init.d` completes before `services.d`, so the account database is already correct when the
|
||||
service execs. The service then starts via `exec s6-setuidgid plex …`, and `s6-setuidgid`
|
||||
**replaces** the process group list from `/etc/group` rather than appending to it:
|
||||
`Plex Media Server` runs as `Uid: 1000 Gid: 1000 Groups: 100 1000`, with root's gid `0` — which
|
||||
PID 1 (`s6-svscan`) does hold — **gone**. A Kubernetes `supplementalGroups` therefore reaches
|
||||
`s6-svscan` and is discarded before the app ever starts. It would be inert *and* it would mislead
|
||||
whoever debugs this next. The renderer asserts its own absence in `render.py --selftest`.
|
||||
- A corollary worth having: because the hook re-`stat`s the device on every start, the host GID
|
||||
**drifting** (a node rebuild renumbering the *systemd dynamic* `render` group) is self-healing.
|
||||
No GID is pinned anywhere in this repo.
|
||||
- **Scope of that measurement, stated so nobody over-reads it.** "Mounting the device is
|
||||
sufficient" was established for an image that resolves the device group itself. A FUTURE
|
||||
consumer that runs non-root and has no equivalent init hook would still need the group solved
|
||||
somehow — and the answer for that case is a **renderer** change (per-image handling, or the
|
||||
device plugin), not a new ServiceDefinition field. The enum is what keeps that option open. The
|
||||
first such consumer must re-measure rather than assume this bullet covers it.
|
||||
- **`DirectoryOrCreate` is load-bearing — do not tighten it to `Directory`.** A node that comes back
|
||||
without the render device (rebuild, BIOS change, driver regression, hardware swap) must degrade the
|
||||
workload to **software transcoding**, not fail the mount. `apps-wave-0` is `wait: true` and
|
||||
`apps-wave-1` `dependsOn` it, so a failed hostPath mount stalls the **whole app tier** for the sake
|
||||
of a feature that is not load-bearing. `/dev` is tmpfs, so the empty directory the kubelet creates
|
||||
on such a node leaves nothing durable behind.
|
||||
- **The cost of that choice, stated plainly:** a missing device is **silent**. The workload
|
||||
software-transcodes and nothing reports it. There is no metric to alert on — media apps here
|
||||
render no `metricsEndpoint`, so no ServiceMonitor exists — and an alert that cannot fire is worse
|
||||
than none (see §`observability`'s 2026-09-13 correction). The only proof of hardware transcode is
|
||||
an operator observing it in the application's own dashboard.
|
||||
- **Deliberately NOT rendered** from this field: no `privileged: true` (Kyverno
|
||||
`restrict-host-namespaces` is Enforce on app namespaces and would reject it, and the render node
|
||||
does not need it); no `nodeSelector` (placement is a separate concern with its own justification —
|
||||
this field is about the device, not about where the pod lands); no resource requests/limits (there
|
||||
is no app-container resource render path, and inventing one here would be scope creep).
|
||||
- Absent ⇒ **nothing at all** is rendered; every service that does not opt in stays byte-identical.
|
||||
Ignored for `kind: vm` — a VM receives devices through KubeVirt's own `domain.devices`.
|
||||
- **Requirements:** **R-PLAT-02** (the PaaS provides the host capability; the app declares need).
|
||||
|
||||
### `ingressRoutes[]` (`host` / `port`) — HTTPRoute + Certificate + DNS
|
||||
- Each entry renders an **HTTPRoute** attaching `host` → the service `port` on the shared Gateway.
|
||||
- TLS: the Gateway terminates with the cert-manager **wildcard `*.home.keithjackson.dev`** certificate
|
||||
@@ -202,7 +261,7 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
undeclared), **R-GITOPS-02** (the allowance is authored in the one reviewable object rather than
|
||||
surviving as a live-applied cluster object). Auditable via Hubble (R-SEC-03, R-OBS-04).
|
||||
|
||||
### `auth` (`mode` / `groups[]`) — identity integration (Keycloak / oauth2-proxy)
|
||||
### `auth` (`mode` / `groups[]` / `apiRoutes[]`) — identity integration (Keycloak / oauth2-proxy)
|
||||
- `oidc` → register the app as a **Keycloak OIDC client** (declarative realm/client config, GitOps;
|
||||
client secret SOPS-encrypted). App does its own login redirect.
|
||||
- `forward-auth` → protect the service's `ingressRoutes` at the **Gateway** with **oauth2-proxy →
|
||||
@@ -213,6 +272,20 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
- `groups[]` → the Keycloak groups permitted access, enforced by the OIDC client mappers /
|
||||
oauth2-proxy `allowed_groups`. **Schema requires ≥1 group for `oidc`/`forward-auth`** — access is
|
||||
"in an authorized group," not merely "logged in."
|
||||
- `apiRoutes[]` (**`forward-auth` only**) → one repeatable **`--api-route=<regex>`** per entry on
|
||||
the per-app oauth2-proxy sidecar, making matching paths answer **401** instead of a **302** to
|
||||
Keycloak. Default `[]` renders **no flag at all**, which is the common case. It exists because
|
||||
oauth2-proxy's built-in AJAX guard fires only on an exact `Accept: application/json`, which
|
||||
these apps' JavaScript does not consistently send: the XHR is 302'd, the browser follows it
|
||||
invisibly, the live SSO session satisfies it silently, and the call resolves 200 carrying the
|
||||
app's **HTML** where the JavaScript expected JSON — and a POST following a 302 is re-issued as
|
||||
a GET with its **body dropped**. It is **per-service and must never become a renderer
|
||||
constant**: the path shape belongs to the app (radarr `/signalr/`, jackett `/api/`, pihole
|
||||
none), and a wrong constant does not merely fail to help — a path a **browser navigates to**,
|
||||
answered 401, shows a logged-out operator a bare error page and never offers a sign-in. Each
|
||||
entry must be **anchored at `^/`** and continue with literal path characters; `^/`, `^/.*`,
|
||||
`.*`, an unanchored `/api/` and any alternation are **rejected** by the schema and by the
|
||||
renderer (alternation is refused rather than supported because the flag is repeatable).
|
||||
- Keycloak, oauth2-proxy, and app→Keycloak / proxy→Keycloak / Keycloak→DB flows are ordinary
|
||||
workloads under default-deny with explicit allows (ties back to `dependencies`).
|
||||
- **Requirements:** **R-IDP-02** (protect OIDC-capable and bare UIs), **R-IDP-06** (group-based
|
||||
@@ -271,6 +344,12 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
## Cross-field invariants the renderer / L0-L1 tests rely on
|
||||
1. **kind ↔ source:** container⇒`image`, vm⇒`vmTemplate` (schema-enforced).
|
||||
2. **auth ↔ groups:** `oidc`/`forward-auth` ⇒ non-empty `groups` (schema-enforced; R-IDP-06).
|
||||
2a. **auth ↔ apiRoutes:** a non-empty `auth.apiRoutes` ⇒ `auth.mode: forward-auth`
|
||||
(schema-enforced). It renders onto the per-app oauth2-proxy sidecar, and no sidecar exists for
|
||||
`oidc`/`native`/`none`, so declaring it elsewhere would be a **silent no-op** — the operator
|
||||
believes the app's XHRs now answer 401 and nothing changed. Every entry must be anchored at
|
||||
`^/` (schema-enforced **and** enforced in `catalog/render.py`, so a breadth mistake is caught by
|
||||
`render-golden` even on a run where the schema stage did not execute).
|
||||
3. **default-deny is unconditional:** every namespace gets the deny baseline regardless of
|
||||
`dependencies`; `dependencies` only *adds* allows (R-NET-01/02). The same holds for
|
||||
`inClusterClients`: it adds one namespace+port ingress allow and never relaxes the baseline.
|
||||
@@ -286,6 +365,11 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
`external-dns.alpha.kubernetes.io/controller: none`, and no `public` service's route carries it.
|
||||
Derived from `exposure` alone; enforced on every gate run by
|
||||
`conformance/tests/l0-static/internal-services-publish-no-public-dns` (R-DNS-03).
|
||||
8. **`hardware` is inert until opted into, and its mechanism is renderer-local.** An SD with no
|
||||
`hardware` key renders no volume and no volumeMount; an SD *with* it gains no `securityContext`
|
||||
either. Both directions are proven by `render.py --selftest`. The `/dev/dri` hostPath and the
|
||||
`DirectoryOrCreate` type appear in `catalog/render.py` and nowhere else, so the capability's
|
||||
implementation can be replaced without touching a single ServiceDefinition.
|
||||
|
||||
## Requirement traceability (contract-level)
|
||||
Primary **R-GITOPS-02** (single reviewable object wires every requirement). Field-level: R-NET-01/02
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
|
||||
"title": "ServiceDefinition",
|
||||
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
|
||||
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked proLine truncated
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "source"],
|
||||
@@ -139,6 +139,19 @@
|
||||
"supplementalGroups": { "type": "array", "items": { "type": "integer" }, "description": "Additional GIDs added to the first process in each container." }
|
||||
}
|
||||
},
|
||||
"hardware": {
|
||||
"type": "object",
|
||||
"description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume and a matching mount on the app container, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",
|
||||
"additionalProperties": false,
|
||||
"minProperties": 1,
|
||||
"properties": {
|
||||
"gpu": {
|
||||
"type": "string",
|
||||
"description": "GPU capability class this service needs. `intel-quicksync` = the host's Intel Quick Sync Video render device (the iGPU's /dev/dri/renderD* node) for hardware video transcode/decode. Enum, not a free-form device path, so the satisfying mechanism stays swappable (see the parent description). Extend the enum when a genuinely different capability class appears (e.g. nvidia-nvenc), never to name a different path to the same one.",
|
||||
"enum": ["intel-quicksync"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"exposure": {
|
||||
"type": "string",
|
||||
"description": "internal = LAN-only (internal DNS + Gateway on LAN); public = additionally edge port-forwarded 80/443 (docs/09; R-DNS/R-NET).",
|
||||
@@ -311,6 +324,16 @@
|
||||
"items": { "type": "string", "minLength": 1 },
|
||||
"uniqueItems": true,
|
||||
"default": []
|
||||
},
|
||||
"apiRoutes": {
|
||||
"type": "array",
|
||||
"description": "forward-auth ONLY: path regexes this app serves as a MACHINE API, which the oauth2-proxy sidecar must answer with 401 instead of a 302 to Keycloak (one --api-route per entry; the flag is repeatable and singular -- --api-routes does not exist in v7.8.1 and the proxy refuses to start with it). WHY THIS IS PER-APP AND NOT A RENDERER CONSTANT: oauth2-proxy's built-in AJAX guard fires only on an EXACT `Accept: application/json`, which these apps' JavaScript does not consistently send, so an unauthenticated XHR gets a 302 that the browser follows invisibly, the live Keycloak SSO session satisfies silently, and the call resolves 200 carrying the app's HTML where the JavaScript expected JSON -- no prompt, no error, and on a write it is worse, because a browser following a 302 re-issues a POST as a GET and DROPS THE BODY. Measured on jackett 2026-09-30: 33 x 302 against 2 x 401 on /api/ paths, with 4 of 6 POSTs answered by the proxy and never reaching the app (access-log upstream field `-`). Measured on radarr 2026-10-01, 23h after the cookie fix deployed: 232 x 302, ALL of them POST /signalr/messages/negotiate from the operator's own browser -- a left-open tab whose cookie the rename orphaned, polling forever, silently redirected every time and NEVER re-prompting. But the path shape is a property of the APP, not of the pattern: radarr's broken path is /signalr/ and jackett's is /api/, and pihole serves no /api/ at all, so a renderer-wide constant would apply one app's path shape to five. Declare only paths this app actually serves as JSON/RPC and that a browser never NAVIGATES to. BREADTH IS THE DANGER: too broad and a plain page load answers 401 instead of signing the operator in, i.e. the service looks permanently broken to a logged-out user and no sign-in is ever offered. The item pattern therefore REQUIRES each entry to be anchored at `^/` and to continue with literal path characters, which rejects `^/`, `^/.*`, `.*` and any unanchored or alternated regex; alternation is refused rather than supported because the flag is repeatable, so one regex per route costs nothing and keeps each entry readable. N/A for oidc/native/none (no sidecar is rendered).",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^\\^/[A-Za-z0-9._~/-]+(\\.\\*|\\$)?$"
|
||||
},
|
||||
"uniqueItems": true,
|
||||
"default": []
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -458,6 +481,26 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"$comment": "auth.apiRoutes only has a consumer under forward-auth: it renders --api-route onto the per-app oauth2-proxy sidecar, and no sidecar is rendered for oidc/native/none. Declaring it elsewhere would be a SILENT NO-OP -- the operator believes the app's XHRs now answer 401 and nothing changed -- so it is rejected at the schema instead (R-IDP-02).",
|
||||
"if": {
|
||||
"required": ["auth"],
|
||||
"properties": {
|
||||
"auth": {
|
||||
"required": ["apiRoutes"],
|
||||
"properties": { "apiRoutes": { "minItems": 1 } }
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"auth": {
|
||||
"required": ["mode"],
|
||||
"properties": { "mode": { "const": "forward-auth" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"$comment": "public exposure must not be unauthenticated: an explicit auth.mode 'none' is disallowed on public services (docs/13 v1.1, docs/15, R-IDP-02). Absent auth defaults to forward-auth at render time, so it is permitted here.",
|
||||
"if": {
|
||||
|
||||
Reference in new issue
Block a user