Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2279d14ec5 |
No files matched your search
+37
-30
@@ -141,6 +141,15 @@ KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
|
||||
# nowhere else -- no catalog entry moves.
|
||||
DRI_VOLUME_NAME = "dri"
|
||||
DRI_DEVICE_DIR = "/dev/dri"
|
||||
# GID of the host `render` group, which owns /dev/dri/renderD128 (mode 0660, root:render).
|
||||
# MEASURED 2026-09-30 on ALL THREE prod nodes (HP EliteDesk 800 G5 Mini, i7-9700T, UHD
|
||||
# Graphics 630, i915 bound -- not xe): GID 992, group name `render`, identical on each.
|
||||
# This is a *systemd dynamic* allocation (systemd-udev creates `render` out of the dynamic
|
||||
# system range at first boot), NOT a Debian/LSB static GID, so it is NOT guaranteed stable:
|
||||
# a node rebuild, or a reinstall that creates system groups in a different order, can move
|
||||
# it. If hardware transcode stops working after a node rebuild, re-measure
|
||||
# `getent group render` on every node before assuming anything else.
|
||||
INTEL_RENDER_GID = 992
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -506,22 +515,6 @@ def r_deployment(s: dict) -> dict:
|
||||
# directory the kubelet creates on such a node leaves nothing durable behind.
|
||||
#
|
||||
# The mount is deliberately NOT readOnly: VA-API opens the render node read-write.
|
||||
#
|
||||
# `podSecurityContext.supplementalGroups` is DELIBERATELY NOT SET here, and adding it
|
||||
# would be a regression, not a hardening. MEASURED in the running container on prod-2,
|
||||
# 2026-09-30: the image's own cont-init hook
|
||||
# `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` `find`s every char device
|
||||
# under /dev/dri, `stat -c '%g'` each one's GID, `groupadd`s a matching group INSIDE the
|
||||
# container when none exists, and `usermod -a -G` the app user into it -- every start,
|
||||
# nothing hardcoded. cont-init.d completes before services.d, so the account database
|
||||
# s6 reads at exec time already carries the group. The service is then started with
|
||||
# `exec s6-setuidgid plex ...`, and `s6-setuidgid` REPLACES the process group list from
|
||||
# /etc/group rather than appending to it: `Plex Media Server` runs with
|
||||
# `Uid: 1000 Gid: 1000 Groups: 100 1000` -- root's gid 0, which PID 1 (s6-svscan) does
|
||||
# have, is GONE. Anything Kubernetes puts in supplementalGroups reaches s6-svscan and is
|
||||
# then discarded before the app ever starts. MOUNTING THE DEVICE IS SUFFICIENT; the
|
||||
# group handling is the image's job and it does it correctly. A supplementalGroups entry
|
||||
# here would be inert AND would mislead whoever debugs this next.
|
||||
hw_gpu = (s.get("hardware") or {}).get("gpu")
|
||||
if hw_gpu == "intel-quicksync":
|
||||
volume_mounts.append({"name": DRI_VOLUME_NAME, "mountPath": DRI_DEVICE_DIR})
|
||||
@@ -554,8 +547,19 @@ def r_deployment(s: dict) -> dict:
|
||||
# pod spec. Its primary use is `fsGroup`, which makes a mounted Longhorn PVC group-writable by a
|
||||
# non-root image (e.g. semaphore runs as UID 1001 and cannot otherwise write its config PVC).
|
||||
# Absent => the key is never set, so apps that declare none render byte-identical.
|
||||
if s.get("podSecurityContext"):
|
||||
pod_spec["securityContext"] = s["podSecurityContext"]
|
||||
#
|
||||
# `hardware.gpu: intel-quicksync` additionally needs the host `render` GID in
|
||||
# supplementalGroups, because /dev/dri/renderD128 is root:render 0660 and the container
|
||||
# does not run as root. It is MERGED into whatever the SD already declares rather than
|
||||
# replacing it -- an SD that sets its own supplementalGroups keeps every one of them.
|
||||
psc = dict(s.get("podSecurityContext") or {})
|
||||
if hw_gpu == "intel-quicksync":
|
||||
groups = list(psc.get("supplementalGroups") or [])
|
||||
if INTEL_RENDER_GID not in groups:
|
||||
groups.append(INTEL_RENDER_GID)
|
||||
psc["supplementalGroups"] = groups
|
||||
if psc:
|
||||
pod_spec["securityContext"] = psc
|
||||
|
||||
if volumes:
|
||||
pod_spec["volumes"] = volumes
|
||||
@@ -2516,14 +2520,10 @@ def _selftest() -> int:
|
||||
{"name": "dri", "mountPath": "/dev/dri"} in hw_mounts,
|
||||
"hardware.gpu=intel-quicksync must mount /dev/dri READ-WRITE on the app container",
|
||||
)
|
||||
# The device mount is the WHOLE feature: no pod securityContext is invented. The image's
|
||||
# own cont-init hook adds the app user to the device's group inside the container, and
|
||||
# s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway (see the
|
||||
# emission site). Asserting the absence is what stops someone adding an inert one back.
|
||||
check(
|
||||
"securityContext" not in hw_pod,
|
||||
"hardware.gpu must NOT invent a pod securityContext (supplementalGroups is inert "
|
||||
"under s6-setuidgid; the image's cont-init hook handles the device group)",
|
||||
hw_pod.get("securityContext", {}).get("supplementalGroups")
|
||||
== [INTEL_RENDER_GID],
|
||||
"hardware.gpu=intel-quicksync must add the host render GID to supplementalGroups",
|
||||
)
|
||||
# Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce)
|
||||
# rejects `privileged: true` in an app namespace, and placement is a separate concern
|
||||
@@ -2546,8 +2546,8 @@ def _selftest() -> int:
|
||||
and "securityContext" not in nohw_pod,
|
||||
"an SD without `hardware` must gain no dri volume, no mount and no securityContext",
|
||||
)
|
||||
# An SD that declares BOTH keeps its own podSecurityContext exactly as written -- the
|
||||
# hardware field neither adds to it nor rewrites it.
|
||||
# Merge, not clobber: an SD that declares its own supplementalGroups keeps them AND
|
||||
# gains the render GID, and its other podSecurityContext keys survive untouched.
|
||||
hwm_pod = next(
|
||||
d[1]
|
||||
for d in render(
|
||||
@@ -2555,14 +2555,21 @@ def _selftest() -> int:
|
||||
"name": "hwm",
|
||||
"source": {"image": "a:1"},
|
||||
"hardware": {"gpu": "intel-quicksync"},
|
||||
"podSecurityContext": {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
"podSecurityContext": {
|
||||
"fsGroup": 1001,
|
||||
"supplementalGroups": [44],
|
||||
},
|
||||
}
|
||||
)
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
"hardware.gpu must leave the SD's own podSecurityContext untouched",
|
||||
hwm_pod["securityContext"] == {
|
||||
"fsGroup": 1001,
|
||||
"supplementalGroups": [44, INTEL_RENDER_GID],
|
||||
},
|
||||
"hardware.gpu must MERGE the render GID into the SD's own supplementalGroups "
|
||||
"(both GIDs, fsGroup preserved), never clobber them",
|
||||
)
|
||||
|
||||
# 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
date: 2026-09-30
|
||||
branch: feat/sd-hardware-capability
|
||||
title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` mounts /dev/dri and nothing else; the enum is what keeps the mechanism swappable
|
||||
title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` renders the /dev/dri wiring, and the enum is what keeps the mechanism swappable
|
||||
---
|
||||
|
||||
## What changed
|
||||
@@ -13,8 +13,8 @@ This PR delivers the capability; consuming it (Plex) is separate work on top.
|
||||
- `schemas/servicedefinition.schema.json` — adds `hardware` (object, `additionalProperties: false`,
|
||||
`minProperties: 1`) with `gpu` (enum `["intel-quicksync"]`). Schema version bumped **1.7.0 →
|
||||
1.8.0**, changelog entry prepended to the schema `description` per that file's own convention.
|
||||
- `catalog/render.py` — two module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`) and the
|
||||
emission in `r_deployment`.
|
||||
- `catalog/render.py` — three module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`,
|
||||
`INTEL_RENDER_GID = 992`) and the emission in `r_deployment`.
|
||||
- `schemas/renderer-contract.md` — new `### hardware` field-mapping section, plus cross-field
|
||||
invariant **8** in that file's numbered list.
|
||||
- `schemas/README.md` — its `**Schema version:**` line said `1.0.0`; it had been stale since 1.1.0.
|
||||
@@ -22,45 +22,22 @@ This PR delivers the capability; consuming it (Plex) is separate work on top.
|
||||
bullet to "Cross-field rules" for the new enum. That was the whole of the staleness; nothing
|
||||
ballooned.
|
||||
|
||||
**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container only (never a
|
||||
sidecar), and it is exactly two things:
|
||||
**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container / pod spec
|
||||
only (never a sidecar):
|
||||
|
||||
1. a hostPath volume `dri` → `/dev/dri`, `type: DirectoryOrCreate`;
|
||||
2. a read-write `volumeMount` of `dri` at `/dev/dri` on the app container (deliberately not
|
||||
`readOnly` — VA-API opens the render node read-write).
|
||||
`readOnly` — VA-API opens the render node read-write);
|
||||
3. `INTEL_RENDER_GID` **merged into** `podSecurityContext.supplementalGroups`, never clobbering
|
||||
what the SD already declares.
|
||||
|
||||
**Mounting the device is sufficient.** It emits **no** `privileged: true` (Kyverno
|
||||
`restrict-host-namespaces` is Enforce on app namespaces and it is not needed), **no** `nodeSelector`
|
||||
(placement is a separate concern with its own justification), **no** resource requests (there is no
|
||||
app-container resource render path), and — the one that is worth spelling out — **no
|
||||
`podSecurityContext.supplementalGroups`**.
|
||||
|
||||
**Why no `supplementalGroups`, when `/dev/dri/renderD128` is `root:render 0660`.** An earlier
|
||||
revision of this branch merged the host `render` GID into `supplementalGroups`. **It was measured to
|
||||
be a no-op and has been removed.** In the running container on prod-2, 2026-09-30:
|
||||
|
||||
PID 1 s6-svscan Uid: 0 Gid: 0 Groups: 0
|
||||
pid 246 Plex Media Server Uid: 1000 Gid: 1000 Groups: 100 1000
|
||||
|
||||
`/etc/services.d/plex/run` ends in `exec s6-setuidgid plex …`, and the child's group list is exactly
|
||||
the `/etc/group` account-database list — root's gid `0`, which PID 1 holds, has **vanished**. That is
|
||||
a `setgroups()` that *replaces* rather than appends, so anything Kubernetes puts in
|
||||
`supplementalGroups` reaches `s6-svscan` and is discarded before the app ever starts. What actually
|
||||
works is the image's own `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner`, which `find`s
|
||||
every char device under `/dev/dri`, `stat -c '%g'`s its GID, `groupadd`s a matching group inside the
|
||||
container when none exists, and `usermod -a -G`s the app user into it — every start, nothing
|
||||
hardcoded; `cont-init.d` completes before `services.d`, so the database is correct at exec time.
|
||||
|
||||
Shipping an inert `supplementalGroups` would have been worse than shipping nothing: it would read as
|
||||
the mechanism to whoever debugs this next. The reasoning lives in a comment **at the emission site**
|
||||
and in `renderer-contract.md`, and `render.py --selftest` now asserts the *absence* of a
|
||||
`securityContext`, so it cannot be "helpfully" added back without a test failing. A pleasant
|
||||
corollary: because the hook re-`stat`s the device every start, host GID drift is **self-healing**,
|
||||
and no GID is pinned anywhere in this repo.
|
||||
It emits **no** `privileged: true` (Kyverno `restrict-host-namespaces` is Enforce on app namespaces
|
||||
and it is not needed), **no** `nodeSelector` (placement is a separate concern with its own
|
||||
justification) and **no** resource requests (there is no app-container resource render path).
|
||||
|
||||
**The design decision worth defending: the field is an enum of intent, not a device path.** The SD
|
||||
says *what class of host device the workload needs*; the hostPath and the mount exist only in
|
||||
`catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource
|
||||
says *what class of host device the workload needs*; the hostPath, the mount and the GID exist only
|
||||
in `catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource
|
||||
requests, no hostPath at all) is then a renderer change with zero catalog edits. Both the schema
|
||||
`description` and the contract section say this explicitly, because the obvious "simplification" —
|
||||
letting an SD name `/dev/dri` — would push a platform implementation detail into the operator-facing
|
||||
@@ -77,15 +54,15 @@ behind. Do not "tighten" it to `Directory`. The reasoning is recorded at the emi
|
||||
|
||||
Measured, in this worktree, on this branch:
|
||||
|
||||
- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** New cases in the
|
||||
existing `check(...)` style: the positive (volume + mount, and *no* `securityContext`, *no*
|
||||
privileged, *no* nodeSelector), the negative that matters (an SD without `hardware` gains no
|
||||
volume, no volumeMount and no `securityContext` — mirroring the existing `podSecurityContext`
|
||||
negative), and a pass-through case (an SD declaring both `hardware` and its own
|
||||
`podSecurityContext: {fsGroup: 1001, supplementalGroups: [44]}` keeps it **exactly** as written).
|
||||
- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** Three new cases
|
||||
were added in the existing `check(...)` style: the positive (volume + mount + GID, and *no*
|
||||
privileged/nodeSelector), the negative (an SD without `hardware` gains no volume, no volumeMount
|
||||
and no `securityContext`), and the merge case (`supplementalGroups: [44]` + the field →
|
||||
`[44, 992]`, with `fsGroup` preserved).
|
||||
- **The self-tests actually bite.** I mutated the guard (`if hw_gpu == "intel-quicksync":` → `if
|
||||
True:`) in a scratch copy and re-ran: **rc 1**, failing on "an SD without `hardware` must gain no
|
||||
dri volume, no mount and no securityContext". A test that cannot fail is not a test; this one can.
|
||||
True:`) in a scratch copy and re-ran: rc 1 with three named failures, including "an SD without
|
||||
`hardware` must gain no dri volume, no mount and no securityContext". A test that cannot fail is
|
||||
not a test; this one can.
|
||||
- **Re-render is a zero diff.**
|
||||
`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` then
|
||||
`git diff --stat apps/` → **empty**, and `git status --porcelain apps/` → **empty** (so the two
|
||||
@@ -97,22 +74,21 @@ Measured, in this worktree, on this branch:
|
||||
(`additionalProperties: false`); `hardware: {}` rejected (`minProperties: 1`).
|
||||
- **Hand-checked the emitted YAML** by rendering a throwaway SD carrying `hardware`, a Longhorn
|
||||
volume and its own `podSecurityContext: {runAsUser: 1000, supplementalGroups: [44]}`. Result:
|
||||
`volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `volumes` = the PVC then
|
||||
`{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`; `securityContext` =
|
||||
`{runAsUser: 1000, supplementalGroups: [44]}` — i.e. **unchanged**, nothing appended. That toy SD
|
||||
was never written into `catalog/`.
|
||||
`volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `securityContext` =
|
||||
`{runAsUser: 1000, supplementalGroups: [44, 992]}`; `volumes` = the PVC then
|
||||
`{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`. That toy SD was never written
|
||||
into `catalog/`.
|
||||
- **GATE: `rc 3` — UNDETERMINED, NOT green.** Run in the pinned toolbox from this worktree with
|
||||
both CLAUDE.md §3 workarounds (`DEV_KUBECONFIG=/dev/null`, and an `ENGINE` shim bind-mounting the
|
||||
main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f`; the run's
|
||||
`HEAD` was `d7df88c9c264`, which differs from the final commit only by this table being written
|
||||
into this file. Touched requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Per stage:
|
||||
main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f` … `HEAD @
|
||||
9f0db1495cfd`; touched requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Per stage:
|
||||
|
||||
| stage | result |
|
||||
|---|---|
|
||||
| spec-immutability | **SKIPPED** — sibling spec repo not at `/work/../home-network-k3s`. Expected from a worktree; tolerated by §3. |
|
||||
| decisions-log | pass |
|
||||
| lint | pass |
|
||||
| l0 | pass — 19 offline cases, `0 failure(s), 0 skip(s), 0 unwired` |
|
||||
| l0 | pass |
|
||||
| render-golden | pass — *"all 10 onboarded service golden(s) reproduce from the canonical render"* |
|
||||
| l1 | pass — 3 offline checks, `0 failure(s), 0 skip(s), 0 unwired`; coverage 51/51 requirements scaffolded |
|
||||
| approval-policy | pass (`event='none'`, not a pull_request event — N/A locally; it will evaluate for real on the PR) |
|
||||
@@ -132,17 +108,14 @@ Measured, in this worktree, on this branch:
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **Whether containerd's device cgroup permits a non-privileged pod to `open()` a bind-mounted char
|
||||
device. Still unverified, and it is the single largest technical risk in this workstream.**
|
||||
containerd installs a per-container **device cgroup** allow-list, and whether
|
||||
`/dev/dri/renderD128` arriving as a bind-mounted device node is permitted by it on this
|
||||
k3s/containerd version is **unmeasured**. Settling it requires applying a pod, which is an
|
||||
operator action, not an agent one. The container's PID 1 running as uid 0 helps on the DAC side
|
||||
but does **not** bypass the cgroup gate. If it blocks, the mount will succeed, the image's group
|
||||
hook will do its job, and the `open()` will still fail — and the fallback is the Intel GPU device
|
||||
plugin, which *sets* the cgroup entry. Note that that fallback is a **renderer-only change**: no
|
||||
ServiceDefinition moves, because the authoring surface is an intent enum. That is the design
|
||||
justifying itself, and it is the reason to keep the enum even if the hostPath mechanism loses.
|
||||
- **Whether a non-privileged pod can actually `open()` this hostPath char device at all. This is
|
||||
the largest technical risk in the workstream and nobody has tested it.** `supplementalGroups` gets
|
||||
the DAC permission bits right, but containerd installs a **device cgroup** allow-list per
|
||||
container, and whether `/dev/dri/renderD128` reaching the container as a bind-mounted device node
|
||||
is also permitted by that cgroup on this k3s/containerd version is **unmeasured**. If it is not,
|
||||
the mount will succeed, the group will be present, and the `open()` will still fail `EPERM` — and
|
||||
the design would need the device plugin (which sets the cgroup entry) rather than a hostPath. The
|
||||
first real consumer proves or disproves this; nothing in this PR does.
|
||||
- **That a missing device degrades silently — and nothing will tell you.** This is the accepted cost
|
||||
of `DirectoryOrCreate`. The workload will software-transcode and **no signal reports it**: media
|
||||
apps in this catalog declare no `metricsEndpoint`, so no ServiceMonitor exists, and there is no
|
||||
@@ -150,21 +123,19 @@ Measured, in this worktree, on this branch:
|
||||
is worse than none — see the 2026-09-13 correction in `renderer-contract.md` §`observability`,
|
||||
where twelve monitors discovered zero targets. The only proof of hardware transcode is an operator
|
||||
seeing it in the application's own dashboard.
|
||||
- **The feature now depends on a hook inside an image that floats.** The `cont-init.d` behaviour
|
||||
above was read from digest `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), but the
|
||||
deployment runs `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. **A silent pull can
|
||||
change the behaviour this feature depends on, with no commit anywhere in this repo to show for
|
||||
it** — and the failure would be the silent software-transcode described above. I have not fixed
|
||||
that here (it is the Plex wiring's tag/digest decision, not the capability's), but it should not
|
||||
be forgotten.
|
||||
- **GID drift is NOT on this list any more, deliberately.** An earlier revision pinned
|
||||
`INTEL_RENDER_GID = 992` and listed "a node rebuild could move it" as an open risk. The hook
|
||||
re-`stat`s the device on every start, so drift is self-healing and no GID is pinned anywhere. The
|
||||
risk is retired, not overlooked.
|
||||
- **Anything on the live cluster.** I ran no `kubectl` and I opened no shell in any container. The
|
||||
node facts (device present, `root:render` 0660, `i915` bound, no device plugin, no
|
||||
`gpu.intel.com/*` advertised), the in-container measurements quoted above, and the Kyverno
|
||||
posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups`
|
||||
- **Whether GID 992 survives a node rebuild.** It was measured on all three prod nodes on
|
||||
2026-09-30 and was identical on each, but `render` is a *systemd dynamic* group allocation, not a
|
||||
Debian/LSB static GID. A reinstall that creates system groups in a different order can move it,
|
||||
and nothing in this repo would notice. Recorded in the constant's comment; re-measure `getent
|
||||
group render` on every node after any rebuild.
|
||||
- **Whether `supplementalGroups` is the load-bearing mechanism or belt-and-braces.** A concurrent
|
||||
agent was checking whether the pms-docker image's `45-plex-hw-transcode-and-connected-tuner` init
|
||||
hook resolves the render group dynamically at startup. **No finding reached me**, so the constant's
|
||||
comment describes the GID and its provenance without claiming which mechanism does the work.
|
||||
Update that comment if the answer arrives.
|
||||
- **Anything on the live cluster.** I ran no `kubectl`. The node facts above (device present,
|
||||
`root:render` 0660, GID 992, `i915` bound, no device plugin, no `gpu.intel.com/*` advertised) and
|
||||
the Kyverno posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups`
|
||||
restriction; no PSA labels on app namespaces) are **prior recon handed to me, not re-measured by
|
||||
me**. Per CLAUDE.md §2b, treat them as second-hand until re-queried.
|
||||
- **That CI exercises the self-tests. It does not.** `git grep -- --selftest ci/` returns nothing —
|
||||
|
||||
@@ -169,29 +169,16 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
Intel GPU device plugin (`resources.limits: gpu.intel.com/i915`) is a renderer change with **zero**
|
||||
catalog edits and zero re-review of per-service device wiring. Extend the enum for a genuinely
|
||||
different capability class (e.g. `nvidia-nvenc`); never to name a second route to the same one.
|
||||
- `gpu: intel-quicksync` → on the **app container only** (never a sidecar), exactly two things:
|
||||
- `gpu: intel-quicksync` → on the **app container / pod spec only** (never a sidecar):
|
||||
1. a **hostPath volume** `dri` → `/dev/dri`, **`type: DirectoryOrCreate`**;
|
||||
2. a **read-write volumeMount** of `dri` at `/dev/dri` on the app container (VA-API opens the
|
||||
render node read-write, so it is deliberately not `readOnly`).
|
||||
|
||||
**Mounting the device is sufficient. That is the whole mapping.**
|
||||
- **`podSecurityContext.supplementalGroups` is deliberately NOT rendered, and adding it would be a
|
||||
regression rather than a hardening.** `/dev/dri/renderD128` is `root:render` `0660`, so the obvious
|
||||
move is to push the host `render` GID in — and it does not work. **Measured in the running
|
||||
container on prod-2, 2026-09-30:** the image's own
|
||||
`/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` hook `find`s every char device under
|
||||
`/dev/dri`, `stat -c '%g'` each one's GID, `groupadd`s a matching group *inside the container* when
|
||||
none exists, and `usermod -a -G` the app user into it — every start, nothing hardcoded.
|
||||
`cont-init.d` completes before `services.d`, so the account database is already correct when the
|
||||
service execs. The service then starts via `exec s6-setuidgid plex …`, and `s6-setuidgid`
|
||||
**replaces** the process group list from `/etc/group` rather than appending to it:
|
||||
`Plex Media Server` runs as `Uid: 1000 Gid: 1000 Groups: 100 1000`, with root's gid `0` — which
|
||||
PID 1 (`s6-svscan`) does hold — **gone**. A Kubernetes `supplementalGroups` therefore reaches
|
||||
`s6-svscan` and is discarded before the app ever starts. It would be inert *and* it would mislead
|
||||
whoever debugs this next. The renderer asserts its own absence in `render.py --selftest`.
|
||||
- A corollary worth having: because the hook re-`stat`s the device on every start, the host GID
|
||||
**drifting** (a node rebuild renumbering the *systemd dynamic* `render` group) is self-healing.
|
||||
No GID is pinned anywhere in this repo.
|
||||
render node read-write, so it is deliberately not `readOnly`);
|
||||
3. the host `render` **GID merged into `podSecurityContext.supplementalGroups`** — merged, never
|
||||
clobbered, so an SD that declares its own supplemental groups keeps all of them and gains one.
|
||||
`/dev/dri/renderD128` is `root:render` mode `0660`, so a non-root container needs the group.
|
||||
The GID is the single constant `INTEL_RENDER_GID` in `catalog/render.py` (992, **measured
|
||||
2026-09-30 on all three prod nodes**; it is a *systemd dynamic* allocation, not a static GID,
|
||||
so a node rebuild can move it — re-measure `getent group render` before assuming).
|
||||
- **`DirectoryOrCreate` is load-bearing — do not tighten it to `Directory`.** A node that comes back
|
||||
without the render device (rebuild, BIOS change, driver regression, hardware swap) must degrade the
|
||||
workload to **software transcoding**, not fail the mount. `apps-wave-0` is `wait: true` and
|
||||
@@ -339,9 +326,9 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
Derived from `exposure` alone; enforced on every gate run by
|
||||
`conformance/tests/l0-static/internal-services-publish-no-public-dns` (R-DNS-03).
|
||||
8. **`hardware` is inert until opted into, and its mechanism is renderer-local.** An SD with no
|
||||
`hardware` key renders no volume and no volumeMount; an SD *with* it gains no `securityContext`
|
||||
either. Both directions are proven by `render.py --selftest`. The `/dev/dri` hostPath and the
|
||||
`DirectoryOrCreate` type appear in `catalog/render.py` and nowhere else, so the capability's
|
||||
`hardware` key renders no volume, no volumeMount and no `securityContext` — proven by
|
||||
`render.py --selftest`. The `/dev/dri` hostPath, the `DirectoryOrCreate` type and
|
||||
`INTEL_RENDER_GID` appear in `catalog/render.py` and nowhere else, so the capability's
|
||||
implementation can be replaced without touching a single ServiceDefinition.
|
||||
|
||||
## Requirement traceability (contract-level)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
|
||||
"title": "ServiceDefinition",
|
||||
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 Line truncated
|
||||
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with a /dev/dri hostPath volume (`DirectoryOrCreate`), a read-write /dev/dri volumeMount on the app container, and the host `render` GID merged into podSecurityContext.supplementalGroups, and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "source"],
|
||||
@@ -141,7 +141,7 @@
|
||||
},
|
||||
"hardware": {
|
||||
"type": "object",
|
||||
"description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume and a matching mount on the app container, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",
|
||||
"description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume + the host's `render` GID in supplementalGroups, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",
|
||||
"additionalProperties": false,
|
||||
"minProperties": 1,
|
||||
"properties": {
|
||||
|
||||
Reference in new issue
Block a user