Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5
|
1040d253be |
No files matched your search
@@ -133,6 +133,15 @@ OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1"
|
||||
OAUTH2_PROXY_PORT = 4180
|
||||
KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
|
||||
|
||||
# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------
|
||||
# The ServiceDefinition declares the CAPABILITY (schemas/servicedefinition.schema.json
|
||||
# `hardware.gpu`); everything below is the MECHANISM that currently satisfies it and is
|
||||
# deliberately confined to this file. Swapping to the Intel GPU device plugin
|
||||
# (`gpu.intel.com/i915` resource requests instead of a hostPath) is a change here and
|
||||
# nowhere else -- no catalog entry moves.
|
||||
DRI_VOLUME_NAME = "dri"
|
||||
DRI_DEVICE_DIR = "/dev/dri"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Defaults (renderer-contract.md "Default posture") — materialize BEFORE mapping.
|
||||
@@ -483,6 +492,46 @@ def r_deployment(s: dict) -> dict:
|
||||
"persistentVolumeClaim": {"claimName": f"{s['name']}-{vol['name']}"},
|
||||
}
|
||||
)
|
||||
# hardware capability (schema-optional): `hardware.gpu: intel-quicksync` means this
|
||||
# workload needs the host iGPU's render node for hardware video transcode. The SD says
|
||||
# only that; the wiring below is the platform's current mechanism for satisfying it.
|
||||
#
|
||||
# `type: DirectoryOrCreate` is DELIBERATE and load-bearing -- do NOT "tighten" it to
|
||||
# `Directory`. A node that comes back without the render device (rebuild, BIOS change,
|
||||
# driver regression, hardware swap) must degrade the workload to SOFTWARE transcoding,
|
||||
# not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` dependsOn it, so a
|
||||
# failed hostPath mount stalls the entire app tier for the sake of a feature that is not
|
||||
# load-bearing. `DirectoryOrCreate` turns that failure mode into "transcodes in
|
||||
# software" instead of "the app tier does not reconcile". /dev is tmpfs, so the empty
|
||||
# directory the kubelet creates on such a node leaves nothing durable behind.
|
||||
#
|
||||
# The mount is deliberately NOT readOnly: VA-API opens the render node read-write.
|
||||
#
|
||||
# `podSecurityContext.supplementalGroups` is DELIBERATELY NOT SET here, and adding it
|
||||
# would be a regression, not a hardening. MEASURED in the running container on prod-2,
|
||||
# 2026-09-30: the image's own cont-init hook
|
||||
# `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` `find`s every char device
|
||||
# under /dev/dri, `stat -c '%g'` each one's GID, `groupadd`s a matching group INSIDE the
|
||||
# container when none exists, and `usermod -a -G` the app user into it -- every start,
|
||||
# nothing hardcoded. cont-init.d completes before services.d, so the account database
|
||||
# s6 reads at exec time already carries the group. The service is then started with
|
||||
# `exec s6-setuidgid plex ...`, and `s6-setuidgid` REPLACES the process group list from
|
||||
# /etc/group rather than appending to it: `Plex Media Server` runs with
|
||||
# `Uid: 1000 Gid: 1000 Groups: 100 1000` -- root's gid 0, which PID 1 (s6-svscan) does
|
||||
# have, is GONE. Anything Kubernetes puts in supplementalGroups reaches s6-svscan and is
|
||||
# then discarded before the app ever starts. MOUNTING THE DEVICE IS SUFFICIENT; the
|
||||
# group handling is the image's job and it does it correctly. A supplementalGroups entry
|
||||
# here would be inert AND would mislead whoever debugs this next.
|
||||
hw_gpu = (s.get("hardware") or {}).get("gpu")
|
||||
if hw_gpu == "intel-quicksync":
|
||||
volume_mounts.append({"name": DRI_VOLUME_NAME, "mountPath": DRI_DEVICE_DIR})
|
||||
volumes.append(
|
||||
{
|
||||
"name": DRI_VOLUME_NAME,
|
||||
"hostPath": {"path": DRI_DEVICE_DIR, "type": "DirectoryOrCreate"},
|
||||
}
|
||||
)
|
||||
|
||||
if volume_mounts:
|
||||
container["volumeMounts"] = volume_mounts
|
||||
|
||||
@@ -2440,6 +2489,82 @@ def _selftest() -> int:
|
||||
"an SD without podSecurityContext must not gain a pod securityContext",
|
||||
)
|
||||
|
||||
# 6b) hardware.gpu: intel-quicksync -> /dev/dri hostPath volume + RW mount + the host
|
||||
# `render` GID. The SD declares the CAPABILITY; these assertions pin the MECHANISM
|
||||
# that currently satisfies it, so a change to it is a deliberate, visible edit.
|
||||
hw_docs = render(
|
||||
{
|
||||
"name": "hw",
|
||||
"source": {"image": "a:1"},
|
||||
"hardware": {"gpu": "intel-quicksync"},
|
||||
}
|
||||
)
|
||||
hw_pod = next(
|
||||
d[1] for d in hw_docs if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
{
|
||||
"name": "dri",
|
||||
"hostPath": {"path": "/dev/dri", "type": "DirectoryOrCreate"},
|
||||
}
|
||||
in hw_pod.get("volumes", []),
|
||||
"hardware.gpu=intel-quicksync must render a /dev/dri hostPath volume "
|
||||
"with type DirectoryOrCreate (a GPU-less node must degrade, not fail the mount)",
|
||||
)
|
||||
hw_mounts = hw_pod["containers"][0].get("volumeMounts", [])
|
||||
check(
|
||||
{"name": "dri", "mountPath": "/dev/dri"} in hw_mounts,
|
||||
"hardware.gpu=intel-quicksync must mount /dev/dri READ-WRITE on the app container",
|
||||
)
|
||||
# The device mount is the WHOLE feature: no pod securityContext is invented. The image's
|
||||
# own cont-init hook adds the app user to the device's group inside the container, and
|
||||
# s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway (see the
|
||||
# emission site). Asserting the absence is what stops someone adding an inert one back.
|
||||
check(
|
||||
"securityContext" not in hw_pod,
|
||||
"hardware.gpu must NOT invent a pod securityContext (supplementalGroups is inert "
|
||||
"under s6-setuidgid; the image's cont-init hook handles the device group)",
|
||||
)
|
||||
# Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce)
|
||||
# rejects `privileged: true` in an app namespace, and placement is a separate concern
|
||||
# with its own justification -- this field is about the DEVICE only.
|
||||
check(
|
||||
"privileged" not in hw_pod.get("securityContext", {})
|
||||
and "securityContext" not in hw_pod["containers"][0]
|
||||
and "nodeSelector" not in hw_pod,
|
||||
"hardware.gpu must emit no privileged securityContext and no nodeSelector",
|
||||
)
|
||||
# The regression that matters: absence renders NOTHING. Mirrors the podSecurityContext
|
||||
# negative above -- an SD that does not opt in must be byte-identical to before.
|
||||
nohw_pod = next(
|
||||
d[1] for d in render({"name": "nohw", "source": {"image": "a:1"}})
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
"volumes" not in nohw_pod
|
||||
and "volumeMounts" not in nohw_pod["containers"][0]
|
||||
and "securityContext" not in nohw_pod,
|
||||
"an SD without `hardware` must gain no dri volume, no mount and no securityContext",
|
||||
)
|
||||
# An SD that declares BOTH keeps its own podSecurityContext exactly as written -- the
|
||||
# hardware field neither adds to it nor rewrites it.
|
||||
hwm_pod = next(
|
||||
d[1]
|
||||
for d in render(
|
||||
{
|
||||
"name": "hwm",
|
||||
"source": {"image": "a:1"},
|
||||
"hardware": {"gpu": "intel-quicksync"},
|
||||
"podSecurityContext": {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
}
|
||||
)
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
"hardware.gpu must leave the SD's own podSecurityContext untouched",
|
||||
)
|
||||
|
||||
# 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that
|
||||
# fronts the Gateway path (fail-closed), while oidc/native/none apps are unchanged.
|
||||
fa_docs = render(
|
||||
|
||||
@@ -1,283 +0,0 @@
|
||||
---
|
||||
date: 2026-09-30
|
||||
branch: feat/node-health-render-device
|
||||
title: The Intel render device is now self-re-measuring, not a fact someone checked once
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
### Why this probe has value at all
|
||||
|
||||
**Nothing else in the platform would notice the device going missing.** Plex exposes no
|
||||
ServiceMonitor, so no alert fires on a silent fall back to software transcoding. And the
|
||||
manifest side mounts the device with `hostPath type: DirectoryOrCreate`, which means a node
|
||||
*without* the device mounts an **empty directory** — Plex starts, runs, and degrades quietly
|
||||
rather than failing loudly. There is no crash loop, no failed mount, no alert: just a hot CPU.
|
||||
This check is the only thing in the system that would surface it, which is the actual
|
||||
justification for the role carrying it.
|
||||
|
||||
### The change
|
||||
|
||||
`roles/node_health` grew a graphics / render-device probe. Nothing was installed, no module
|
||||
was loaded, no group was changed, nothing was rebooted: the prod nodes already have a working
|
||||
Intel render device (measured by hand on all three on 2026-09-30) and the point of this change
|
||||
is that the measurement now **re-runs forever** instead of decaying silently in a handoff note.
|
||||
|
||||
The failure this exists to catch is silent by construction. Plex hardware-transcodes through
|
||||
`/dev/dri/renderD128`; if that node stops existing — node rebuild, a BIOS change that disables
|
||||
the onboard IGD, a kernel that binds a different DRM driver, a driver regression — Plex falls
|
||||
back to software transcoding on its own and the only symptom is a hot CPU.
|
||||
|
||||
Four files:
|
||||
|
||||
- `files/node_health_collect.py` — new `collect_graphics()`, wired into the **unprivileged**
|
||||
pass (`main()`). Everything it reads is world-readable: `lstat()` on `/dev/dri/*`,
|
||||
`grp.getgrgid()` for the owning group name, the `/sys/class/drm/card*/device/driver` symlink
|
||||
target for the bound driver, `/sys/bus/pci/devices/*/class` for display-class (PCI base class
|
||||
`0x03`) devices whether or not a driver is bound, `/sys/class/drm/card*/engine/*` for the
|
||||
`vcs*`/`vecs*` media engines, `/proc/modules`, `/proc/cmdline`, and
|
||||
`/sys/module/{i915,xe}/parameters/enable_guc`. `lspci` and `vainfo` were added to the
|
||||
existing tool probe; both are optional and neither is depended on.
|
||||
- `tasks/assess.yml` — one shared derive task plus three checks: `graphics.render_device`
|
||||
(PASS/FAIL/SKIP), `graphics.drm_driver` (PASS/WARN/FAIL/SKIP), `graphics.media_stack`
|
||||
(INFO/SKIP).
|
||||
- `defaults/main.yml` — `node_health_graphics_expected_driver: i915`, so the one comparison the
|
||||
check makes is data rather than task logic.
|
||||
- `templates/summary.txt.j2` — a `GRAPHICS / RENDER DEVICE` section, and
|
||||
`roles/node_health/README.md` gained a section explaining the design and two new "known
|
||||
limits".
|
||||
|
||||
Three design rulings worth stating, because each is the kind of thing a later edit would get
|
||||
wrong:
|
||||
|
||||
1. **The render node's GID is reported, never asserted.** It read `root:render`, gid **992**,
|
||||
mode `0660` on all three prod nodes. The check does *not* compare against `992`. `render` is
|
||||
a **systemd dynamic** allocation (`systemd-sysusers`), not a Debian static gid from
|
||||
`base-passwd`, so a rebuild can legitimately move it — and Plex's container entrypoint stats
|
||||
the device and joins whichever group owns it, so drift genuinely is not a fault. The
|
||||
observed group name and number appear in the detail line, which makes drift *visible*
|
||||
without making it an error. A hard `!= 992` would be a false alarm waiting for a reinstall.
|
||||
2. **The PCI scan is deliberately separate from the DRM scan.** `/sys/class/drm/card*` exists
|
||||
only once a driver has *bound*, so a GPU with nothing attached has no `card0` and would
|
||||
otherwise be indistinguishable from a machine with no GPU. Scanning the PCI bus separately
|
||||
is what lets "no display device at all" be an honest **SKIP** (a VM guest — the same
|
||||
convention `thermal.max` already uses for "no sensor on this hardware") while "a GPU is on
|
||||
the bus and there is still no render node" is a **FAIL**.
|
||||
3. **GuC/HuC and host VAAPI userspace are INFO and must stay INFO.** GuC/HuC off
|
||||
(`i915.enable_guc=0`) is an accepted state here and costs low-power H.264 encode *quality*
|
||||
only. No host `vainfo` is not a defect at all — Plex ships its own libva and media driver
|
||||
inside its container. Grading either would put a permanent WARN on a chosen configuration,
|
||||
or invite someone to `apt install vainfo` on a node to clear a health check, and this role
|
||||
installs nothing.
|
||||
|
||||
`xe` bound instead of `i915` is a **WARN**, not a FAIL: the device still works, but the two
|
||||
drivers do not share a userspace media stack. On these nodes `xe.ko.xz` exists on disk and is
|
||||
bound to nothing.
|
||||
|
||||
## What I verified
|
||||
|
||||
### The real prod run (read-only, all three nodes)
|
||||
|
||||
`--list-hosts` first, every time. It named exactly three hosts and nothing else:
|
||||
|
||||
play #1 (k3s_prod): Inspect node hardware health (read-only)
|
||||
pattern: ['k3s_prod']
|
||||
hosts (3): k3s-prod-3 k3s-prod-2 k3s-prod-1
|
||||
|
||||
Then `--syntax-check` (rc 0), then the run:
|
||||
|
||||
ansible-playbook playbooks/node-health.yml -e node_health_hosts=k3s_prod --limit k3s_prod
|
||||
|
||||
k3s-prod-1 : ok=51 changed=3 unreachable=0 failed=0 skipped=1
|
||||
k3s-prod-2 : ok=51 changed=2 unreachable=0 failed=0 skipped=1
|
||||
k3s-prod-3 : ok=51 changed=2 unreachable=0 failed=0 skipped=1
|
||||
PLAY rc=0
|
||||
|
||||
(The `changed=` counts are the *controller-side* artifact writes under
|
||||
`infra/ansible/_artifacts/node-health/`, delegated to localhost. Nothing on any node changed —
|
||||
the collector task is `changed_when: false` and the role touches nothing.)
|
||||
|
||||
All three nodes graded **PASS / PASS / INFO** on the new checks, 0 FAIL overall:
|
||||
|
||||
k3s-prod-1 VERDICT 0 FAIL · 1 WARN · 18 PASS · 2 SKIP · 6 INFO
|
||||
k3s-prod-2 VERDICT 0 FAIL · 3 WARN · 15 PASS · 3 SKIP · 6 INFO
|
||||
k3s-prod-3 VERDICT 0 FAIL · 2 WARN · 17 PASS · 2 SKIP · 6 INFO
|
||||
|
||||
[PASS] graphics.render_device — DRM render node present for hardware transcoding (/dev/dri/renderD*)
|
||||
[PASS] graphics.drm_driver — DRM driver bound to the display device (expected i915)
|
||||
[INFO] graphics.media_stack — Media engines, GuC/HuC firmware and host VAAPI userspace (informational)
|
||||
|
||||
(The pre-existing WARNs and SKIPs are other checks — SMART, memattr, resolvers — and are not
|
||||
mine; none of them moved.)
|
||||
|
||||
The new report section, **byte-identical on all three nodes**:
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
GRAPHICS / RENDER DEVICE (the Plex hardware-transcoding path)
|
||||
--------------------------------------------------------------------------------
|
||||
pci 0000:00:02.0 00:02.0 VGA compatible controller [0300]: Intel Corporation CoffeeLake-S GT2 [UHD Graphics 630] [8086:3e98] (rev 02)
|
||||
driver i915
|
||||
drm card0 driver i915 at 0000:00:02.0 engines bcs0 rcs0 vcs0 vecs0
|
||||
dev card0 character special mode 0660 root:video gid 44
|
||||
dev renderD128 character special mode 0660 root:render gid 992
|
||||
GuC/HuC NOT READ — /sys/module/i915/parameters/enable_guc is not
|
||||
readable without root (i915 module parameters are mode 0400)
|
||||
no i915/xe parameter on the kernel cmdline
|
||||
modules amdgpu=false, i915=true, nouveau=false, nvidia=false, radeon=false, xe=false (/proc/modules — corroboration only; the bound-driver symlink above decides)
|
||||
vainfo NOT installed on this host — not a defect: Plex ships its own libva/media driver inside its container
|
||||
|
||||
That reproduces every item of the 2026-09-30 hand measurement independently: `renderD128`
|
||||
character special, `root:render`, **gid 992**, mode `0660`; `card0` `root:video` gid 44 mode
|
||||
`0660`; `i915` bound; `xe` **not** loaded and bound to nothing; `[8086:3e98] (rev 02)`
|
||||
CoffeeLake-S GT2 UHD Graphics 630 at `00:02.0`; `vcs0` and `vecs0` live; no host `vainfo`.
|
||||
|
||||
**One measured correction to my own phase-1 assumption:** `lspci` **is** present on all three
|
||||
prod nodes (`/usr/bin/lspci` — Debian ships `pciutils` by default). I had written that it was
|
||||
absent. It is still *treated* as optional, because `pciutils` is genuinely not declared in
|
||||
`node_baseline_packages` and depending on an undeclared package would be wrong — but the
|
||||
comments and README that asserted it was missing were factually wrong and are now corrected.
|
||||
Its presence is why the `pci` line above carries a readable device name rather than bare IDs.
|
||||
|
||||
### The gate
|
||||
|
||||
DEV_KUBECONFIG=/dev/null ENGINE=<shim> ./infra/devcontainer/dev.sh run bash -c './ci/gate.sh'
|
||||
GATE rc=3
|
||||
|
||||
**rc 3 — UNDETERMINED. NOT green.** Nothing failed, and nothing is certified either. Exactly
|
||||
three stages did not run, all three the known local ceiling from a worktree, and **no fourth
|
||||
skip**:
|
||||
|
||||
[skip] spec-immutability spec repo not present at /work/../home-network-k3s (set SPEC_REPO)
|
||||
[skip] gitea-settings SETTINGS_API_TOKEN is not set — cannot read the live Gitea rule
|
||||
[skip] conformance no cluster configured (CLUSTER_AVAILABLE!=true) — L2/L3 NOT RUN
|
||||
|
||||
[ ok ] decisions-log: pass [ ok ] lint: pass [ ok ] l0: pass
|
||||
[ ok ] render-golden: pass [ ok ] l1: pass [ ok ] approval-policy: pass
|
||||
|
||||
Both CLAUDE.md §3 worktree workarounds were needed and both worked: `DEV_KUBECONFIG=/dev/null`
|
||||
(the credential in the main checkout was **not** copied) and an `ENGINE` shim that bind-mounts
|
||||
the main checkout's `.git` read-only. No `tracked_files()` traceback appeared, which is how I
|
||||
know the shim took effect rather than the git stages silently degrading.
|
||||
|
||||
`ansible-lint` was **not** run, because **`ci/` does not invoke it anywhere** — I grepped
|
||||
`ci/` and `.gitea/` and the lint stage is `yamllint -c ci/yamllint.yaml` only. I am not
|
||||
claiming to have matched a lint convention that does not exist.
|
||||
|
||||
### The CI gate on the PR — a green tick that is exit 3, read from the log
|
||||
|
||||
The required context `gate / gate (pull_request)` reported **`success`, "Successful in 1m40s"**
|
||||
on commit `a78f010`. That description matters: a *skipped* Gitea job reports `success` with
|
||||
"Has been skipped", so this one genuinely ran, and 1m40s matches CLAUDE.md §2's ~1m35s.
|
||||
|
||||
**But the green tick does NOT mean `gate.sh` returned 0.** `.gitea/workflows/gate.yml:138` is
|
||||
`run: ./ci/gate.sh || [ "$?" -eq 3 ]` — the job tolerates rc 3 by design. So I fetched the job
|
||||
log (`/actions/runs/173/jobs/0/logs`; the `/api/v1/.../tasks/{id}/logs` endpoints 404 on this
|
||||
Gitea, the web route serves it) and read the gate's own headline:
|
||||
|
||||
[fail] MERGE GATE: UNDETERMINED — greenness was NOT ESTABLISHED.
|
||||
[fail] Stages that did not run: spec-immutability
|
||||
|
||||
**So CI is also rc 3, with exactly ONE stage skipped — `spec-immutability`.** The green Gitea
|
||||
check is the tolerance swallowing that 3, not a rc 0.
|
||||
|
||||
**CI verified two things my local run could not, and both came back green:**
|
||||
|
||||
[ ok ] stage gitea-settings: pass
|
||||
[ ok ] enable_push = false enable_status_check = true
|
||||
[ ok ] required_approvals = 1 block_admin_merge_override = true
|
||||
[ ok ] block_on_rejected_reviews = true dismiss_stale_approvals = true
|
||||
[ ok ] runners online: 1 (>= 1)
|
||||
[ ok ] Gitea settings match the declaration.
|
||||
[ ok ] stage approval-policy: pass
|
||||
[ ok ] no architecturally/security-relevant paths touched — the GATE is the approver.
|
||||
|
||||
My local run **skipped** `gitea-settings` (no `SETTINGS_API_TOKEN`) and could not give
|
||||
`approval-policy` a PR number. CI has the secret and `REQUIRE_GITEA_SETTINGS: "true"`, so a skip
|
||||
there would have failed the job. That is measured confirmation that the live Gitea matches
|
||||
`infra/gitea/expected-branch-protection.json`, and that `infra/ansible/**` needs no human
|
||||
reviewer. `lint` passed over 405 files. `conformance` SKIPPED in CI too
|
||||
(`CLUSTER_AVAILABLE != true`), so L2/L3 remains unverified there as well.
|
||||
|
||||
Corroborated independently: `GET /repos/.../branches/main` →
|
||||
`protected=true user_can_push=false enable_status_check=true required_approvals=1`,
|
||||
contexts `["gate / gate (pull_request)"]`; `GET /admin/actions/runners` → `incluster-dind
|
||||
status=online`. And note the commit's **combined** state is `pending`, not success, because
|
||||
`gate / conformance (pull_request)` sits at "Blocked by required conditions" — that context is
|
||||
not in the required list, so it does not block, but the PR is not "all green" either.
|
||||
|
||||
The gate's `lint` stage passed with one `line too long` **warning** on
|
||||
`roles/node_health/tasks/assess.yml:189` — that is pre-existing, in the SMART block, well
|
||||
above my insertion point, and warnings do not fail the stage.
|
||||
|
||||
### Offline work (before the lock was released)
|
||||
|
||||
- `python3 -m py_compile` on the collector: clean.
|
||||
- `collect_graphics()` executed for real against this workstation's hardware (an unrelated
|
||||
machine, Raptor Lake-S UHD Graphics + an NVIDIA discrete GPU) and returned the whole
|
||||
structure with no exception, correctly reporting two cards, two render nodes, the bound
|
||||
driver per card, the `vcs0/vcs1/vecs0` media engines, and `enable_guc` as *not readable
|
||||
without root* rather than as a fabricated `0`.
|
||||
- That last point is a real finding and changed the design: `/sys/module/i915/parameters/*` is
|
||||
**mode 0400**, so an unprivileged pass cannot read `enable_guc`. It now reports
|
||||
`NOT READ — <reason>` and `/proc/cmdline` is read as the world-readable second source. An
|
||||
earlier draft would have silently reported a null.
|
||||
- Two other bugs found and fixed by running it rather than reasoning about it:
|
||||
`/dev/dri/by-path` is a **directory** and was being `lstat()`ed as a device; and
|
||||
`/sys/class/drm` carries one entry per *connector* (`card0-HDMI-A-1`), which had to be
|
||||
excluded from the card scan.
|
||||
- All three checks and the new template section were rendered through a standalone Jinja
|
||||
harness (Ansible is not on the host — CLAUDE.md §5) against **seven** fact sets: the real
|
||||
workstation; GPU present with the render node removed; no display device at all; `xe` bound
|
||||
in place of `i915`; a display device with **no** driver bound; `renderD128` present but not a
|
||||
character device; and a shape with neither `lspci` nor `vainfo`. Zero template errors, and
|
||||
the statuses came out as designed — `FAIL/PASS/INFO`, `SKIP/SKIP/SKIP`, `PASS/WARN/INFO`,
|
||||
`FAIL/FAIL/INFO`, `FAIL/PASS/INFO` respectively. The harness supplies stand-ins for the
|
||||
Ansible-only filters (`bool`, `dict2items`, `flatten`, `zip`, `regex_replace`) and matches
|
||||
Ansible's `trim_blocks=True`.
|
||||
**Read that for exactly what it is: evidence that the LOGIC is right, and no evidence
|
||||
whatsoever that the HARDWARE DETECTION is.** Six of those seven fact sets were hand-mutated
|
||||
dictionaries, not machines. They are not hardware tests and must not be written up as if
|
||||
they were.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **The FAIL path has never fired on a real node.** Every prod node has a working render
|
||||
device, and this role is read-only, so a device cannot be taken away to prove the check
|
||||
catches its absence. The failure branches were exercised against **synthetic fact sets
|
||||
only** (see above). I am not claiming the failure path is tested against hardware.
|
||||
- **`xe` detection is likewise untested against reality.** No node here has `xe` bound — the
|
||||
module exists on disk as `xe.ko.xz` and is bound to nothing — so the WARN branch was only
|
||||
ever driven by a mutated fact set. If `xe` ever does bind, the *shape* of
|
||||
`/sys/class/drm/card*` under it is something I have assumed matches i915's (same
|
||||
`device/driver` symlink, same `engine/*` directories). That assumption is unverified.
|
||||
- **The media-engine sysfs path is confirmed on these nodes but not proven portable.**
|
||||
`vcs0`/`vecs0` were read from `/sys/class/drm/card0/engine/*` on all three prod nodes, so it
|
||||
is measured here. I did **not** confirm against kernel source that no other kernel version or
|
||||
driver publishes engines elsewhere. If one does, the check reports "none exposed" — an INFO
|
||||
line, not a false verdict, which is why this failure mode is tolerable.
|
||||
- **`enable_guc` is still unread on prod.** The report says `NOT READ — mode 0400`, which is
|
||||
honest but means I have **not** confirmed by measurement that GuC/HuC is off on these nodes;
|
||||
I only know the kernel cmdline carries no `i915.enable_guc=` override. The brief's ground
|
||||
truth says `i915.enable_guc=0`; my check does not corroborate that value, it declines to
|
||||
guess it. Reading it would need root, and it is INFO either way, so I left it alone.
|
||||
- **Not a transcode test.** These checks prove the render *device* exists, is a character
|
||||
device, and which driver owns it. They do **not** prove a transcode succeeds — that needs a
|
||||
VAAPI round trip inside Plex's own container with Plex's own libva, and is out of scope for a
|
||||
node-level check that installs nothing.
|
||||
- **Greenness was never established, locally or in CI.** Both runs returned **rc 3**. My local
|
||||
run skipped three stages (`spec-immutability`, `gitea-settings`, `conformance`); CI skipped
|
||||
one (`spec-immutability`) and its own headline says `MERGE GATE: UNDETERMINED`. **I am not
|
||||
describing this PR as green**, and the green Gitea tick must not be read as one: the workflow
|
||||
line is `./ci/gate.sh || [ "$?" -eq 3 ]`, so it is a tolerated 3.
|
||||
- **L2/L3 conformance did not run anywhere** — `CLUSTER_AVAILABLE != true` in both. Whatever
|
||||
runtime requirements this touches remain UNVERIFIED; only the L0/L1 offline subset passed.
|
||||
- **`spec-immutability` ran in neither.** The sibling spec repo is checked out in neither my
|
||||
worktree nor the CI workspace, so nothing confirms this change does not drift from the spec.
|
||||
It is the expected, tolerated skip — but it is still a skip, not a pass.
|
||||
- **No claim about the live platform beyond what was printed.** Per CLAUDE.md §2b this repo is
|
||||
not evidence about prod; the only prod facts I assert are the ones quoted verbatim above.
|
||||
Branch protection and runner state I did query directly, and the CI `gitea-settings` stage
|
||||
passed — so those two I assert on measurement, not on this file's prose.
|
||||
- **The stale dev inventory was left alone deliberately.** `hosts.ini` still defines the
|
||||
`k3s-dev-*` groups as the default target of most playbooks and the dev cluster no longer
|
||||
exists. That is real, it is out of this change's scope, and it is unclaimed work.
|
||||
@@ -0,0 +1,176 @@
|
||||
---
|
||||
date: 2026-09-30
|
||||
branch: feat/sd-hardware-capability
|
||||
title: ServiceDefinition gains a `hardware` capability field — `gpu: intel-quicksync` mounts /dev/dri and nothing else; the enum is what keeps the mechanism swappable
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
A new optional top-level ServiceDefinition property, `hardware`, with one optional member,
|
||||
`gpu`, whose enum today has exactly one value: `intel-quicksync`. **No catalog entry uses it.**
|
||||
This PR delivers the capability; consuming it (Plex) is separate work on top.
|
||||
|
||||
- `schemas/servicedefinition.schema.json` — adds `hardware` (object, `additionalProperties: false`,
|
||||
`minProperties: 1`) with `gpu` (enum `["intel-quicksync"]`). Schema version bumped **1.7.0 →
|
||||
1.8.0**, changelog entry prepended to the schema `description` per that file's own convention.
|
||||
- `catalog/render.py` — two module-level constants (`DRI_VOLUME_NAME`, `DRI_DEVICE_DIR`) and the
|
||||
emission in `r_deployment`.
|
||||
- `schemas/renderer-contract.md` — new `### hardware` field-mapping section, plus cross-field
|
||||
invariant **8** in that file's numbered list.
|
||||
- `schemas/README.md` — its `**Schema version:**` line said `1.0.0`; it had been stale since 1.1.0.
|
||||
Synced to 1.8.0 and annotated that the schema `description` is the changelog of record. Added one
|
||||
bullet to "Cross-field rules" for the new enum. That was the whole of the staleness; nothing
|
||||
ballooned.
|
||||
|
||||
**What the renderer emits** for `hardware.gpu: intel-quicksync`, on the app container only (never a
|
||||
sidecar), and it is exactly two things:
|
||||
|
||||
1. a hostPath volume `dri` → `/dev/dri`, `type: DirectoryOrCreate`;
|
||||
2. a read-write `volumeMount` of `dri` at `/dev/dri` on the app container (deliberately not
|
||||
`readOnly` — VA-API opens the render node read-write).
|
||||
|
||||
**Mounting the device is sufficient.** It emits **no** `privileged: true` (Kyverno
|
||||
`restrict-host-namespaces` is Enforce on app namespaces and it is not needed), **no** `nodeSelector`
|
||||
(placement is a separate concern with its own justification), **no** resource requests (there is no
|
||||
app-container resource render path), and — the one that is worth spelling out — **no
|
||||
`podSecurityContext.supplementalGroups`**.
|
||||
|
||||
**Why no `supplementalGroups`, when `/dev/dri/renderD128` is `root:render 0660`.** An earlier
|
||||
revision of this branch merged the host `render` GID into `supplementalGroups`. **It was measured to
|
||||
be a no-op and has been removed.** In the running container on prod-2, 2026-09-30:
|
||||
|
||||
PID 1 s6-svscan Uid: 0 Gid: 0 Groups: 0
|
||||
pid 246 Plex Media Server Uid: 1000 Gid: 1000 Groups: 100 1000
|
||||
|
||||
`/etc/services.d/plex/run` ends in `exec s6-setuidgid plex …`, and the child's group list is exactly
|
||||
the `/etc/group` account-database list — root's gid `0`, which PID 1 holds, has **vanished**. That is
|
||||
a `setgroups()` that *replaces* rather than appends, so anything Kubernetes puts in
|
||||
`supplementalGroups` reaches `s6-svscan` and is discarded before the app ever starts. What actually
|
||||
works is the image's own `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner`, which `find`s
|
||||
every char device under `/dev/dri`, `stat -c '%g'`s its GID, `groupadd`s a matching group inside the
|
||||
container when none exists, and `usermod -a -G`s the app user into it — every start, nothing
|
||||
hardcoded; `cont-init.d` completes before `services.d`, so the database is correct at exec time.
|
||||
|
||||
Shipping an inert `supplementalGroups` would have been worse than shipping nothing: it would read as
|
||||
the mechanism to whoever debugs this next. The reasoning lives in a comment **at the emission site**
|
||||
and in `renderer-contract.md`, and `render.py --selftest` now asserts the *absence* of a
|
||||
`securityContext`, so it cannot be "helpfully" added back without a test failing. A pleasant
|
||||
corollary: because the hook re-`stat`s the device every start, host GID drift is **self-healing**,
|
||||
and no GID is pinned anywhere in this repo.
|
||||
|
||||
**The design decision worth defending: the field is an enum of intent, not a device path.** The SD
|
||||
says *what class of host device the workload needs*; the hostPath and the mount exist only in
|
||||
`catalog/render.py`. Migrating to the Intel GPU device plugin (`gpu.intel.com/i915` resource
|
||||
requests, no hostPath at all) is then a renderer change with zero catalog edits. Both the schema
|
||||
`description` and the contract section say this explicitly, because the obvious "simplification" —
|
||||
letting an SD name `/dev/dri` — would push a platform implementation detail into the operator-facing
|
||||
contract and make that migration a catalog-wide edit.
|
||||
|
||||
**`DirectoryOrCreate` is load-bearing.** A node that comes back without the render device (rebuild,
|
||||
BIOS change, driver regression, hardware swap) must degrade the workload to software transcoding,
|
||||
not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` `dependsOn` it, so a failed
|
||||
hostPath mount stalls the entire app tier for the sake of a feature that is not load-bearing.
|
||||
`/dev` is tmpfs, so the empty directory the kubelet creates on such a node leaves nothing durable
|
||||
behind. Do not "tighten" it to `Directory`. The reasoning is recorded at the emission site.
|
||||
|
||||
## What I verified
|
||||
|
||||
Measured, in this worktree, on this branch:
|
||||
|
||||
- **`python3 catalog/render.py --selftest` → `render.py selftest: PASS`, rc 0.** New cases in the
|
||||
existing `check(...)` style: the positive (volume + mount, and *no* `securityContext`, *no*
|
||||
privileged, *no* nodeSelector), the negative that matters (an SD without `hardware` gains no
|
||||
volume, no volumeMount and no `securityContext` — mirroring the existing `podSecurityContext`
|
||||
negative), and a pass-through case (an SD declaring both `hardware` and its own
|
||||
`podSecurityContext: {fsGroup: 1001, supplementalGroups: [44]}` keeps it **exactly** as written).
|
||||
- **The self-tests actually bite.** I mutated the guard (`if hw_gpu == "intel-quicksync":` → `if
|
||||
True:`) in a scratch copy and re-ran: **rc 1**, failing on "an SD without `hardware` must gain no
|
||||
dri volume, no mount and no securityContext". A test that cannot fail is not a test; this one can.
|
||||
- **Re-render is a zero diff.**
|
||||
`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation` then
|
||||
`git diff --stat apps/` → **empty**, and `git status --porcelain apps/` → **empty** (so the two
|
||||
not-onboarded by-products, `apps/wave-1/keycloak.yaml` and `apps/wave-0/vm-demo.yaml`, were
|
||||
rewritten byte-identically and needed no revert). The change is inert until a service opts in.
|
||||
- **Schema validity and catalog conformance.** `Draft202012Validator.check_schema` passes; all
|
||||
**12** `catalog/*.yaml` validate with **0** errors. On the new field specifically: `gpu:
|
||||
intel-quicksync` accepted; `gpu: /dev/dri` rejected (enum); `hardware: {gpu: …, path: …}` rejected
|
||||
(`additionalProperties: false`); `hardware: {}` rejected (`minProperties: 1`).
|
||||
- **Hand-checked the emitted YAML** by rendering a throwaway SD carrying `hardware`, a Longhorn
|
||||
volume and its own `podSecurityContext: {runAsUser: 1000, supplementalGroups: [44]}`. Result:
|
||||
`volumeMounts` = config then `{name: dri, mountPath: /dev/dri}`; `volumes` = the PVC then
|
||||
`{name: dri, hostPath: {path: /dev/dri, type: DirectoryOrCreate}}`; `securityContext` =
|
||||
`{runAsUser: 1000, supplementalGroups: [44]}` — i.e. **unchanged**, nothing appended. That toy SD
|
||||
was never written into `catalog/`.
|
||||
- **GATE: `rc 3` — UNDETERMINED, NOT green.** Run in the pinned toolbox from this worktree with
|
||||
both CLAUDE.md §3 workarounds (`DEV_KUBECONFIG=/dev/null`, and an `ENGINE` shim bind-mounting the
|
||||
main checkout's `.git` at its own absolute path). Diff base `origin/main @ 0fbf544d5e2f`; the run's
|
||||
`HEAD` was `d7df88c9c264`, which differs from the final commit only by this table being written
|
||||
into this file. Touched requirements `R-DNS-03 R-GITOPS-02 R-IDP-02`. Per stage:
|
||||
|
||||
| stage | result |
|
||||
|---|---|
|
||||
| spec-immutability | **SKIPPED** — sibling spec repo not at `/work/../home-network-k3s`. Expected from a worktree; tolerated by §3. |
|
||||
| decisions-log | pass |
|
||||
| lint | pass |
|
||||
| l0 | pass — 19 offline cases, `0 failure(s), 0 skip(s), 0 unwired` |
|
||||
| render-golden | pass — *"all 10 onboarded service golden(s) reproduce from the canonical render"* |
|
||||
| l1 | pass — 3 offline checks, `0 failure(s), 0 skip(s), 0 unwired`; coverage 51/51 requirements scaffolded |
|
||||
| approval-policy | pass (`event='none'`, not a pull_request event — N/A locally; it will evaluate for real on the PR) |
|
||||
| gitea-settings | **SKIPPED** — `SETTINGS_API_TOKEN` unset. Expected; branch protection and runner registration are therefore UNVERIFIED by this run. |
|
||||
| conformance (L2/L3) | **SKIPPED** — `CLUSTER_AVAILABLE!=true`. Phase-0 standing skip. |
|
||||
|
||||
The gate's own verdict: *"Stages that did not run: spec-immutability gitea-settings"*. **No stage
|
||||
failed, and no stage skipped for a reason caused by this change.** rc 3 is reported as rc 3; I am
|
||||
not calling this green.
|
||||
- **Blast radius of the schema change:** `hardware` is absent from every catalog entry, so
|
||||
`apply_defaults` never materializes it and `s.get("hardware")` is `None` on all 12 — which is what
|
||||
the zero diff above demonstrates rather than asserts.
|
||||
- **Touched no sensitive path.** Changes are confined to `schemas/`, `catalog/`, `decisions/`; per
|
||||
`infra/gitea/approval-policy.json` none of those are sensitive. `conformance/**` and `clusters/**`
|
||||
are untouched. I did not run `--emit-waves` (it writes `clusters/base/`, and the set of files
|
||||
under `apps/` did not change).
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **Whether containerd's device cgroup permits a non-privileged pod to `open()` a bind-mounted char
|
||||
device. Still unverified, and it is the single largest technical risk in this workstream.**
|
||||
containerd installs a per-container **device cgroup** allow-list, and whether
|
||||
`/dev/dri/renderD128` arriving as a bind-mounted device node is permitted by it on this
|
||||
k3s/containerd version is **unmeasured**. Settling it requires applying a pod, which is an
|
||||
operator action, not an agent one. The container's PID 1 running as uid 0 helps on the DAC side
|
||||
but does **not** bypass the cgroup gate. If it blocks, the mount will succeed, the image's group
|
||||
hook will do its job, and the `open()` will still fail — and the fallback is the Intel GPU device
|
||||
plugin, which *sets* the cgroup entry. Note that that fallback is a **renderer-only change**: no
|
||||
ServiceDefinition moves, because the authoring surface is an intent enum. That is the design
|
||||
justifying itself, and it is the reason to keep the enum even if the hostPath mechanism loses.
|
||||
- **That a missing device degrades silently — and nothing will tell you.** This is the accepted cost
|
||||
of `DirectoryOrCreate`. The workload will software-transcode and **no signal reports it**: media
|
||||
apps in this catalog declare no `metricsEndpoint`, so no ServiceMonitor exists, and there is no
|
||||
metric to alert on. I deliberately did **not** propose an alert, because an alert that cannot fire
|
||||
is worse than none — see the 2026-09-13 correction in `renderer-contract.md` §`observability`,
|
||||
where twelve monitors discovered zero targets. The only proof of hardware transcode is an operator
|
||||
seeing it in the application's own dashboard.
|
||||
- **The feature now depends on a hook inside an image that floats.** The `cont-init.d` behaviour
|
||||
above was read from digest `sha256:e0ab27395614…` (`plexmediaserver 1.43.4.10903`), but the
|
||||
deployment runs `plexinc/pms-docker:latest` with `imagePullPolicy: Always`. **A silent pull can
|
||||
change the behaviour this feature depends on, with no commit anywhere in this repo to show for
|
||||
it** — and the failure would be the silent software-transcode described above. I have not fixed
|
||||
that here (it is the Plex wiring's tag/digest decision, not the capability's), but it should not
|
||||
be forgotten.
|
||||
- **GID drift is NOT on this list any more, deliberately.** An earlier revision pinned
|
||||
`INTEL_RENDER_GID = 992` and listed "a node rebuild could move it" as an open risk. The hook
|
||||
re-`stat`s the device on every start, so drift is self-healing and no GID is pinned anywhere. The
|
||||
risk is retired, not overlooked.
|
||||
- **Anything on the live cluster.** I ran no `kubectl` and I opened no shell in any container. The
|
||||
node facts (device present, `root:render` 0660, `i915` bound, no device plugin, no
|
||||
`gpu.intel.com/*` advertised), the in-container measurements quoted above, and the Kyverno
|
||||
posture (`restrict-host-namespaces` Enforce; no hostPath or `supplementalGroups`
|
||||
restriction; no PSA labels on app namespaces) are **prior recon handed to me, not re-measured by
|
||||
me**. Per CLAUDE.md §2b, treat them as second-hand until re-queried.
|
||||
- **That CI exercises the self-tests. It does not.** `git grep -- --selftest ci/` returns nothing —
|
||||
no gate stage invokes the renderer's self-test. They are the executable spec and house style, and
|
||||
I ran them by hand, but a future regression in this code path **will not be caught by the gate**.
|
||||
Wiring `--selftest` into a gate stage is a real gap and is not part of this PR (`ci/` is a
|
||||
sensitive path and would drag in a human reviewer for a change unrelated to this one).
|
||||
- **Runtime behaviour of any kind.** The gate proves the L0/L1 offline subset only;
|
||||
`vars.CLUSTER_AVAILABLE` is unset, so L2/L3 conformance does not run in CI.
|
||||
@@ -19,9 +19,9 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
|
||||
|
||||
## Contract
|
||||
|
||||
- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu`
|
||||
(and `lspci` only if it happens to be installed), and — only when it can escalate —
|
||||
`smartctl`. It writes nothing to the node and **installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is
|
||||
- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu`,
|
||||
and — only when it can escalate — `smartctl`. It writes nothing to the node and
|
||||
**installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is
|
||||
absent the SMART checks SKIP with that as the stated reason (operator ruling 2026-08-31:
|
||||
package installation is `node_baseline`'s job, never a health check's, never a manual
|
||||
`apt-get`).
|
||||
@@ -50,9 +50,6 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
|
||||
| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite **and whether it survives a reboot** (see [Swap off is not the same as swap disabled](#swap-off-is-not-the-same-as-swap-disabled)), EDAC correctable/uncorrectable counters |
|
||||
| `memory.physical_total` | **is this machine using the RAM it has?** `/proc/zoneinfo` present-vs-managed pages (no root, metal or VM) and `MemTotal` vs the DIMM capacity SMBIOS reports (`dmidecode -t 17`, root, bare metal). Cause-independent by design — see [The EFI memattr check](#the-efi-memattr-check) |
|
||||
| `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) |
|
||||
| `graphics.render_device` | is there a usable `/dev/dri/renderD*` character device — the path Plex hardware-transcodes through. See [The render-device check](#the-render-device-check) |
|
||||
| `graphics.drm_driver` | which DRM driver is bound to the display device, against `node_health_graphics_expected_driver` (`i915` here) |
|
||||
| `graphics.media_stack` | **INFO only**: `vcs*`/`vecs*` media engines, GuC/HuC firmware state, host VAAPI userspace. None of these is ever graded — see below for why |
|
||||
| `thermal.max` | hottest sensor across thermal zones and hwmon chips |
|
||||
| `pressure.cpu` / `.memory` / `.io` | PSI `some avg60` — dev learned to care about CPU PSI specifically |
|
||||
| `network.link`, `network.errors`, `network.resolvers` | link state/speed, NIC error counters, DNS upstream redundancy per address family |
|
||||
@@ -94,78 +91,6 @@ used to blame that state on "this node has not been baselined yet", which sent t
|
||||
back to re-run the role that had already run.
|
||||
|
||||
|
||||
## The render-device check
|
||||
|
||||
Plex transcodes on the iGPU through `/dev/dri/renderD128`. **When that node stops existing,
|
||||
nothing fails loudly.** Plex falls back to software transcoding by itself; the only symptom
|
||||
is a hot CPU and stuttering clients. A node rebuild, a BIOS change that disables the onboard
|
||||
IGD, a kernel that binds a different DRM driver, or a driver regression all produce exactly
|
||||
that silent degradation.
|
||||
|
||||
The state was measured by hand on all three prod nodes on 2026-09-30 and found working. **A
|
||||
fact measured once decays silently**, so it is re-measured on every health run instead of
|
||||
living in a runbook. Nothing is installed, no module is loaded, no group is changed — the
|
||||
role's read-only contract is unchanged.
|
||||
|
||||
| fact | where it comes from | root? |
|
||||
|---|---|---|
|
||||
| `/dev/dri` contents, ownership, mode, char-vs-not | `lstat()` on each entry | no |
|
||||
| owning group **name** for the render node's gid | `grp.getgrgid()` | no |
|
||||
| bound DRM driver | `/sys/class/drm/card*/device/driver` symlink target | no |
|
||||
| is the iGPU on the bus at all | `/sys/bus/pci/devices/*/class` (PCI base class `0x03`) | no |
|
||||
| media engines (`vcs*`, `vecs*`) | `/sys/class/drm/card*/engine/*` | no |
|
||||
| module load state | `/proc/modules` | no |
|
||||
| GuC/HuC | `/sys/module/i915/parameters/enable_guc` + `/proc/cmdline` | **parameter file is mode 0400** |
|
||||
|
||||
### The PCI scan is separate from the DRM scan on purpose
|
||||
|
||||
`/sys/class/drm/card*` exists only once a driver has **bound**. A machine whose iGPU sits on
|
||||
the bus with nothing attached has no `card0` at all — and that is precisely the failure being
|
||||
hunted. So the display device is looked for on the PCI bus independently, from sysfs, which
|
||||
is always there. That is what lets `graphics.render_device` distinguish
|
||||
|
||||
- *no GPU in this machine* → **SKIP** (a VM guest; nothing to grade, the same convention
|
||||
`thermal.max` uses for "no sensor on this hardware"), from
|
||||
- *a GPU is on the bus and there is still no render node* → **FAIL**.
|
||||
|
||||
`lspci` is used **only** to decorate the PCI entry with a readable name, and it is treated as
|
||||
optional because it is: `pciutils` is **not** declared in `node_baseline_packages`, so nothing
|
||||
in this repo guarantees it. It did turn out to be present on all three prod nodes
|
||||
(`/usr/bin/lspci`, measured 2026-09-30 — Debian installs `pciutils` by default), but depending
|
||||
on that would mean depending on an undeclared package. Absent, `description` is `null`, the
|
||||
numeric IDs are printed instead, and no verdict changes.
|
||||
|
||||
### The render node's GID is reported, never asserted
|
||||
|
||||
The render device was `root:render`, **gid 992**, mode `0660` on all three prod nodes on
|
||||
2026-09-30. The check does **not** compare against `992`, and must not be changed to.
|
||||
`render` is a **systemd dynamic allocation** (`systemd-sysusers`), not a Debian static gid
|
||||
from `base-passwd`, so a rebuilt node can legitimately land on a different number. Plex's own
|
||||
container entrypoint stats the device and joins whichever group owns it, so drift genuinely
|
||||
is not a fault. Recording the observed group name and gid in the detail line makes drift
|
||||
**visible** without making it an error — a hard `!= 992` would be a false alarm waiting for a
|
||||
reinstall.
|
||||
|
||||
What *is* configurable is the expected DRM driver, `node_health_graphics_expected_driver`
|
||||
(default `i915`), so the verdict stays out of task logic. `xe` bound instead of `i915` is a
|
||||
**WARN**, not a FAIL: the device still works, but the two drivers do not share a userspace
|
||||
media stack and a libva/iHD build that transcodes correctly against one is not automatically
|
||||
right for the other. On these nodes `xe.ko.xz` exists on disk and is bound to nothing.
|
||||
|
||||
### Two things that are informational and must stay that way
|
||||
|
||||
- **GuC/HuC firmware not loaded** (`i915.enable_guc=0`) is a known, accepted state here. It
|
||||
costs low-power H.264 encode *quality* only; decode and encode both still run on the media
|
||||
engines. Grading it would put a permanent WARN on a chosen configuration.
|
||||
- **No host VAAPI userspace / no `vainfo`** is **not a defect**. Plex ships its own libva and
|
||||
media driver inside its container, so the host needs neither. Flagging it would invite
|
||||
someone to `apt install vainfo` on a node to clear a health check — and this role installs
|
||||
nothing.
|
||||
|
||||
`enable_guc` is one of i915's `0400` module parameters, so an unprivileged pass usually
|
||||
cannot read it. It reports as an honest `NOT READ — <reason>` rather than a fabricated `0`,
|
||||
with `/proc/cmdline` as the world-readable second source for an explicit boot override.
|
||||
|
||||
## The EFI memattr check
|
||||
|
||||
`infra/PROD-PROVISIONING.md` §0b is authoritative; this is only what the check needs.
|
||||
@@ -348,10 +273,3 @@ stays `null` — no claim is made either way.
|
||||
but the wear/endurance context SKIPs rather than guessing.
|
||||
- `ioerr_cnt` counts commands the kernel saw fail, which includes benign rejected or
|
||||
unsupported commands during discovery. Attribution needs `dmesg`, which needs root.
|
||||
- `graphics.render_device`'s **FAIL path has never fired against a real node** — every prod
|
||||
node has a working render device, and the role is read-only so one cannot be taken away to
|
||||
prove it. The failure branches were exercised against synthetic fact sets only. The same
|
||||
applies to the `xe`-bound WARN: no node here has `xe` bound.
|
||||
- `graphics.*` proves the render **device** exists and which driver owns it. It does **not**
|
||||
prove a transcode succeeds — that needs a VAAPI round trip inside Plex's own container,
|
||||
with Plex's own libva, which is out of scope for a node-level check that installs nothing.
|
||||
@@ -35,24 +35,6 @@ node_health_try_privileged: true
|
||||
# --nocheck=standby: never spin up a sleeping disk just to read it. Read-only hygiene.
|
||||
node_health_smartctl_args: "--json=c --all --nocheck=standby"
|
||||
|
||||
# --- Graphics / render device ---------------------------------------------------------
|
||||
# The DRM driver this site EXPECTS to be bound to its display device. Measured on all three
|
||||
# prod nodes 2026-09-30: `i915` on the CoffeeLake-S GT2 [UHD Graphics 630] iGPU of every HP
|
||||
# EliteDesk 800 G5 Mini. The newer `xe` module is present on disk as xe.ko.xz but is bound to
|
||||
# nothing, and that difference matters enough to surface: the two drivers do not share a
|
||||
# userspace media stack (a libva/iHD build that works against i915 is not automatically the
|
||||
# right one for xe), so a silent switch would change Plex's transcoding path. A site on AMD
|
||||
# or NVIDIA sets this to `amdgpu` / `nouveau` / `nvidia` rather than editing task logic.
|
||||
#
|
||||
# WHAT IS DELIBERATELY *NOT* A SETTING HERE: the numeric GID of the render node's owning
|
||||
# group. `render` is a systemd DYNAMIC allocation (systemd-sysusers), not a Debian static
|
||||
# one, so a rebuilt node may legitimately land on a different number — it was 992 on all
|
||||
# three prod nodes on 2026-09-30, and an assertion against that literal would be a false
|
||||
# alarm waiting to happen. Plex's own container entrypoint stats the device and joins
|
||||
# whichever group owns it, so drift genuinely is not a fault. The observed group name and GID
|
||||
# are therefore REPORTED in the check detail and never graded.
|
||||
node_health_graphics_expected_driver: i915
|
||||
|
||||
# --- Thresholds ---------------------------------------------------------------------
|
||||
# All verdicts come from here, so a site can retune without touching task logic.
|
||||
node_health_thresholds:
|
||||
|
||||
@@ -35,13 +35,10 @@
|
||||
# =============================================================================
|
||||
|
||||
import glob
|
||||
import grp
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -759,212 +756,6 @@ def collect_pressure():
|
||||
return out
|
||||
|
||||
|
||||
# --- Graphics / DRM render device -----------------------------------------------------
|
||||
# WHY THIS IS HERE AT ALL. Plex transcodes on the iGPU through /dev/dri/renderD128. If that
|
||||
# node stops existing — a node rebuild, a BIOS change that disables the IGD, a kernel that
|
||||
# binds a different DRM driver — nothing breaks loudly: Plex silently falls back to software
|
||||
# transcoding and the only symptom is a hot CPU. So the render device is measured on every
|
||||
# health run rather than being a fact someone established once by hand.
|
||||
#
|
||||
# EVERYTHING BELOW IS UNPRIVILEGED. /dev/dri is world-traversable, the device nodes are
|
||||
# stat()able by anyone, /sys/class/drm and /sys/bus/pci/devices are world-readable, and so
|
||||
# are the module parameters under /sys/module. Nothing here needs root, so nothing here
|
||||
# belongs in the privileged pass. (GuC/HuC *load* status lives in debugfs, which does need
|
||||
# root AND debugfs mounted; the module PARAMETER below is the unprivileged stand-in and is
|
||||
# what actually decides whether the firmware is loaded at all.)
|
||||
#
|
||||
# THE PCI SCAN IS DELIBERATELY SEPARATE FROM THE DRM SCAN. /sys/class/drm/card* only exists
|
||||
# once a driver has BOUND. A machine whose iGPU is on the bus with no driver bound has no
|
||||
# card0 at all, and that is precisely the failure this is meant to catch — so the display
|
||||
# device is looked for on the PCI bus independently, from sysfs, which is always there.
|
||||
# `lspci` is used ONLY to decorate that with a human-readable name, and it is treated as
|
||||
# optional because it IS optional: pciutils is not declared in node_baseline_packages, so
|
||||
# nothing in this repo guarantees it. It did turn out to be present on all three prod nodes
|
||||
# (/usr/bin/lspci, measured 2026-09-30 — Debian installs pciutils by default), but relying on
|
||||
# that would make the collector depend on an undeclared package. A missing lspci leaves
|
||||
# `description` null, the numeric IDs are reported instead, and no verdict changes.
|
||||
DISPLAY_PCI_CLASS_PREFIX = "0x03" # PCI base class 03 = display controller (VGA/3D/other)
|
||||
# DRM drivers worth reporting the load state of. The bound-driver symlink is authoritative
|
||||
# (a built-in driver never appears in /proc/modules); this is corroboration only, and it is
|
||||
# how "xe.ko.xz exists on disk but is bound to nothing" becomes a visible fact.
|
||||
DRM_MODULES = ("i915", "xe", "amdgpu", "radeon", "nouveau", "nvidia")
|
||||
|
||||
|
||||
def _uid_name(uid):
|
||||
try:
|
||||
return pwd.getpwuid(uid).pw_name
|
||||
except (KeyError, OverflowError, TypeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _gid_name(gid):
|
||||
"""Group NAME for a numeric gid, or None when nothing resolves it.
|
||||
|
||||
Reported as an OBSERVED value, never asserted against a literal. The `render` group is
|
||||
a systemd DYNAMIC allocation (it is created by systemd-sysusers, not fixed by Debian's
|
||||
static base-passwd list), so its gid can legitimately differ after a rebuild. Plex's own
|
||||
container init stats the device and joins whatever group owns it, so a moved gid is
|
||||
drift to notice, not a fault to fail on."""
|
||||
try:
|
||||
return grp.getgrgid(gid).gr_name
|
||||
except (KeyError, OverflowError, TypeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _module_param(mod, param):
|
||||
"""One /sys/module/<mod>/parameters/<param>, with WHY it is missing when it is.
|
||||
|
||||
Three outcomes that must not be conflated: a real value; "the module exposes no such
|
||||
parameter" (not loaded, built in, or a driver that simply has no such knob — xe has no
|
||||
enable_guc, GuC is mandatory there); and "the file exists but is mode 0400". Most i915
|
||||
module parameters are `i915_param_named(..., 0400, ...)` — root-only — so an
|
||||
unprivileged pass legitimately cannot read them. Returning null with the reason keeps
|
||||
that an honest UNKNOWN instead of a fabricated 0. read_text() also records the
|
||||
permission denial in UNREADABLE, so the summary lists it with everything else root
|
||||
would add."""
|
||||
path = "/sys/module/%s/parameters/%s" % (mod, param)
|
||||
if not os.path.exists(path):
|
||||
return {"value": None, "reason": "%s exposes no %s parameter (module not loaded, "
|
||||
"built in, or no such knob on this driver)" % (mod, param)}
|
||||
value = read_text(path)
|
||||
if value is None:
|
||||
return {"value": None, "reason": "%s is not readable without root (i915 module "
|
||||
"parameters are mode 0400)" % path}
|
||||
return {"value": value, "reason": None}
|
||||
|
||||
|
||||
def _dev_node(path):
|
||||
"""stat() one /dev/dri entry. lstat, so a symlink is reported as the symlink it is."""
|
||||
try:
|
||||
st = os.lstat(path)
|
||||
except OSError as exc:
|
||||
UNREADABLE.append({"path": path, "reason": "stat failed: %s" % exc})
|
||||
return None
|
||||
return {
|
||||
"path": path,
|
||||
"name": os.path.basename(path),
|
||||
"char_device": stat.S_ISCHR(st.st_mode),
|
||||
"mode": "%04o" % stat.S_IMODE(st.st_mode),
|
||||
"uid": st.st_uid,
|
||||
"gid": st.st_gid,
|
||||
"owner": _uid_name(st.st_uid),
|
||||
"group": _gid_name(st.st_gid),
|
||||
}
|
||||
|
||||
|
||||
def collect_graphics():
|
||||
dri_dir = "/dev/dri"
|
||||
dri_present = os.path.isdir(dri_dir)
|
||||
devices = []
|
||||
if dri_present:
|
||||
for path in sorted(glob.glob(dri_dir + "/*")):
|
||||
# /dev/dri/by-path/ is a directory of symlinks, not a device. Skipping
|
||||
# directories rather than filtering to char devices is deliberate: anything
|
||||
# in here that is NOT a character special file is an anomaly worth seeing, and
|
||||
# `char_device: false` is how it becomes visible.
|
||||
if os.path.isdir(path):
|
||||
continue
|
||||
entry = _dev_node(path)
|
||||
if entry is not None:
|
||||
devices.append(entry)
|
||||
render_nodes = [d for d in devices if d["name"].startswith("renderD")]
|
||||
|
||||
# Bound DRM devices. /sys/class/drm also holds one entry per CONNECTOR
|
||||
# (card0-HDMI-A-1, card0-DP-1 ...) — those are outputs, not devices, and are skipped.
|
||||
cards = []
|
||||
for cdir in sorted(glob.glob("/sys/class/drm/card[0-9]*")):
|
||||
name = os.path.basename(cdir)
|
||||
if "-" in name:
|
||||
continue
|
||||
driver = None
|
||||
drvlink = cdir + "/device/driver"
|
||||
if os.path.islink(drvlink):
|
||||
driver = os.path.basename(os.path.realpath(drvlink))
|
||||
address = None
|
||||
devlink = cdir + "/device"
|
||||
if os.path.islink(devlink):
|
||||
address = os.path.basename(os.path.realpath(devlink))
|
||||
# i915/xe publish one directory per hardware engine. vcs* is a video (H.264/HEVC)
|
||||
# engine and vecs* the video-enhancement engine: those two are what a transcode
|
||||
# actually runs on, so their presence is the difference between "a GPU is bound"
|
||||
# and "this GPU can do media work".
|
||||
engines = sorted(
|
||||
{
|
||||
os.path.basename(p)
|
||||
for p in (glob.glob(cdir + "/engine/*") + glob.glob(cdir + "/device/engine/*"))
|
||||
if os.path.isdir(p)
|
||||
}
|
||||
)
|
||||
cards.append(
|
||||
{
|
||||
"card": name,
|
||||
"driver": driver,
|
||||
"pci_address": address,
|
||||
"vendor_id": read_text(cdir + "/device/vendor"),
|
||||
"device_id": read_text(cdir + "/device/device"),
|
||||
"engines": engines,
|
||||
"media_engines": [e for e in engines if e.startswith(("vcs", "vecs"))],
|
||||
}
|
||||
)
|
||||
|
||||
# Display-class devices on the PCI bus, driver bound or not.
|
||||
pci_display = []
|
||||
for pdir in sorted(glob.glob("/sys/bus/pci/devices/*")):
|
||||
class_id = read_text(pdir + "/class") or ""
|
||||
if not class_id.startswith(DISPLAY_PCI_CLASS_PREFIX):
|
||||
continue
|
||||
address = os.path.basename(pdir)
|
||||
driver = None
|
||||
if os.path.islink(pdir + "/driver"):
|
||||
driver = os.path.basename(os.path.realpath(pdir + "/driver"))
|
||||
description = None
|
||||
rc, out, _ = run(["lspci", "-nn", "-s", address])
|
||||
if rc == 0 and out.strip():
|
||||
description = out.strip().splitlines()[0]
|
||||
pci_display.append(
|
||||
{
|
||||
"address": address,
|
||||
"class_id": class_id or None,
|
||||
"vendor_id": read_text(pdir + "/vendor"),
|
||||
"device_id": read_text(pdir + "/device"),
|
||||
"revision": read_text(pdir + "/revision"),
|
||||
"driver": driver,
|
||||
"description": description,
|
||||
}
|
||||
)
|
||||
|
||||
loaded = set()
|
||||
for line in (read_text("/proc/modules") or "").splitlines():
|
||||
fields = line.split()
|
||||
if fields:
|
||||
loaded.add(fields[0])
|
||||
modules = {m: (m in loaded) for m in DRM_MODULES}
|
||||
|
||||
# GuC/HuC. i915's `enable_guc` is a bitmask: 0 = neither, 1 = GuC submission, 2 = HuC
|
||||
# firmware, 3 = both, -1 = whatever the driver defaults to on this platform. It is the
|
||||
# one place the state is visible without root AND without debugfs — but the sysfs file
|
||||
# is mode 0400, so unprivileged it usually reads back as an honest UNKNOWN. The kernel
|
||||
# command line is the world-readable second source: an explicit `i915.enable_guc=` there
|
||||
# is authoritative about what was ASKED for, even when the parameter file cannot be read.
|
||||
uc = {mod: {"enable_guc": _module_param(mod, "enable_guc")} for mod in ("i915", "xe")}
|
||||
cmdline_params = [
|
||||
token
|
||||
for token in (read_text("/proc/cmdline") or "").split()
|
||||
if token.startswith(tuple("%s." % m for m in DRM_MODULES))
|
||||
]
|
||||
|
||||
return {
|
||||
"dri_present": dri_present,
|
||||
"devices": devices,
|
||||
"render_nodes": render_nodes,
|
||||
"drm_cards": cards,
|
||||
"pci_display": pci_display,
|
||||
"modules": modules,
|
||||
"uc": uc,
|
||||
"cmdline_params": cmdline_params,
|
||||
}
|
||||
|
||||
|
||||
# --- Storage ------------------------------------------------------------------------
|
||||
PSEUDO_FS = {
|
||||
"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "cgroup2", "pstore",
|
||||
@@ -1304,15 +1095,7 @@ def collect_os():
|
||||
# we only DETECT, and the role skips the SMART checks with an accurate reason.
|
||||
def collect_tools():
|
||||
tools = {}
|
||||
# lspci (pciutils) and vainfo (libva-utils) are BOTH optional and NEITHER is declared in
|
||||
# node_baseline_packages, so nothing in this repo guarantees either. Measured on prod
|
||||
# 2026-09-30: lspci IS there (/usr/bin/lspci — Debian ships pciutils by default) and
|
||||
# vainfo is NOT. lspci only decorates the PCI display entry with a readable name; vainfo
|
||||
# is not needed on the host at all, because Plex ships its own libva and media driver
|
||||
# inside its container. They are probed so the report can state their presence or absence
|
||||
# as a fact instead of leaving the reader to wonder — and so nothing here is ever tempted
|
||||
# to install them. This role installs nothing.
|
||||
for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu", "lspci", "vainfo"):
|
||||
for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu"):
|
||||
tools[tool] = which(tool)
|
||||
return tools
|
||||
|
||||
@@ -1344,7 +1127,6 @@ def main():
|
||||
"os": collect_os(),
|
||||
"dmi": collect_dmi(),
|
||||
"cpu": collect_cpu(),
|
||||
"graphics": collect_graphics(),
|
||||
"memory": collect_memory(),
|
||||
"thermal": collect_thermal(),
|
||||
"pressure": collect_pressure(),
|
||||
|
||||
@@ -652,204 +652,6 @@
|
||||
~ (', lscpu Virtualization=' ~ c.virtualization if c.virtualization else '')
|
||||
~ (', running as a guest of ' ~ c.hypervisor_vendor if c.is_virtual_guest | bool else '') }}
|
||||
|
||||
# --- Graphics / render device ----------------------------------------------------------
|
||||
# WHY A HEALTH CHECK AND NOT A NOTE IN A RUNBOOK. Plex transcodes on the iGPU through
|
||||
# /dev/dri/renderD128. When that node stops existing nothing fails loudly — Plex falls back
|
||||
# to software transcoding on its own and the only symptom is a hot CPU and stuttering
|
||||
# clients. A node rebuild, a BIOS setting that disables the onboard IGD, or a kernel that
|
||||
# binds a different DRM driver all produce exactly that silent degradation. It was measured
|
||||
# by hand on all three prod nodes on 2026-09-30 and found working; a fact measured once
|
||||
# decays silently, so it is re-measured on every health run instead.
|
||||
#
|
||||
# READ-ONLY, AND UNPRIVILEGED. Every fact behind these three checks comes from the
|
||||
# unprivileged pass: stat() on /dev/dri, the /sys/class/drm bound-driver symlink,
|
||||
# /sys/bus/pci/devices class codes, /proc/modules and /proc/cmdline. The role installs
|
||||
# nothing (no vainfo, no pciutils), loads no module and changes no group — see the role
|
||||
# README. The ONE fact that is root-gated is i915's enable_guc module parameter (mode 0400),
|
||||
# and it reports as an honest UNKNOWN with the reason rather than as a 0.
|
||||
#
|
||||
# THE THREE VERDICTS ARE SPLIT ON PURPOSE:
|
||||
# graphics.render_device the OUTCOME — can anything hardware-transcode on this node.
|
||||
# graphics.drm_driver the CAUSE side — what is bound, and is it what this site expects.
|
||||
# graphics.media_stack INFO only. GuC/HuC and host VAAPI userspace both belong here and
|
||||
# NEITHER is ever a failure; see the task's own comment for why.
|
||||
- name: Assess — locate the display device and its bound driver (once, shared by three checks)
|
||||
ansible.builtin.set_fact:
|
||||
node_health_render:
|
||||
# Display-class PCI devices (base class 0x03), driver bound or not. This is the
|
||||
# question "is there a GPU in this machine at all", and it is answered from the PCI
|
||||
# bus rather than from /sys/class/drm because a card* entry only exists once a driver
|
||||
# has BOUND — so a GPU with nothing bound to it has no card* and would otherwise look
|
||||
# identical to a machine with no GPU.
|
||||
display_devices: "{{ g.pci_display }}"
|
||||
bound_drivers: >-
|
||||
{{ g.pci_display | map(attribute='driver') | select('string') | unique | list }}
|
||||
# Only a CHARACTER SPECIAL file is a usable render node. Anything else called renderD*
|
||||
# is an anomaly, not a device, and must not be counted as one.
|
||||
render_chardevs: "{{ g.render_nodes | selectattr('char_device') | list }}"
|
||||
# Video (vcs*) and video-enhancement (vecs*) engines across every bound card — the
|
||||
# difference between "a GPU is bound" and "this GPU can do media work".
|
||||
media_engines: >-
|
||||
{{ g.drm_cards | map(attribute='media_engines') | flatten | unique | sort | list }}
|
||||
vars:
|
||||
ansible_become: false
|
||||
g: "{{ node_health_facts.graphics }}"
|
||||
|
||||
- name: Assess — DRM render node for hardware transcoding
|
||||
ansible.builtin.set_fact:
|
||||
node_health_checks: "{{ node_health_checks + [{
|
||||
'id': 'graphics.render_device',
|
||||
'title': 'DRM render node present for hardware transcoding (/dev/dri/renderD*)',
|
||||
'status': render_status,
|
||||
'detail': render_detail }] }}"
|
||||
vars:
|
||||
ansible_become: false
|
||||
g: "{{ node_health_facts.graphics }}"
|
||||
r: "{{ node_health_render }}"
|
||||
# SKIP, not PASS and not FAIL, when the machine has no display device at all: there is
|
||||
# then nothing to grade, and the same convention thermal.max already uses for "no sensor
|
||||
# on this hardware" applies. FAIL is reserved for the state that actually breaks Plex —
|
||||
# a GPU IS on the bus and there is still no render node.
|
||||
render_status: >-
|
||||
{{ 'SKIP' if (r.display_devices | length) == 0
|
||||
else ('PASS' if (r.render_chardevs | length) > 0 else 'FAIL') }}
|
||||
dri_listing: >-
|
||||
{{ (g.devices | map(attribute='name') | join(', ')) if (g.devices | length) > 0
|
||||
else 'nothing' }}
|
||||
render_detail: >-
|
||||
{%- if (r.display_devices | length) == 0 -%}
|
||||
this machine exposes no display-class PCI device (base class 0x03) at all, so there is
|
||||
no GPU here for a render node to belong to and hardware transcoding is not available on
|
||||
this node by construction. Normal for a VM guest. Measured from /sys/bus/pci/devices,
|
||||
not assumed.
|
||||
{%- elif (r.render_chardevs | length) > 0 -%}
|
||||
{{ r.render_chardevs | length }} render node(s):
|
||||
{% for d in r.render_chardevs %}{{ d.path }} (character special, mode {{ d.mode }},
|
||||
owner {{ d.owner | default(d.uid, true) }}, group {{ d.group | default('unresolved', true) }}
|
||||
gid {{ d.gid }}){{ ', ' if not loop.last else '' }}{% endfor %}.
|
||||
/dev/dri holds: {{ dri_listing }}.
|
||||
THE OWNING GROUP AND GID ARE REPORTED AS OBSERVED, NEVER ASSERTED against a literal:
|
||||
`render` is a systemd DYNAMIC allocation rather than a Debian static one, so a rebuilt
|
||||
node may legitimately land on a different number (all three prod nodes read gid 992 on
|
||||
2026-09-30), and Plex's container entrypoint stats the device and joins whichever group
|
||||
owns it. Drift is therefore visible in this line without being a failure.
|
||||
{%- else -%}
|
||||
HARDWARE TRANSCODING IS BROKEN ON THIS NODE. A display-class device IS present on the
|
||||
PCI bus ({{ r.display_devices | map(attribute='address') | join(', ') }}) but there is
|
||||
no usable character-special render node: /dev/dri
|
||||
{{ 'does not exist' if not (g.dri_present | bool) else 'holds ' ~ dri_listing }}.
|
||||
{{ ('Note that /dev/dri holds ' ~ (g.render_nodes | map(attribute='name') | join(', '))
|
||||
~ ' — present, but NOT character special, which is an anomaly in its own right. ')
|
||||
if (g.render_nodes | length) > 0 else '' }}
|
||||
Nothing fails loudly in this state — Plex silently falls back to software transcoding
|
||||
and the only symptom is CPU load. Check graphics.drm_driver below for whether a DRM
|
||||
driver bound at all, then whether the onboard IGD is still enabled in BIOS.
|
||||
{%- endif -%}
|
||||
|
||||
- name: Assess — DRM driver bound to the display device
|
||||
ansible.builtin.set_fact:
|
||||
node_health_checks: "{{ node_health_checks + [{
|
||||
'id': 'graphics.drm_driver',
|
||||
'title': 'DRM driver bound to the display device (expected ' ~ expected ~ ')',
|
||||
'status': driver_status,
|
||||
'detail': driver_detail }] }}"
|
||||
vars:
|
||||
ansible_become: false
|
||||
g: "{{ node_health_facts.graphics }}"
|
||||
r: "{{ node_health_render }}"
|
||||
expected: "{{ node_health_graphics_expected_driver }}"
|
||||
# WARN rather than FAIL when a DIFFERENT driver is bound (the `xe` case): the device
|
||||
# still works, but the userspace media stack is not the same one, so it is a change the
|
||||
# operator must see before a transcode regression surfaces as a user complaint. FAIL is
|
||||
# for nothing bound at all, which is the cause of a missing render node.
|
||||
driver_status: >-
|
||||
{{ 'SKIP' if (r.display_devices | length) == 0
|
||||
else ('FAIL' if (r.bound_drivers | length) == 0
|
||||
else ('PASS' if expected in r.bound_drivers else 'WARN')) }}
|
||||
driver_detail: >-
|
||||
{%- if (r.display_devices | length) == 0 -%}
|
||||
no display-class PCI device on this machine, so no DRM driver can be bound to one.
|
||||
Measured from /sys/bus/pci/devices.
|
||||
{%- else -%}
|
||||
{% for p in r.display_devices %}{{ p.address }}
|
||||
{{ p.description | default(((p.vendor_id | default('?', true)) ~ ':' ~ (p.device_id | default('?', true))), true) }}
|
||||
-> driver {{ p.driver | default('NONE BOUND', true) }}{{ '; ' if not loop.last else '' }}{% endfor %}.
|
||||
{{ ('DRM cards: ' ~ (g.drm_cards | map(attribute='card') | zip(g.drm_cards | map(attribute='driver'))
|
||||
| map('join', '=') | join(', ')) ~ '. ') if (g.drm_cards | length) > 0
|
||||
else 'No /sys/class/drm/card* entry exists, which is consistent with no driver being bound. ' }}
|
||||
Module load state (/proc/modules, corroboration only — a built-in driver never appears
|
||||
there, the bound-driver symlink above is what decides):
|
||||
{{ g.modules | dict2items | map(attribute='key') | zip(g.modules | dict2items | map(attribute='value') | map('string'))
|
||||
| map('join', '=') | join(', ') }}.
|
||||
{%- if (r.bound_drivers | length) == 0 %}
|
||||
NOTHING IS BOUND to the display device. That is why there is no render node: the DRM
|
||||
subsystem creates /dev/dri/renderD* only when a driver attaches. Check whether the
|
||||
module exists for this kernel and whether it was blacklisted or failed to probe.
|
||||
{%- elif expected in r.bound_drivers %}
|
||||
This is the expected driver for this site (node_health_graphics_expected_driver).
|
||||
{%- else %}
|
||||
EXPECTED `{{ expected }}`, GOT `{{ r.bound_drivers | join(', ') }}`. The device still
|
||||
works, so this is not an outage — but the two DRM drivers do not share a userspace
|
||||
media stack, and a libva/iHD build that transcodes correctly against one is not
|
||||
automatically right for the other. Confirm Plex still hardware-transcodes before
|
||||
treating this as benign.
|
||||
{%- endif %}
|
||||
{%- endif -%}
|
||||
|
||||
# INFO ONLY, AND THAT IS A DELIBERATE RULING, NOT AN OVERSIGHT:
|
||||
#
|
||||
# GuC/HuC firmware not loaded (i915.enable_guc=0) — the accepted state on these nodes. It
|
||||
# costs low-power H.264 encode QUALITY, nothing else; decode and encode both still work
|
||||
# through the media engines. Grading it would put a permanent WARN on a configuration the
|
||||
# operator has chosen.
|
||||
# No host VAAPI userspace / no `vainfo` — NOT a defect. Plex ships its own libva and media
|
||||
# driver inside its container, so the host needs neither. Flagging it would invite someone
|
||||
# to `apt install vainfo` on a node to clear a health check, and this role installs nothing.
|
||||
#
|
||||
# The media ENGINES are the fact worth watching in this line: vcs* (video) and vecs*
|
||||
# (video-enhancement) are what a transcode actually runs on.
|
||||
- name: Assess — media engines, GuC/HuC firmware and host VAAPI userspace (informational)
|
||||
ansible.builtin.set_fact:
|
||||
node_health_checks: "{{ node_health_checks + [{
|
||||
'id': 'graphics.media_stack',
|
||||
'title': 'Media engines, GuC/HuC firmware and host VAAPI userspace (informational)',
|
||||
'status': ('SKIP' if (node_health_render.display_devices | length) == 0 else 'INFO'),
|
||||
'detail': media_detail }] }}"
|
||||
vars:
|
||||
ansible_become: false
|
||||
g: "{{ node_health_facts.graphics }}"
|
||||
r: "{{ node_health_render }}"
|
||||
# `.get()` rather than a subscript: uc only carries the two Intel drivers, and a site
|
||||
# that set the expected driver to amdgpu must get a clean "no such parameters" line
|
||||
# instead of an undefined-key traceback.
|
||||
guc: "{{ g.uc.get(node_health_graphics_expected_driver) }}"
|
||||
guc_text: >-
|
||||
{{ ('enable_guc=' ~ guc.enable_guc.value) if (guc is mapping and guc.enable_guc.value is not none)
|
||||
else ('NOT READ — ' ~ (guc.enable_guc.reason | default('no reason recorded', true))
|
||||
if guc is mapping
|
||||
else 'no /sys/module/' ~ node_health_graphics_expected_driver ~ '/parameters at all') }}
|
||||
media_detail: >-
|
||||
{%- if (r.display_devices | length) == 0 -%}
|
||||
no display-class PCI device on this machine, so there is no media engine, no GuC/HuC
|
||||
firmware and nothing to transcode with.
|
||||
{%- else -%}
|
||||
media engines:
|
||||
{{ (r.media_engines | join(', ')) if (r.media_engines | length) > 0
|
||||
else 'NONE exposed under /sys/class/drm/card*/engine — a bound driver with no vcs*/vecs* engine cannot do media work' }}
|
||||
(all engines: {{ (g.drm_cards | map(attribute='engines') | flatten | unique | sort | join(', ')) | default('none', true) }}).
|
||||
GuC/HuC: {{ guc_text }}.
|
||||
{{ ('Kernel cmdline carries ' ~ (g.cmdline_params | join(' '))) if (g.cmdline_params | length) > 0
|
||||
else 'No i915/xe parameter on the kernel cmdline' }}.
|
||||
GuC/HuC being off is a KNOWN, ACCEPTED state here and is never graded: it costs
|
||||
low-power H.264 encode quality only, and both decode and encode still run on the
|
||||
engines above.
|
||||
Host VAAPI userspace: vainfo
|
||||
{{ ('present at ' ~ node_health_facts.tools.vainfo) if node_health_facts.tools.vainfo
|
||||
else 'NOT installed on this host' }}
|
||||
— also never graded, because Plex ships its own libva and media driver inside its
|
||||
container, so the host needs none. This role installs nothing.
|
||||
{%- endif -%}
|
||||
|
||||
# --- Thermals -----------------------------------------------------------------------
|
||||
- name: Assess — temperatures
|
||||
ansible.builtin.set_fact:
|
||||
|
||||
@@ -136,37 +136,6 @@
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
GRAPHICS / RENDER DEVICE (the Plex hardware-transcoding path)
|
||||
--------------------------------------------------------------------------------
|
||||
{% set gfx = node_health_facts.graphics %}
|
||||
{% if gfx.pci_display %}
|
||||
{% for p in gfx.pci_display %}
|
||||
pci {{ '%-12s' | format(p.address) }} {{ p.description | default(((p.vendor_id | default('?', true)) ~ ':' ~ (p.device_id | default('?', true)) ~ ' class ' ~ (p.class_id | default('?', true))), true) }}
|
||||
driver {{ p.driver | default('NONE BOUND', true) }}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
pci no display-class device (PCI base class 0x03) on this machine — no GPU here
|
||||
{% endif %}
|
||||
{% for c in gfx.drm_cards %}
|
||||
drm {{ '%-12s' | format(c.card) }} driver {{ '%-8s' | format(c.driver | default('-', true)) }} at {{ c.pci_address | default('-', true) }} engines {{ c.engines | join(' ') | default('none exposed', true) }}
|
||||
{% endfor %}
|
||||
{% if gfx.dri_present %}
|
||||
{% for d in gfx.devices %}
|
||||
dev {{ '%-12s' | format(d.name) }} {{ '%-18s' | format('character special' if d.char_device else 'NOT A CHAR DEVICE') }} mode {{ d.mode }} {{ d.owner | default(d.uid, true) }}:{{ d.group | default('(gid unresolved)', true) }} gid {{ d.gid }}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
dev /dev/dri DOES NOT EXIST — no DRM device node on this node at all
|
||||
{% endif %}
|
||||
{# GID is printed, never asserted: `render` is a systemd dynamic allocation, so a rebuild may
|
||||
legitimately move it and Plex's container init joins whichever group owns the device. See
|
||||
defaults/main.yml (node_health_graphics_expected_driver) for the full reasoning. #}
|
||||
{% set guc = gfx.uc.get(node_health_graphics_expected_driver) %}
|
||||
GuC/HuC {{ ('enable_guc=' ~ guc.enable_guc.value) if (guc and guc.enable_guc.value is not none) else (('NOT READ — ' ~ (guc.enable_guc.reason | default('no reason recorded', true))) if guc else ('no /sys/module/' ~ node_health_graphics_expected_driver ~ '/parameters')) | regex_replace('\s+', ' ') | trim | wordwrap(64) | indent(14) }}
|
||||
{{ ('kernel cmdline: ' ~ (gfx.cmdline_params | join(' '))) if gfx.cmdline_params else 'no i915/xe parameter on the kernel cmdline' }}
|
||||
modules {% for m, loaded in gfx.modules.items() %}{{ m }}={{ loaded | string | lower }}{{ ', ' if not loop.last }}{% endfor %} (/proc/modules — corroboration only; the bound-driver symlink above decides)
|
||||
vainfo {{ node_health_facts.tools.vainfo | default('NOT installed on this host — not a defect: Plex ships its own libva/media driver inside its container', true) }}
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
PRESSURE STALL (/proc/pressure — % of wall time stalled)
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
+4
-1
@@ -15,7 +15,7 @@ docs/13 exactly. Do not diverge from it here — escalate spec problems, don't f
|
||||
| `servicedefinition.schema.json` | Formal JSON Schema (draft 2020-12) for a ServiceDefinition. Enforces enums, required fields, patterns, and cross-field invariants. |
|
||||
| `renderer-contract.md` | Authoritative field → rendered-resource mapping, with requirement IDs. The contract other workstreams consume. |
|
||||
|
||||
**Schema version:** 1.0.0 (in the schema `description`). Treat any change as an interface change:
|
||||
**Schema version:** 1.8.0 (in the schema `description`, which carries the full per-version changelog — this line had said `1.0.0` since 1.1.0 and is now synced). Treat any change as an interface change:
|
||||
propose on the task board, update dependent renderers/tests in the same change, never silently break a
|
||||
consumer (coordination-protocol §2).
|
||||
|
||||
@@ -35,6 +35,9 @@ consumer (coordination-protocol §2).
|
||||
- `ingressRoutes[].host` must match `<label>.home.keithjackson.dev`.
|
||||
- `storage[].class ∈ {longhorn, bulk}`; `exposure ∈ {internal, public}`; `egress ∈ {internet, vpn}`;
|
||||
`auth.mode ∈ {oidc, forward-auth, native, none}`.
|
||||
- `hardware.gpu ∈ {intel-quicksync}` — a host **capability class**, never a device path. The enum
|
||||
is the contract: the mechanism that satisfies it lives only in `catalog/render.py` (see
|
||||
`renderer-contract.md` §`hardware`).
|
||||
- `additionalProperties: false` everywhere — unknown keys are rejected (catches typos in review/CI).
|
||||
|
||||
## Validate examples against the schema — L1, no cluster (R-TEST-02)
|
||||
|
||||
@@ -160,6 +160,58 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
- **Requirements:** **R-STATE-01** (rebuild survival), **R-STATE-02** (Longhorn replication),
|
||||
**R-STATE-04** (bulk tier). (docs/04)
|
||||
|
||||
### `hardware` (`gpu`) — host device capability
|
||||
- **The field is INTENT, never MECHANISM.** `hardware.gpu` is an **enum of capability classes**
|
||||
(today: `intel-quicksync`), not a device path, not a hostPath, not a resource request. The
|
||||
ServiceDefinition says *what class of host device this workload needs*; how the platform hands it
|
||||
over is `catalog/render.py`'s business and lives nowhere else. **Do not "simplify" this into a raw
|
||||
`/dev/...` path.** The entire payoff is that migrating from the hostPath mechanism below to the
|
||||
Intel GPU device plugin (`resources.limits: gpu.intel.com/i915`) is a renderer change with **zero**
|
||||
catalog edits and zero re-review of per-service device wiring. Extend the enum for a genuinely
|
||||
different capability class (e.g. `nvidia-nvenc`); never to name a second route to the same one.
|
||||
- `gpu: intel-quicksync` → on the **app container only** (never a sidecar), exactly two things:
|
||||
1. a **hostPath volume** `dri` → `/dev/dri`, **`type: DirectoryOrCreate`**;
|
||||
2. a **read-write volumeMount** of `dri` at `/dev/dri` on the app container (VA-API opens the
|
||||
render node read-write, so it is deliberately not `readOnly`).
|
||||
|
||||
**Mounting the device is sufficient. That is the whole mapping.**
|
||||
- **`podSecurityContext.supplementalGroups` is deliberately NOT rendered, and adding it would be a
|
||||
regression rather than a hardening.** `/dev/dri/renderD128` is `root:render` `0660`, so the obvious
|
||||
move is to push the host `render` GID in — and it does not work. **Measured in the running
|
||||
container on prod-2, 2026-09-30:** the image's own
|
||||
`/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` hook `find`s every char device under
|
||||
`/dev/dri`, `stat -c '%g'` each one's GID, `groupadd`s a matching group *inside the container* when
|
||||
none exists, and `usermod -a -G` the app user into it — every start, nothing hardcoded.
|
||||
`cont-init.d` completes before `services.d`, so the account database is already correct when the
|
||||
service execs. The service then starts via `exec s6-setuidgid plex …`, and `s6-setuidgid`
|
||||
**replaces** the process group list from `/etc/group` rather than appending to it:
|
||||
`Plex Media Server` runs as `Uid: 1000 Gid: 1000 Groups: 100 1000`, with root's gid `0` — which
|
||||
PID 1 (`s6-svscan`) does hold — **gone**. A Kubernetes `supplementalGroups` therefore reaches
|
||||
`s6-svscan` and is discarded before the app ever starts. It would be inert *and* it would mislead
|
||||
whoever debugs this next. The renderer asserts its own absence in `render.py --selftest`.
|
||||
- A corollary worth having: because the hook re-`stat`s the device on every start, the host GID
|
||||
**drifting** (a node rebuild renumbering the *systemd dynamic* `render` group) is self-healing.
|
||||
No GID is pinned anywhere in this repo.
|
||||
- **`DirectoryOrCreate` is load-bearing — do not tighten it to `Directory`.** A node that comes back
|
||||
without the render device (rebuild, BIOS change, driver regression, hardware swap) must degrade the
|
||||
workload to **software transcoding**, not fail the mount. `apps-wave-0` is `wait: true` and
|
||||
`apps-wave-1` `dependsOn` it, so a failed hostPath mount stalls the **whole app tier** for the sake
|
||||
of a feature that is not load-bearing. `/dev` is tmpfs, so the empty directory the kubelet creates
|
||||
on such a node leaves nothing durable behind.
|
||||
- **The cost of that choice, stated plainly:** a missing device is **silent**. The workload
|
||||
software-transcodes and nothing reports it. There is no metric to alert on — media apps here
|
||||
render no `metricsEndpoint`, so no ServiceMonitor exists — and an alert that cannot fire is worse
|
||||
than none (see §`observability`'s 2026-09-13 correction). The only proof of hardware transcode is
|
||||
an operator observing it in the application's own dashboard.
|
||||
- **Deliberately NOT rendered** from this field: no `privileged: true` (Kyverno
|
||||
`restrict-host-namespaces` is Enforce on app namespaces and would reject it, and the render node
|
||||
does not need it); no `nodeSelector` (placement is a separate concern with its own justification —
|
||||
this field is about the device, not about where the pod lands); no resource requests/limits (there
|
||||
is no app-container resource render path, and inventing one here would be scope creep).
|
||||
- Absent ⇒ **nothing at all** is rendered; every service that does not opt in stays byte-identical.
|
||||
Ignored for `kind: vm` — a VM receives devices through KubeVirt's own `domain.devices`.
|
||||
- **Requirements:** **R-PLAT-02** (the PaaS provides the host capability; the app declares need).
|
||||
|
||||
### `ingressRoutes[]` (`host` / `port`) — HTTPRoute + Certificate + DNS
|
||||
- Each entry renders an **HTTPRoute** attaching `host` → the service `port` on the shared Gateway.
|
||||
- TLS: the Gateway terminates with the cert-manager **wildcard `*.home.keithjackson.dev`** certificate
|
||||
@@ -286,6 +338,11 @@ platform must render, and cites the requirement each mapping satisfies. Changing
|
||||
`external-dns.alpha.kubernetes.io/controller: none`, and no `public` service's route carries it.
|
||||
Derived from `exposure` alone; enforced on every gate run by
|
||||
`conformance/tests/l0-static/internal-services-publish-no-public-dns` (R-DNS-03).
|
||||
8. **`hardware` is inert until opted into, and its mechanism is renderer-local.** An SD with no
|
||||
`hardware` key renders no volume and no volumeMount; an SD *with* it gains no `securityContext`
|
||||
either. Both directions are proven by `render.py --selftest`. The `/dev/dri` hostPath and the
|
||||
`DirectoryOrCreate` type appear in `catalog/render.py` and nowhere else, so the capability's
|
||||
implementation can be replaced without touching a single ServiceDefinition.
|
||||
|
||||
## Requirement traceability (contract-level)
|
||||
Primary **R-GITOPS-02** (single reviewable object wires every requirement). Field-level: R-NET-01/02
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json",
|
||||
"title": "ServiceDefinition",
|
||||
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).",
|
||||
"description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `<name>.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 Line truncated
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "source"],
|
||||
@@ -139,6 +139,19 @@
|
||||
"supplementalGroups": { "type": "array", "items": { "type": "integer" }, "description": "Additional GIDs added to the first process in each container." }
|
||||
}
|
||||
},
|
||||
"hardware": {
|
||||
"type": "object",
|
||||
"description": "Optional host HARDWARE CAPABILITY the workload needs, for the container kind (docs/13). This field declares INTENT (`what device class this service needs`), never MECHANISM (`which host path to bind-mount`, `which extended resource to request`). That separation is the whole point of the enum: the platform currently satisfies `intel-quicksync` with a /dev/dri hostPath volume and a matching mount on the app container, but a later migration to the Intel GPU device plugin (`gpu.intel.com/i915` resource requests, no hostPath) must be a catalog/render.py change and NOTHING else - no ServiceDefinition may need editing, and no reviewer may have to re-derive per-service device wiring. Do NOT 'simplify' this into a raw device path, a hostPath, or a resource request; that would push a platform implementation detail into the operator-facing contract and make the migration a catalog-wide edit. Absent => no device wiring at all (existing apps render byte-identical). Ignored for kind=vm (a VM gets devices through KubeVirt's own domain.devices).",
|
||||
"additionalProperties": false,
|
||||
"minProperties": 1,
|
||||
"properties": {
|
||||
"gpu": {
|
||||
"type": "string",
|
||||
"description": "GPU capability class this service needs. `intel-quicksync` = the host's Intel Quick Sync Video render device (the iGPU's /dev/dri/renderD* node) for hardware video transcode/decode. Enum, not a free-form device path, so the satisfying mechanism stays swappable (see the parent description). Extend the enum when a genuinely different capability class appears (e.g. nvidia-nvenc), never to name a different path to the same one.",
|
||||
"enum": ["intel-quicksync"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"exposure": {
|
||||
"type": "string",
|
||||
"description": "internal = LAN-only (internal DNS + Gateway on LAN); public = additionally edge port-forwarded 80/443 (docs/09; R-DNS/R-NET).",
|
||||
|
||||
Reference in new issue
Block a user