Merge pull request 'feat(gpu): satisfy hardware.gpu: intel-quicksync with gpu.intel.com/i915 instead of a /dev/dri hostPath (the hostPath was measured not to work: device-cgroup EPERM)' (#37) from feat/quicksync-via-device-plugin into main
Reviewed-on: #37 Reviewed-by: Keith Jackson <[email protected]>
This commit was merged in pull request #37.
This commit is contained in:
jakman85 committed
commit
bdfa72b1c7
6 files changed
+670
-189
No files matched your search
+120
-54
@@ -166,11 +166,14 @@ OAUTH2_PROXY_METRICS_PORT = 44180
|
||||
# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------
|
||||
# The ServiceDefinition declares the CAPABILITY (schemas/servicedefinition.schema.json
|
||||
# `hardware.gpu`); everything below is the MECHANISM that currently satisfies it and is
|
||||
# deliberately confined to this file. Swapping to the Intel GPU device plugin
|
||||
# (`gpu.intel.com/i915` resource requests instead of a hostPath) is a change here and
|
||||
# nowhere else -- no catalog entry moves.
|
||||
DRI_VOLUME_NAME = "dri"
|
||||
DRI_DEVICE_DIR = "/dev/dri"
|
||||
# deliberately confined to this file. This file is the ONLY place the mechanism appears,
|
||||
# which is what let it be swapped on 2026-10-01 -- from a `/dev/dri` hostPath to the Intel
|
||||
# GPU device plugin's extended resource -- with ZERO ServiceDefinition edits.
|
||||
#
|
||||
# The extended resource is advertised by the `intel-gpu-plugin` DaemonSet
|
||||
# (platform/intel-gpu-plugin/, pinned intel/intel-gpu-plugin:0.37.0). MEASURED on all three
|
||||
# prod nodes 2026-10-01: `.status.allocatable["gpu.intel.com/i915"] == 1`.
|
||||
QUICKSYNC_RESOURCE = "gpu.intel.com/i915"
|
||||
|
||||
# --- API routes: 401 instead of 302, PER APP ------------------------------------------
|
||||
# oauth2-proxy answers an unauthenticated request with a 302 to Keycloak. Its built-in AJAX
|
||||
@@ -571,41 +574,59 @@ def r_deployment(s: dict) -> dict:
|
||||
# workload needs the host iGPU's render node for hardware video transcode. The SD says
|
||||
# only that; the wiring below is the platform's current mechanism for satisfying it.
|
||||
#
|
||||
# `type: DirectoryOrCreate` is DELIBERATE and load-bearing -- do NOT "tighten" it to
|
||||
# `Directory`. A node that comes back without the render device (rebuild, BIOS change,
|
||||
# driver regression, hardware swap) must degrade the workload to SOFTWARE transcoding,
|
||||
# not fail the mount: `apps-wave-0` is `wait: true` and `apps-wave-1` dependsOn it, so a
|
||||
# failed hostPath mount stalls the entire app tier for the sake of a feature that is not
|
||||
# load-bearing. `DirectoryOrCreate` turns that failure mode into "transcodes in
|
||||
# software" instead of "the app tier does not reconcile". /dev is tmpfs, so the empty
|
||||
# directory the kubelet creates on such a node leaves nothing durable behind.
|
||||
# THE MECHANISM IS AN EXTENDED RESOURCE, AND THE hostPath IT REPLACED WAS MEASURED NOT
|
||||
# TO WORK. Do NOT "simplify" this back into a `/dev/dri` hostPath volume. That was the
|
||||
# original implementation (merged 2026-09-30) and it was proven broken in the live Plex
|
||||
# pod on k3s-prod-2 on 2026-10-01: the device node was VISIBLE in the container
|
||||
# (`crw-rw---- root video2 226,128 renderD128`) and could not be opened --
|
||||
# `uid 0 O_RDWR -> EPERM` and `uid 1000 O_RDWR -> EPERM`. EPERM from the DEVICE CGROUP,
|
||||
# not EACCES from file permissions, proven three ways: root on the HOST opens the same
|
||||
# device rc=0; the node is unified `cgroup2fs`; and the container's runc `config.json`
|
||||
# read `linux.resources.devices: [{allow: False, access: 'rwm'}]` with
|
||||
# `linux.devices: None` -- one deny-all rule, no allow entry for major 226. A hostPath is
|
||||
# a filesystem operation: it binds the device node in but adds NO cgroup allow rule,
|
||||
# because Kubernetes cannot infer one from a volume.
|
||||
#
|
||||
# The mount is deliberately NOT readOnly: VA-API opens the render node read-write.
|
||||
# The device plugin is what produces the allow rule. Confirmed in the v0.37.0 source:
|
||||
# `createDeviceSpecsFromDrmFiles` emits `DeviceSpec{..., Permissions: "rw"}`, which the
|
||||
# kubelet turns into both the device injection and the matching cgroup allow.
|
||||
#
|
||||
# `limits`, NOT `requests`, AND THAT IS NOT A STYLE CHOICE: an extended resource must be
|
||||
# expressed in `limits`. Kubernetes mirrors `requests` from `limits` automatically, and a
|
||||
# requests-only extended resource is invalid. App container only, never a sidecar -- the
|
||||
# oauth2-proxy sidecar does no transcoding.
|
||||
#
|
||||
# THE FAILURE MODE THIS TRADED INTO, STATED PLAINLY -- it is worse in blast radius, and
|
||||
# unavoidable. BEFORE: a missing device meant `DirectoryOrCreate` mounted an empty dir,
|
||||
# Plex ran and SILENTLY software-transcoded (invisible, but harmless to everything else).
|
||||
# NOW: if the resource is unavailable (plugin broken, removed, or not yet reconciled) the
|
||||
# pod is UNSCHEDULABLE, and because `apps-wave-0` is `wait: true` with `apps-wave-1`
|
||||
# dependsOn it, that STALLS THE APP TIER. There is no graceful-degradation variant: an
|
||||
# extended resource is a hard scheduling constraint, and it is also the only thing that
|
||||
# produces the cgroup allow rule. Plex's schedulability is now coupled to the
|
||||
# intel-gpu-plugin DaemonSet; that is the price of hardware transcoding working at all.
|
||||
# ESCAPE HATCH (one field, no code change): delete the `hardware:` block from
|
||||
# `catalog/plex.yaml` and re-render -- Plex returns to software transcoding and the tier
|
||||
# unblocks. runbooks/plex-quicksync.md carries it.
|
||||
#
|
||||
# `podSecurityContext.supplementalGroups` is DELIBERATELY NOT SET here, and adding it
|
||||
# would be a regression, not a hardening. MEASURED in the running container on prod-2,
|
||||
# 2026-09-30: the image's own cont-init hook
|
||||
# would be a regression, not a hardening. MEASURED in the running container on prod-2
|
||||
# (2026-09-30, re-measured 2026-10-01): the image's own cont-init hook
|
||||
# `/etc/cont-init.d/45-plex-hw-transcode-and-connected-tuner` `find`s every char device
|
||||
# under /dev/dri, `stat -c '%g'` each one's GID, `groupadd`s a matching group INSIDE the
|
||||
# container when none exists, and `usermod -a -G` the app user into it -- every start,
|
||||
# nothing hardcoded. cont-init.d completes before services.d, so the account database
|
||||
# s6 reads at exec time already carries the group. The service is then started with
|
||||
# `exec s6-setuidgid plex ...`, and `s6-setuidgid` REPLACES the process group list from
|
||||
# /etc/group rather than appending to it: `Plex Media Server` runs with
|
||||
# `Uid: 1000 Gid: 1000 Groups: 100 1000` -- root's gid 0, which PID 1 (s6-svscan) does
|
||||
# have, is GONE. Anything Kubernetes puts in supplementalGroups reaches s6-svscan and is
|
||||
# then discarded before the app ever starts. MOUNTING THE DEVICE IS SUFFICIENT; the
|
||||
# group handling is the image's job and it does it correctly. A supplementalGroups entry
|
||||
# here would be inert AND would mislead whoever debugs this next.
|
||||
# nothing hardcoded. It does exactly the same thing to the PLUGIN-INJECTED device, which
|
||||
# is why this swap needs no group work either. cont-init.d completes before services.d,
|
||||
# so the account database s6 reads at exec time already carries the group: measured live,
|
||||
# `Plex Media Server` ran with `Groups: 44 100 992 1000`, where 992 is the host `render`
|
||||
# GID. The service is started with `exec s6-setuidgid plex ...`, and `s6-setuidgid`
|
||||
# REPLACES the process group list from /etc/group rather than appending to it, so
|
||||
# anything Kubernetes puts in supplementalGroups reaches s6-svscan and is discarded
|
||||
# before the app ever starts. A supplementalGroups entry here would be inert AND would
|
||||
# mislead whoever debugs this next.
|
||||
hw_gpu = (s.get("hardware") or {}).get("gpu")
|
||||
if hw_gpu == "intel-quicksync":
|
||||
volume_mounts.append({"name": DRI_VOLUME_NAME, "mountPath": DRI_DEVICE_DIR})
|
||||
volumes.append(
|
||||
{
|
||||
"name": DRI_VOLUME_NAME,
|
||||
"hostPath": {"path": DRI_DEVICE_DIR, "type": "DirectoryOrCreate"},
|
||||
}
|
||||
)
|
||||
container["resources"] = {"limits": {QUICKSYNC_RESOURCE: 1}}
|
||||
|
||||
if volume_mounts:
|
||||
container["volumeMounts"] = volume_mounts
|
||||
@@ -2650,10 +2671,13 @@ def _selftest() -> int:
|
||||
"an SD without podSecurityContext must not gain a pod securityContext",
|
||||
)
|
||||
|
||||
# 6b) hardware.gpu: intel-quicksync -> a /dev/dri hostPath volume and a read-write mount
|
||||
# on the app container, and NOTHING ELSE. The SD declares the CAPABILITY; these
|
||||
# assertions pin the MECHANISM that currently satisfies it -- including what it
|
||||
# deliberately does NOT emit -- so a change to it is a deliberate, visible edit.
|
||||
# 6b) hardware.gpu: intel-quicksync -> a `gpu.intel.com/i915` LIMIT on the app container
|
||||
# and NOTHING ELSE: no volume, no volumeMount, no securityContext, no nodeSelector.
|
||||
# The SD declares the CAPABILITY; these assertions pin the MECHANISM that currently
|
||||
# satisfies it -- including what it deliberately does NOT emit -- so a change to it
|
||||
# is a deliberate, visible edit. The previous mechanism (a /dev/dri hostPath) is
|
||||
# asserted ABSENT: it was measured not to work (EPERM from the device cgroup; see
|
||||
# the emission site), and the absence assertion is what stops it coming back.
|
||||
hw_docs = render(
|
||||
{
|
||||
"name": "hw",
|
||||
@@ -2664,24 +2688,36 @@ def _selftest() -> int:
|
||||
hw_pod = next(
|
||||
d[1] for d in hw_docs if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
hw_app = hw_pod["containers"][0]
|
||||
# The whole mechanism, exactly: an extended resource in `limits`. `requests` is NOT set
|
||||
# by the renderer -- Kubernetes mirrors it from `limits`, and a requests-only extended
|
||||
# resource is invalid. Asserting the exact dict is deliberate: it fails if anyone adds a
|
||||
# CPU/memory resources feature through this field (that would be scope creep, and this
|
||||
# field is about the DEVICE only).
|
||||
check(
|
||||
{
|
||||
"name": "dri",
|
||||
"hostPath": {"path": "/dev/dri", "type": "DirectoryOrCreate"},
|
||||
}
|
||||
in hw_pod.get("volumes", []),
|
||||
"hardware.gpu=intel-quicksync must render a /dev/dri hostPath volume "
|
||||
"with type DirectoryOrCreate (a GPU-less node must degrade, not fail the mount)",
|
||||
hw_app.get("resources") == {"limits": {"gpu.intel.com/i915": 1}},
|
||||
"hardware.gpu=intel-quicksync must render resources.limits "
|
||||
"{gpu.intel.com/i915: 1} on the app container, and nothing else in resources",
|
||||
)
|
||||
hw_mounts = hw_pod["containers"][0].get("volumeMounts", [])
|
||||
# The retired mechanism must stay retired. A hostPath binds the device node in but adds
|
||||
# no device-cgroup allow rule, so the open() was EPERM with the node plainly visible.
|
||||
check(
|
||||
{"name": "dri", "mountPath": "/dev/dri"} in hw_mounts,
|
||||
"hardware.gpu=intel-quicksync must mount /dev/dri READ-WRITE on the app container",
|
||||
not any(
|
||||
"hostPath" in v or v.get("name") == "dri" for v in hw_pod.get("volumes", [])
|
||||
)
|
||||
and not any(
|
||||
m.get("mountPath") == "/dev/dri" or m.get("name") == "dri"
|
||||
for m in hw_app.get("volumeMounts", [])
|
||||
),
|
||||
"hardware.gpu=intel-quicksync must render NO /dev/dri hostPath volume and no "
|
||||
"/dev/dri mount (the hostPath mechanism was measured not to work: device-cgroup "
|
||||
"EPERM; the device plugin's DeviceSpec Permissions: rw is what grants access)",
|
||||
)
|
||||
# The device mount is the WHOLE feature: no pod securityContext is invented. The image's
|
||||
# own cont-init hook adds the app user to the device's group inside the container, and
|
||||
# s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway (see the
|
||||
# emission site). Asserting the absence is what stops someone adding an inert one back.
|
||||
# The resource is the WHOLE feature: no pod securityContext is invented. The image's own
|
||||
# cont-init hook adds the app user to the (plugin-injected) device's group inside the
|
||||
# container, and s6-setuidgid discards any Kubernetes-supplied supplementalGroups anyway
|
||||
# (see the emission site). Asserting the absence is what stops someone adding an inert
|
||||
# one back.
|
||||
check(
|
||||
"securityContext" not in hw_pod,
|
||||
"hardware.gpu must NOT invent a pod securityContext (supplementalGroups is inert "
|
||||
@@ -2689,10 +2725,12 @@ def _selftest() -> int:
|
||||
)
|
||||
# Nothing privileged, and no node pinning: Kyverno restrict-host-namespaces (Enforce)
|
||||
# rejects `privileged: true` in an app namespace, and placement is a separate concern
|
||||
# with its own justification -- this field is about the DEVICE only.
|
||||
# with its own justification -- this field is about the DEVICE only. (The extended
|
||||
# resource now constrains placement to nodes the plugin advertises, which is the
|
||||
# scheduler's own doing, not a nodeSelector this renderer writes.)
|
||||
check(
|
||||
"privileged" not in hw_pod.get("securityContext", {})
|
||||
and "securityContext" not in hw_pod["containers"][0]
|
||||
and "securityContext" not in hw_app
|
||||
and "nodeSelector" not in hw_pod,
|
||||
"hardware.gpu must emit no privileged securityContext and no nodeSelector",
|
||||
)
|
||||
@@ -2705,8 +2743,10 @@ def _selftest() -> int:
|
||||
check(
|
||||
"volumes" not in nohw_pod
|
||||
and "volumeMounts" not in nohw_pod["containers"][0]
|
||||
and "resources" not in nohw_pod["containers"][0]
|
||||
and "securityContext" not in nohw_pod,
|
||||
"an SD without `hardware` must gain no dri volume, no mount and no securityContext",
|
||||
"an SD without `hardware` must gain no resources block, no volume, no mount and no "
|
||||
"securityContext",
|
||||
)
|
||||
# An SD that declares BOTH keeps its own podSecurityContext exactly as written -- the
|
||||
# hardware field neither adds to it nor rewrites it.
|
||||
@@ -2723,9 +2763,35 @@ def _selftest() -> int:
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]
|
||||
check(
|
||||
hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]},
|
||||
hwm_pod["securityContext"] == {"fsGroup": 1001, "supplementalGroups": [44]}
|
||||
and hwm_pod["containers"][0].get("resources")
|
||||
== {"limits": {"gpu.intel.com/i915": 1}},
|
||||
"hardware.gpu must leave the SD's own podSecurityContext untouched",
|
||||
)
|
||||
# The resource goes on the APP container only. A forward-auth app renders an oauth2-proxy
|
||||
# sidecar, which does no transcoding and must not consume one of the node's i915 slots.
|
||||
hws_containers = next(
|
||||
d[1]
|
||||
for d in render(
|
||||
{
|
||||
"name": "hws",
|
||||
"source": {"image": "a:1"},
|
||||
"hardware": {"gpu": "intel-quicksync"},
|
||||
"ingressRoutes": [{"host": "hws.home.keithjackson.dev", "port": 8080}],
|
||||
"auth": {"mode": "forward-auth", "groups": ["media-users"]},
|
||||
}
|
||||
)
|
||||
if d[1]["kind"] == "Deployment"
|
||||
)["spec"]["template"]["spec"]["containers"]
|
||||
check(
|
||||
len(hws_containers) == 2
|
||||
and hws_containers[0].get("resources")
|
||||
== {"limits": {"gpu.intel.com/i915": 1}}
|
||||
and "gpu.intel.com/i915"
|
||||
not in hws_containers[1].get("resources", {}).get("limits", {}),
|
||||
"hardware.gpu must put the i915 limit on the APP container only, never on the "
|
||||
"oauth2-proxy sidecar",
|
||||
)
|
||||
|
||||
# 6c) forward-auth (Option B): an exposed forward-auth app gets an oauth2-proxy sidecar that
|
||||
# fronts the Gateway path (fail-closed), while oidc/native/none apps are unchanged.
|
||||
|
||||
Reference in new issue
Block a user