diff --git a/apps/wave-0/kustomization.yaml b/apps/wave-0/kustomization.yaml index 394f76d..3104706 100644 --- a/apps/wave-0/kustomization.yaml +++ b/apps/wave-0/kustomization.yaml @@ -18,6 +18,7 @@ resources: - pihole-dns.yaml - pihole-oauth2-proxy-credentials.sops.yaml - pihole.yaml +- plex-cert.yaml - plex-claim-secret.sops.yaml - plex-remote.yaml - plex.yaml diff --git a/apps/wave-0/pihole.yaml b/apps/wave-0/pihole.yaml index 025be47..dcd9964 100644 --- a/apps/wave-0/pihole.yaml +++ b/apps/wave-0/pihole.yaml @@ -73,7 +73,7 @@ spec: - name: FTLCONF_dns_rateLimit_count value: '0' - name: FTLCONF_misc_dnsmasq_lines - value: address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP} + value: address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP} - name: FTLCONF_webserver_api_password valueFrom: secretKeyRef: diff --git a/apps/wave-0/plex-cert.yaml b/apps/wave-0/plex-cert.yaml new file mode 100644 index 0000000..840c391 --- /dev/null +++ b/apps/wave-0/plex-cert.yaml @@ -0,0 +1,66 @@ +--- +# ================================================================================================= +# PLEX TLS — a Let's Encrypt certificate for the ONE hostname Plex advertises to plex.tv. +# ================================================================================================= +# HAND-WRITTEN, NOT RENDERED. A non-catalog app-tier unit anchored to the `plex` ServiceDefinition +# and placed into wave 0 by `catalog/render.py --emit-waves` via its EXTRA_UNITS table — the same +# mechanism gitea-actions, pihole-dns and plex-remote use. It is hand-written for the same reason: +# `schemas/servicedefinition.schema.json` is `additionalProperties: false` and has NO field for a +# cert-manager Certificate. (`ingressRoutes` gets TLS for free from the Gateway's wildcard; this is +# the opposite case — a certificate for a workload that terminates its OWN TLS on a raw L4 port.) +# +# WHY PLEX NEEDS ITS OWN CERTIFICATE AT ALL. Plex serves tcp/32400 directly (not through the L7 +# Gateway — the operator chose the native path 2026-10-03, apps/wave-0/plex-remote.yaml) and it +# presents its own `*..plex.direct` certificate there. That name cannot cover +# `plex-vip.${BASE_DOMAIN}`, the hostname we advertise, so without this the advertised `https://` +# URL fails hostname verification for every client. That is not hypothetical: it is the shape of +# the 2026-10-04 outage, where Plex's log carried a continuous storm of +# `CERT: incomplete TLS handshake ... sslv3 alert certificate unknown` from real LAN clients while +# plex.tv could publish nothing and the server vanished from the account. +# +# A SAN match is not what makes Plex FUNCTION — `httpsRequired=0` and `secureConnections` default +# to Preferred, so clients can fall back to cleartext. It is what keeps the REMOTE leg encrypted +# rather than silently downgrading traffic that crosses the internet. +# +# HOW IT REACHES PLEX. catalog/plex.yaml takes `tls.crt`/`tls.key` from this Secret as env +# (`optional: true` — see below), and its `lifecycle.postStart` hook converts the PEM into the +# PKCS#12 bundle Plex's `customCertificatePath` requires and sets the three `customCertificate*` +# prefs. The conversion is redone on EVERY pod start, so a cert-manager RENEWAL is picked up with +# no extra moving part and nothing to remember. +# +# ORDERING, AND WHY THE ENV REFS ARE OPTIONAL. This Certificate must live in the `plex` namespace +# for the Deployment to read its Secret, and that namespace is created by the rendered +# apps/wave-0/plex.yaml ITSELF — so this cannot be issued in an earlier tier (platform-config runs +# before the namespace exists) and lands in the SAME reconcile as the workload. On a cold bootstrap +# the Secret is therefore absent for as long as a DNS01 challenge takes. With hard env refs Plex +# would sit in CreateContainerConfigError until then, and because `apps-wave-0` is `wait: true` +# with `apps-wave-1` dependsOn it, a failed or slow issuance would STALL THE WHOLE APP TIER. Hence +# `optional: true` on both refs (schema 1.11.0, added for exactly this): no certificate means Plex +# still starts and serves on its own plex.direct cert, the hook logs that TLS was not installed, +# and the next start after issuance converges. +# +# ISSUER: ClusterIssuer `letsencrypt-prod` (platform/cert-manager/clusterissuers.yaml), the same +# issuer behind the Gateway's wildcard (platform/gateway/wildcard-certificate.yaml, Ready 28d on +# prod as measured 2026-10-04). A SINGLE name is requested rather than reusing the wildcard, +# because the wildcard's Secret lives in the `gateway` namespace and a Secret cannot be read across +# namespaces; requesting `*.${BASE_DOMAIN}` again here would also burn a duplicate-certificate slot +# against Let's Encrypt's weekly limit for no benefit. A single name is the least-privilege shape: +# this key signs for `plex-vip.${BASE_DOMAIN}` and nothing else. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: plex-vip + namespace: plex + labels: + app.kubernetes.io/name: plex +spec: + # The name Plex ADVERTISES and the name on this certificate must stay identical. They are two + # halves of one decision: catalog/plex.yaml's ADVERTISE_IP, the postStart hook's + # customCertificateDomain, this dnsNames entry, and catalog/pihole.yaml's split-horizon answer. + # Nothing enforces that they agree. + dnsNames: + - plex-vip.${BASE_DOMAIN} + secretName: plex-vip-tls + issuerRef: + kind: ClusterIssuer + name: letsencrypt-prod diff --git a/apps/wave-0/plex.yaml b/apps/wave-0/plex.yaml index e4c3d3e..7d80c83 100644 --- a/apps/wave-0/plex.yaml +++ b/apps/wave-0/plex.yaml @@ -138,7 +138,19 @@ spec: name: plex-claim key: PLEX_CLAIM - name: ADVERTISE_IP - value: ${PLEX_ADVERTISE_URL} + value: https://plex-vip.${BASE_DOMAIN}:32400 + - name: PLEX_TLS_CRT + valueFrom: + secretKeyRef: + name: plex-vip-tls + key: tls.crt + optional: true + - name: PLEX_TLS_KEY + valueFrom: + secretKeyRef: + name: plex-vip-tls + key: tls.key + optional: true resources: limits: gpu.intel.com/i915: 1 @@ -148,6 +160,14 @@ spec: - name: media mountPath: /media readOnly: true + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - "# NOTE \u2014 LOAD-BEARING: SHELL VARIABLES HERE ARE WRITTEN BARE, NEVER BRACE-WRAPPED.\n# This whole script is rewritten by Flux postBuild envsubst BEFORE it ever runs, so a\n# brace-wrapped name is read as a cluster-vars knob: an undeclared one is a HARD\n# Kustomization failure under StrictPostBuildSubstitutions, and a declared one would be\n# replaced with its value. Shell vars are therefore `$PX`, `$TOKEN`, `$P12` \u2014 bare. Only\n# BASE_DOMAIN below is brace-wrapped, because it IS a knob and IS meant to be\n# substituted. Command substitution `$(cmd)` is left alone and needs no escaping. A shell\n# variable written with braces would be blanked or would fail the whole Kustomization \u2014\n# this repo has already lost a gitea OIDC bootstrap to exactly that mistake, and\n# catalog/render.py now refuses to emit it (which is how this very comment got caught).\n# DELIBERATELY NO `set -u`. The TLS env vars are declared `optional: true`, so when the\n# certificate has not issued yet they are genuinely UNSET, not empty \u2014 and under `set -u`\n# the first `[ -n \"$PLEX_TLS_CRT\" ]` would abort the script, which for a postStart hook\n# means kubelet KILLS THE CONTAINER. That is the precise failure the `optional` flag exists\n# to prevent, so turning it back on here would defeat the whole design. (The alternative,\n# a brace-wrapped `:-` default, would need envsubst escaping \u2014 the trap the note above\n# describes.) Every value below is explicitly guarded instead.\nPX=\"/config/Library/Application Support/Plex Media Server/Preferences.xml\"\nCERTDIR=/config/certs\nP12=$CERTDIR/plex-vip.p12\n# The PKCS#12 passphrase protects nothing and is deliberately not a secret: Plex needs it\n# in cleartext in customCertificateKey, and it and the bundle sit on the same PVC. It is a\n# container-format requirement, not an access control.\nP12PASS=plexvip\nLANNETS='10.0.0.0/8,192.168.0.0/16'\nCERTDOMAIN=plex-vip.${BASE_DOMAIN}\nAPI=http://127.0.0.1:32400\n\n# (1) Build the PKCS#12 bundle from the PEM the Secret supplies. Rebuilt on every start so\n# a cert-manager RENEWAL is picked up with no extra moving part.\nif [ -n \"$PLEX_TLS_CRT\" ] && [ -n \"$PLEX_TLS_KEY\" ]; then\n mkdir -p $CERTDIR && chmod 700 $CERTDIR\n printf '%s' \"$PLEX_TLS_CRT\" > $CERTDIR/tls.crt\n printf '%s' \"$PLEX_TLS_KEY\" > $CERTDIR/tls.key\n chmod 600 $CERTDIR/tls.key\n if openssl pkcs12 -export -out $P12.tmp -inkey $CERTDIR/tls.key \\\n -in $CERTDIR/tls.crt -passout pass:$P12PASS 2>/dev/null; then\n mv -f $P12.tmp $P12\n chown 1000:1000 $P12 2>/dev/null || true\n chmod 600 $P12\n echo \"[plex-settings] PKCS#12 bundle written\"\n else\n rm -f $P12.tmp\n echo \"[plex-settings] WARN: pkcs12 export failed; keeping any previous bundle\" >&2\n fi\n rm -f $CERTDIR/tls.key\nelse\n echo \"[plex-settings] WARN: TLS env empty; certificate not installed this start\" >&2\nfi\n\n# (2) Wait, bounded (~90s), for the local API. /identity needs no auth.\ni=0\nwhile [ \"$i\" -lt 45 ]; do\n if [ \"$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 $API/identity 2>/dev/null)\" = \"200\" ]; then\n break\n fi\n i=$(expr $i + 1); sleep 2\ndone\n\n# (3) The prefs API needs the server token. It already lives in Preferences.xml on this\n# PVC; it is never echoed here, and Plex redacts tokens in its own request log. An\n# UNCLAIMED server has none \u2014 a normal state on a fresh /config \u2014 so skip, do not fail.\nTOKEN=$(grep -o 'PlexOnlineToken=\"[^\"]*\"' \"$PX\" 2>/dev/null | head -1 | cut -d'\"' -f2)\nif [ -z \"$TOKEN\" ]; then\n echo \"[plex-settings] not claimed yet \u2014 skipping prefs, will retry next start\" >&2\n exit 0\nfi\n\n# Percent-encode what our values actually contain. Plex takes prefs as QUERY parameters on\n# a PUT, so a raw '/' or ',' would be read as structure rather than data.\nenc() {\n printf '%s' \"$1\" | sed -e 's|%|%25|g' -e 's|/|%2F|g' -e 's|,|%2C|g' \\\n -e 's|:|%3A|g' -e 's| |%20|g'\n}\nsetpref() {\n code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X PUT\ + \ \\\n -H \"X-Plex-Token: $TOKEN\" \"$API/:/prefs?$1=$(enc \"$2\")\" 2>/dev/null)\n if [ \"$code\" = \"200\" ]; then\n echo \"[plex-settings] set $1\"\n else\n echo \"[plex-settings] WARN: $1 -> HTTP $code\" >&2\n fi\n}\n\n# (4) THE LAN DEFINITION \u2014 the reason this hook was written. Without it Plex has NO notion\n# of a local network and tags EVERY client remote. Measured 2026-10-04: 468 requests\n# logged \"Allowed Network (WAN)\" and ZERO \"(LAN)\", including Seerr from inside the\n# cluster, which is why the server showed as Remote to LAN clients. Note allowedNetworks\n# (the ALLOWED_NETWORKS env) only waives AUTH \u2014 it does NOT make a client local. Two\n# different settings; only one of them has an env hook.\nsetpref LanNetworksBandwidth \"$LANNETS\"\n\n# (5) THE CERTIFICATE, which is what makes the `https` in ADVERTISE_IP servable. Set only\n# if the bundle exists \u2014 pointing Plex at a missing file would break its TLS listener.\nif [ -s \"$P12\" ]; then\n setpref customCertificatePath \"$P12\"\n setpref customCertificateKey \"$P12PASS\"\n setpref customCertificateDomain \"$CERTDOMAIN\"\nelse\n echo \"[plex-settings] WARN: no bundle; leaving certificate prefs untouched\" >&2\nfi\nexit 0\n" volumes: - name: config persistentVolumeClaim: diff --git a/catalog/pihole.yaml b/catalog/pihole.yaml index 8ebfdc8..ef3058e 100644 --- a/catalog/pihole.yaml +++ b/catalog/pihole.yaml @@ -134,8 +134,17 @@ env: # then silently wrong: FTL stores `["a","b"]` as ONE array element containing those literal # characters, writes that single bogus line into /etc/pihole/dnsmasq.conf and reports "0 invalid" # while resolution for the zone does not work. Confirmed both ways against the live image. + # plex-vip -> ${PLEX_L4_VIP} (added 2026-10-04) is the LAN HALF of Plex's single advertised + # hostname. Plex publishes ONE URL to plex.tv for both local and remote clients + # (catalog/plex.yaml ADVERTISE_IP): on the LAN this answer points it at the tcp/32400 + # LoadBalancer, while Cloudflare answers the same name publicly with the WAN address, NATed to + # the same VIP. WITHOUT THIS LINE the entry above answers it with ${GATEWAY_VIP} via the zone + # wildcard, where NOTHING listens on 32400 — measured 2026-10-04, plex-vip.${BASE_DOMAIN} + # resolved to the Gateway VIP and a connect to :32400 there returned http=000. That is a silent + # failure: the Plex web UI on plex.${BASE_DOMAIN} keeps working through the Gateway on 443 while + # every client that takes the ADVERTISED connection cannot reach the server at all. - name: FTLCONF_misc_dnsmasq_lines - value: "address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP}" + value: "address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}" # THE ADMIN PASSWORD, previously the ONE operator-set value declared nowhere. Sourced from # apps/wave-0/pihole-bootstrap-secret.sops.yaml (SOPS/age, same wave => same reconcile), which # carries the full rationale and the measurement that found it. diff --git a/catalog/plex.yaml b/catalog/plex.yaml index 7f75d14..752c04c 100644 --- a/catalog/plex.yaml +++ b/catalog/plex.yaml @@ -72,54 +72,211 @@ env: secretKeyRef: name: plex-claim key: PLEX_CLAIM - # ADVERTISE_IP ("Custom server access URLs") — NOW SET. This replaces a DEFERRAL note that stood - # here until 2026-10-04, and the deferral's premise has changed: it weighed only ONE candidate, - # `https://plex.home.keithjackson.dev` through the L7 Gateway. The operator chose the NATIVE L4 - # path instead on 2026-10-03 (apps/wave-0/plex-remote.yaml) so streams go straight to - # ${PLEX_L4_VIP}:32400 and bypass the Envoy hop. What was still missing was a URL whose TLS - # CERTIFICATE validates — and getting that wrong is what took the service down. + # ADVERTISE_IP ("Custom server access URLs") — the ONE hostname this server publishes to plex.tv, + # serving BOTH local and remote clients by split-horizon DNS (operator design, 2026-10-04): + # LAN -> PiHole answers plex-vip.${BASE_DOMAIN} with ${PLEX_L4_VIP} (the tcp/32400 LoadBalancer + # in apps/wave-0/plex-remote.yaml). See catalog/pihole.yaml's split-horizon line, which + # needs a per-host override because the zone wildcard otherwise points at the GATEWAY + # VIP — measured 2026-10-04: plex-vip.${BASE_DOMAIN} resolved to the Gateway, where + # nothing listens on 32400. + # PUBLIC -> Cloudflare answers with the WAN address, NATed to the same VIP (off-repo). # - # THE VALUE IS A `plex.direct` NAME, NOT THE RAW VIP, AND THAT IS THE WHOLE POINT. Plex serves - # 32400 with its own certificate; measured live 2026-10-04 it carries exactly one name: - # CN = *..plex.direct (one SAN, no IP SANs at all) - # so `https://:32400` can NEVER validate. `..plex.direct` is Plex's own - # mechanism for this: it is PUBLIC DNS that resolves to the embedded address, so the name points at - # the LAN VIP while the wildcard cert above matches it. Measured 2026-10-04 with FULL verification - # (no `curl -k`): - # https://192-168-2-24..plex.direct:32400/identity -> 200, TLS verify OK - # https://192.168.2.24:32400/identity -> 000, cert validation FAILED + # WHY A HOSTNAME AND NOT THE `plex.direct` FORM THIS BRIEFLY USED. The first fix for the + # 2026-10-04 outage advertised `https://..plex.direct:32400`, which works + # (and is Plex's own mechanism), but it hard-codes BOTH the VIP and the server's certificate hash + # into a cluster-var — two values nothing could keep in sync, with no alert if either drifted. A + # hostname in our own zone removes that whole class of problem: ${BASE_DOMAIN} is already a knob, + # the VIP lives only in PiHole's answer, and `PLEX_ADVERTISE_URL` is deleted outright. # - # THIS FIELD EXISTS BECAUSE AUTO-DISCOVERY CANNOT WORK HERE. Left empty, Plex publishes the - # addresses it can see itself — which from a pod is the overlay IP (`10.0.2.136`), useless to a LAN - # client. The VIP has to be advertised explicitly; there is no "just clear it" fix. + # THIS IS `https`, WHICH ONLY WORKS BECAUSE WE HAND PLEX OUR OWN CERTIFICATE. Plex serves 32400 + # with its `*..plex.direct` cert, which does NOT cover this name — advertising + # `https://plex-vip...` against the stock cert reproduces the exact failure that caused the + # outage (measured: a storm of `CERT: incomplete TLS handshake ... certificate unknown` from real + # clients, and `curl` without `-k` failing to connect). The `lifecycle.postStart` hook below + # installs the Let's Encrypt cert for this name and points Plex at it, so the advertised URL and + # the served certificate agree. Do not change this to a bare IP, and do not drop the hook and + # leave the `https` here. # - # WHAT WENT WRONG, recorded so it is not re-introduced by hand. customConnections OVERRIDES Plex's - # own publishing, and on 2026-10-04 it held two hand-typed URLs, both uncompletable: - # - `https://192.168.2.24:32400` -> cert cannot cover an IP. Measured: a storm of - # `CERT: incomplete TLS handshake ... sslv3 alert certificate unknown` in Plex's log every few - # seconds, from REAL LAN clients arriving through the VIP. - # - `https://plex.home.keithjackson.dev:32400` -> nothing listens. The Gateway serves this host on - # 443; measured `http=000` against 192.168.2.20:32400. - # With every advertised connection dead, plex.tv had nothing publishable, sat at - # `mapping state 'Mapped - Not Published'`, fell back to Plex Relay, and THE SERVER DISAPPEARED - # FROM THE ACCOUNT — clients reported "not authorized". + # (A SAN match is not what makes Plex *work* — `httpsRequired=0` and `secureConnections` default + # to Preferred, so clients can fall back to cleartext. It is what keeps the remote leg ENCRYPTED + # instead of silently downgrading traffic that crosses the internet.) # - # DECLARING IT HERE rather than in the web UI is the durable half of the fix. The image's - # /etc/cont-init.d/40-plex-first-run runs `setPref "customConnections" "${ADVERTISE_IP}"` (line 127) - # OUTSIDE the PLEX_CLAIM/PlexOnlineToken guard, and its `/.firstRunComplete` marker sits on the - # container's OVERLAY fs, not the /config PVC — so this is re-applied on EVERY pod start and git, - # not a form field, owns the value. Editing this env rolls the Deployment, which is what applies it. - # - # TWO LIMITS, stated rather than discovered later: - # - THE HASH IS SERVER IDENTITY, which is why the whole URL is a cluster-var and not built from - # ${PLEX_L4_VIP}: envsubst cannot dash an IP, and the hash is not derivable from anything in - # this tree. It is not a secret (it ships in the cert and in the published connection list), but - # if Plex reissues under a new hash this URL goes stale and must be re-read from the live cert. - # - THIS RESTORES LAN CLIENTS AND THE ACCOUNT LISTING, NOT WAN. Remote clients still need the - # off-repo tcp/32400 port-forward (runbooks/plex-remote-access.md). Once that exists, the WAN - # twin of this URL can be appended comma-separated — customConnections takes a list. + # The image's /etc/cont-init.d/40-plex-first-run runs `setPref "customConnections" + # "${ADVERTISE_IP}"` (line 127, verified in the running image), OUTSIDE the + # PLEX_CLAIM/PlexOnlineToken guard, and its `/.firstRunComplete` marker is on the container + # OVERLAY fs rather than the /config PVC — so this is re-applied on EVERY pod start and git, not + # a web form, owns the value. Hand-typed custom URLs are what broke this service; they are now + # overwritten on every start. - name: ADVERTISE_IP - value: ${PLEX_ADVERTISE_URL} + value: https://plex-vip.${BASE_DOMAIN}:32400 + # THE TLS MATERIAL FOR THE NAME ABOVE, as PEM text in env rather than a mounted volume. + # `plex-vip-tls` is issued by cert-manager from ClusterIssuer `letsencrypt-prod` for exactly + # `plex-vip.${BASE_DOMAIN}` (apps/wave-0/plex-cert.yaml, same wave => same reconcile as this + # workload). The postStart hook converts it to the PKCS#12 bundle Plex requires. + # + # WHY ENV AND NOT A VOLUME, which would be the conventional choice. MEASURED 2026-10-04: + # `schemas/servicedefinition.schema.json` `storage` accepts only + # [class, export, mountPath, name, readOnly, size, subPath] with `additionalProperties: false`, + # `catalog/render.py` emits NO secret volumes, and no rendered app mounts a Secret as a volume. + # Mounting one therefore needs a schema + renderer change; `env.valueFrom.secretKeyRef` is + # already in the schema and already used by PLEX_CLAIM above. PEM is text, so it survives an env + # var intact (a raw PKCS#12 would not — it is binary and carries NUL bytes). + # + # COST, STATED PLAINLY: the private key is readable from this container's environment + # (/proc/1/environ, `kubectl exec env`) by anything that can already exec into the pod. That is + # the same audience that can read the converted bundle under /config, so the marginal exposure is + # small — but a secret VOLUME would still be the better shape, and adding secret-volume support + # to the schema is the clean follow-up that removes this note. + # + # `optional: true` IS LOAD-BEARING, NOT DEFENSIVE CLUTTER (schema 1.11.0, added for this). The + # Certificate is issued INTO THIS NAMESPACE, and the namespace is created by the rendered + # apps/wave-0/plex.yaml itself — so the cert CANNOT be issued in an earlier tier, and the + # Certificate and this Deployment land in the SAME reconcile. On a cold bootstrap the Secret is + # therefore briefly absent while cert-manager completes a DNS01 challenge. With a HARD ref, Plex + # would sit in CreateContainerConfigError until a certificate issued — and because `apps-wave-0` + # is `wait: true` with `apps-wave-1` dependsOn it, a failed or slow issuance would STALL THE WHOLE + # APP TIER (the same hazard the `hardware` block below documents for the GPU resource). Optional + # instead means: no cert -> Plex still starts and serves, the postStart hook logs that the + # certificate was not installed, and the next start after issuance converges. Plex falls back to + # its own `*.plex.direct` certificate in the meantime, so the degraded state is "advertised URL + # not yet servable over TLS", not "media server down". + - name: PLEX_TLS_CRT + valueFrom: + secretKeyRef: + name: plex-vip-tls + key: tls.crt + optional: true + - name: PLEX_TLS_KEY + valueFrom: + secretKeyRef: + name: plex-vip-tls + key: tls.key + optional: true +lifecycle: + # LEVEL-TRIGGERED PLEX SETTINGS, re-asserted on EVERY container start — the same pattern and the + # same reasoning as catalog/pihole.yaml's and catalog/qbittorrent.yaml's postStart: desired state + # lives in git and is re-asserted unconditionally rather than typed into a web form once. Typing + # settings into the web form is precisely what took this service down on 2026-10-04. + # + # IT EXISTS BECAUSE THE IMAGE HAS NO ENV FOR THESE SETTINGS. Measured 2026-10-04 across the whole + # init surface (/etc/cont-init.d, /etc/services.d, /plex-common.sh, /usr/local/bin): the image + # calls setPref for exactly SIX prefs — MachineIdentifier, ProcessedMachineIdentifier, + # PlexOnlineToken, customConnections, allowedNetworks, TranscoderTempDirectory — and supports NO + # generic PLEX_PREFERENCE* hook of any kind. The two things we need (a LAN definition and a custom + # certificate) are reachable ONLY through Plex's own API, so this hook is the only GitOps path. + # + # IT MUST NEVER FAIL. A postStart hook that exits non-zero makes kubelet KILL the container, so + # every step is bounded and the script ends with an unconditional `exit 0`. A missing token, an + # unissued certificate or an unreachable API degrades to "not applied this start" and is retried + # on the next one; it never crashloops Plex. + postStart: + exec: + command: + - /bin/bash + - -c + - | + # NOTE — LOAD-BEARING: SHELL VARIABLES HERE ARE WRITTEN BARE, NEVER BRACE-WRAPPED. + # This whole script is rewritten by Flux postBuild envsubst BEFORE it ever runs, so a + # brace-wrapped name is read as a cluster-vars knob: an undeclared one is a HARD + # Kustomization failure under StrictPostBuildSubstitutions, and a declared one would be + # replaced with its value. Shell vars are therefore `$PX`, `$TOKEN`, `$P12` — bare. Only + # BASE_DOMAIN below is brace-wrapped, because it IS a knob and IS meant to be + # substituted. Command substitution `$(cmd)` is left alone and needs no escaping. A shell + # variable written with braces would be blanked or would fail the whole Kustomization — + # this repo has already lost a gitea OIDC bootstrap to exactly that mistake, and + # catalog/render.py now refuses to emit it (which is how this very comment got caught). + # DELIBERATELY NO `set -u`. The TLS env vars are declared `optional: true`, so when the + # certificate has not issued yet they are genuinely UNSET, not empty — and under `set -u` + # the first `[ -n "$PLEX_TLS_CRT" ]` would abort the script, which for a postStart hook + # means kubelet KILLS THE CONTAINER. That is the precise failure the `optional` flag exists + # to prevent, so turning it back on here would defeat the whole design. (The alternative, + # a brace-wrapped `:-` default, would need envsubst escaping — the trap the note above + # describes.) Every value below is explicitly guarded instead. + PX="/config/Library/Application Support/Plex Media Server/Preferences.xml" + CERTDIR=/config/certs + P12=$CERTDIR/plex-vip.p12 + # The PKCS#12 passphrase protects nothing and is deliberately not a secret: Plex needs it + # in cleartext in customCertificateKey, and it and the bundle sit on the same PVC. It is a + # container-format requirement, not an access control. + P12PASS=plexvip + LANNETS='10.0.0.0/8,192.168.0.0/16' + CERTDOMAIN=plex-vip.${BASE_DOMAIN} + API=http://127.0.0.1:32400 + + # (1) Build the PKCS#12 bundle from the PEM the Secret supplies. Rebuilt on every start so + # a cert-manager RENEWAL is picked up with no extra moving part. + if [ -n "$PLEX_TLS_CRT" ] && [ -n "$PLEX_TLS_KEY" ]; then + mkdir -p $CERTDIR && chmod 700 $CERTDIR + printf '%s' "$PLEX_TLS_CRT" > $CERTDIR/tls.crt + printf '%s' "$PLEX_TLS_KEY" > $CERTDIR/tls.key + chmod 600 $CERTDIR/tls.key + if openssl pkcs12 -export -out $P12.tmp -inkey $CERTDIR/tls.key \ + -in $CERTDIR/tls.crt -passout pass:$P12PASS 2>/dev/null; then + mv -f $P12.tmp $P12 + chown 1000:1000 $P12 2>/dev/null || true + chmod 600 $P12 + echo "[plex-settings] PKCS#12 bundle written" + else + rm -f $P12.tmp + echo "[plex-settings] WARN: pkcs12 export failed; keeping any previous bundle" >&2 + fi + rm -f $CERTDIR/tls.key + else + echo "[plex-settings] WARN: TLS env empty; certificate not installed this start" >&2 + fi + + # (2) Wait, bounded (~90s), for the local API. /identity needs no auth. + i=0 + while [ "$i" -lt 45 ]; do + if [ "$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 $API/identity 2>/dev/null)" = "200" ]; then + break + fi + i=$(expr $i + 1); sleep 2 + done + + # (3) The prefs API needs the server token. It already lives in Preferences.xml on this + # PVC; it is never echoed here, and Plex redacts tokens in its own request log. An + # UNCLAIMED server has none — a normal state on a fresh /config — so skip, do not fail. + TOKEN=$(grep -o 'PlexOnlineToken="[^"]*"' "$PX" 2>/dev/null | head -1 | cut -d'"' -f2) + if [ -z "$TOKEN" ]; then + echo "[plex-settings] not claimed yet — skipping prefs, will retry next start" >&2 + exit 0 + fi + + # Percent-encode what our values actually contain. Plex takes prefs as QUERY parameters on + # a PUT, so a raw '/' or ',' would be read as structure rather than data. + enc() { + printf '%s' "$1" | sed -e 's|%|%25|g' -e 's|/|%2F|g' -e 's|,|%2C|g' \ + -e 's|:|%3A|g' -e 's| |%20|g' + } + setpref() { + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X PUT \ + -H "X-Plex-Token: $TOKEN" "$API/:/prefs?$1=$(enc "$2")" 2>/dev/null) + if [ "$code" = "200" ]; then + echo "[plex-settings] set $1" + else + echo "[plex-settings] WARN: $1 -> HTTP $code" >&2 + fi + } + + # (4) THE LAN DEFINITION — the reason this hook was written. Without it Plex has NO notion + # of a local network and tags EVERY client remote. Measured 2026-10-04: 468 requests + # logged "Allowed Network (WAN)" and ZERO "(LAN)", including Seerr from inside the + # cluster, which is why the server showed as Remote to LAN clients. Note allowedNetworks + # (the ALLOWED_NETWORKS env) only waives AUTH — it does NOT make a client local. Two + # different settings; only one of them has an env hook. + setpref LanNetworksBandwidth "$LANNETS" + + # (5) THE CERTIFICATE, which is what makes the `https` in ADVERTISE_IP servable. Set only + # if the bundle exists — pointing Plex at a missing file would break its TLS listener. + if [ -s "$P12" ]; then + setpref customCertificatePath "$P12" + setpref customCertificateKey "$P12PASS" + setpref customCertificateDomain "$CERTDOMAIN" + else + echo "[plex-settings] WARN: no bundle; leaving certificate prefs untouched" >&2 + fi + exit 0 # HARDWARE TRANSCODING — the host iGPU's render device (schema 1.8.0 `hardware`; R-PLAT-02). # INTENT ONLY, AND THE MECHANISM DELIBERATELY DOES NOT LIVE HERE. This says "this workload needs an # Intel Quick Sync render device"; it does NOT name a device path and it does NOT name a resource. diff --git a/catalog/render.py b/catalog/render.py index 894d6f0..277226f 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -2209,6 +2209,23 @@ EXTRA_UNITS: dict = { "plex-remote.yaml", ], }, + # Plex's own TLS certificate for the single hostname it advertises to plex.tv + # (apps/wave-0/plex-cert.yaml). `above: False` co-locates it in plex's OWN wave, which is the + # only placement available: the Certificate must land in the `plex` namespace, and that + # namespace is created by the rendered plex.yaml in this very wave, so it CANNOT be issued + # earlier. The workload tolerates the resulting gap by marking the env refs `optional` rather + # than by ordering -- see the header of plex-cert.yaml. + # + # Non-catalog because the ServiceDefinition schema has no field for a cert-manager Certificate + # (additionalProperties is false); `ingressRoutes` inherits the Gateway wildcard instead, which + # is the opposite case to a workload terminating its own TLS on a raw L4 port. + "plex-cert": { + "anchor": "plex", + "above": False, + "files": [ + "plex-cert.yaml", + ], + }, } diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index c5f15d1..26482bf 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -101,7 +101,7 @@ data: # tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db: # S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH; pihole + pihole-dns: PIHOLE_DNS_VIP, LAN_CIDR, plus # GATEWAY_VIP/UNIFI_L4_VIP/CRAFTY_L4_VIP read BY pihole to author its LAN split-horizon answers; - # plex-remote: PLEX_L4_VIP; plex: PLEX_ADVERTISE_URL), + # plex-remote + pihole: PLEX_L4_VIP), # velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION, # VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an # UNSUBSTITUTED path would ship the literal string into the cluster. @@ -189,32 +189,13 @@ data: # dev -> 192.168.1.57 (base default; next free after pihole-dns .56 in dev's .53-.60 pool) # prod -> 192.168.2.24 (next free after pihole-dns .23 in prod's .20-.30 pool; see clusters/prod) PLEX_L4_VIP: "192.168.1.57" # Plex: tcp/32400 remote access (NOT the web UI, which is L7) + # SECOND CONSUMER since 2026-10-04: catalog/pihole.yaml reads this knob to answer + # plex-vip.${BASE_DOMAIN} on the LAN, which is the local half of the single hostname Plex + # advertises to plex.tv. Without that per-host answer the zone wildcard sends the name to + # ${GATEWAY_VIP}, where nothing listens on 32400. So this value is now typed into THREE places + # that must agree: this knob, the WAN port-forward on the SOHO router (off-repo), and PiHole's + # answer -- and only the first two were true before. - # PLEX_ADVERTISE_URL — the connection URL Plex publishes to plex.tv ("Custom server access URLs", - # the ADVERTISE_IP env in catalog/plex.yaml, which the image writes into the `customConnections` - # preference on every pod start). Read catalog/plex.yaml's ADVERTISE_IP block for the full why; - # the short version is the only thing that matters when editing this line: - # - # IT MUST BE A `..plex.direct` NAME, NEVER A BARE IP. Plex serves 32400 - # with a `*..plex.direct` wildcard certificate and NO IP SAN, so `https://:32400` - # fails TLS validation for every client. That exact mistake is what broke this service on - # 2026-10-04: with no completable connection, plex.tv published nothing, the server vanished from - # the account and clients reported "not authorized". The dashed name is public DNS that resolves - # to the embedded address, so it points at the LAN VIP AND matches the cert. - # - # WHY IT IS A WHOLE URL AND NOT BUILT FROM ${PLEX_L4_VIP}: postBuild substitution is plain - # envsubst — it cannot turn 192.168.2.24 into 192-168-2-24 — and `` is this server's - # certificate identity, derivable from nothing in this tree. Re-read it from the live cert - # (`openssl s_client -connect :32400 | openssl x509 -noout -subject`) if Plex ever reissues. - # - # The dashed IP here MUST track ${PLEX_L4_VIP} above. They are two halves of one address and - # nothing checks that they agree. - # - # dev -> PLACEHOLDER, non-functional. dev is decommissioned and has no claimed Plex server, so - # there is no real hash to put here; the base default only has to be a defined, correctly - # shaped string (StrictPostBuildSubstitutions makes an UNDEFINED var a hard failure). - # prod -> the real 192.168.2.24 + live server hash; see clusters/prod/kustomization.yaml. - PLEX_ADVERTISE_URL: "https://192-168-1-57.0000000000000000000000000000000f.plex.direct:32400" # The FLAT LAN this cluster's nodes and LB VIPs live on, in CIDR form. Consumed as the ingress # peer of the PiHole DNS CiliumNetworkPolicy (apps/wave-0/pihole-dns.yaml): a resolver reachable diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index 0046ed6..e518e64 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -496,16 +496,6 @@ patches: - op: replace path: /data/PLEX_L4_VIP value: "192.168.2.24" - # PLEX_ADVERTISE_URL — the prod VIP above in Plex's `plex.direct` form, which is the ONLY form - # that validates: the server's cert is `*.130072f20c334d6c8f63d0dfef87309b.plex.direct` with no - # IP SAN (read off the live listener 2026-10-04), so a bare `https://192.168.2.24:32400` fails - # TLS for every client — the 2026-10-04 outage. Measured with full verification, no `curl -k`: - # this URL answers /identity with 200 and TLS verify OK; the bare-IP form fails to connect. - # THE DASHED IP MUST MATCH /data/PLEX_L4_VIP ABOVE (192.168.2.24 -> 192-168-2-24); nothing - # enforces that. The hash is server identity, not a secret, and changes only if Plex reissues. - - op: replace - path: /data/PLEX_ADVERTISE_URL - value: "https://192-168-2-24.130072f20c334d6c8f63d0dfef87309b.plex.direct:32400" # The prod flat LAN. This is the 192.168.2.0/24 segment the SOHO ROUTER at 192.168.2.1 owns — # NOT pfSense's 192.168.1.0/24, which is a different network and not in the LAN-DNS path. # It scopes the ingress allow for PiHole's DNS listener to LAN clients only. diff --git a/decisions/2026-10-04-plex-vip-hostname-and-lan-networks.md b/decisions/2026-10-04-plex-vip-hostname-and-lan-networks.md new file mode 100644 index 0000000..0c3917b --- /dev/null +++ b/decisions/2026-10-04-plex-vip-hostname-and-lan-networks.md @@ -0,0 +1,151 @@ +--- +date: 2026-10-04 +branch: feat/plex-vip-hostname-and-lan-networks +title: Plex — one advertised hostname for local and remote, its own LE certificate, and a LAN definition so LAN clients stop being reported as remote +--- + +## What changed + +Two operator-reported symptoms, after the 2026-10-04 "server missing from the account" fix +(decisions/2026-10-04-plex-advertise-plexdirect.md) had restored publishing: + +1. **LAN clients were shown as Remote.** Root cause, measured: `LanNetworksBandwidth` was **ABSENT**, + so Plex had no definition of a local network and classified *everything* as WAN — **468 requests + logged `Allowed Network (WAN)` and ZERO `(LAN)`**, including Seerr dialling from inside the + cluster. `allowedNetworks` (the `ALLOWED_NETWORKS` env) only waives **auth**; it does not make a + client local. Two different settings, and only one of them has an env hook. +2. **The advertised URL was a single-purpose, fragile value.** The previous fix advertised + `https://..plex.direct:32400`, which works, but hard-codes the VIP *and* + the server's certificate hash into a cluster-var — two values nothing could keep in sync. + +**Operator design adopted (2026-10-04): ONE hostname, `plex-vip.${BASE_DOMAIN}`, serving both +audiences by split-horizon DNS** — PiHole answers it with `${PLEX_L4_VIP}` on the LAN, Cloudflare +answers it publicly with the WAN address, NATed to the same VIP. + +- **`catalog/plex.yaml`:** `ADVERTISE_IP` → `https://plex-vip.${BASE_DOMAIN}:32400`. New + `lifecycle.postStart` hook (level-triggered, same idiom as pihole's and qbittorrent's) that + re-asserts on **every** start: `LanNetworksBandwidth`, and the three `customCertificate*` prefs + after converting the PEM to the PKCS#12 bundle Plex requires. New `PLEX_TLS_CRT`/`PLEX_TLS_KEY` + env from the `plex-vip-tls` Secret, both `optional: true`. +- **`apps/wave-0/plex-cert.yaml` (new, hand-written):** a cert-manager `Certificate` for exactly + `plex-vip.${BASE_DOMAIN}` from ClusterIssuer `letsencrypt-prod`. Non-catalog because the schema has + no field for a Certificate. A single name, not the wildcard: the wildcard's Secret lives in + `gateway` and Secrets are not readable across namespaces, and re-requesting `*.${BASE_DOMAIN}` + would burn a duplicate-certificate slot for no benefit. +- **`schemas/servicedefinition.schema.json` → 1.11.0:** adds `optional` to + `env.valueFrom.secretKeyRef` / `configMapKeyRef`. **This is load-bearing, not tidiness.** The + Certificate must be issued into the `plex` namespace, and that namespace is created by the + rendered `apps/wave-0/plex.yaml` itself — so it cannot be issued in an earlier tier and lands in + the *same* reconcile as the workload. With hard refs, Plex would sit in + `CreateContainerConfigError` until a certificate issued, and `apps-wave-0` is `wait: true` with + `apps-wave-1` dependsOn it, so a slow or failed DNS01 challenge would have **stalled the whole app + tier** — the hazard `catalog/plex.yaml` already documents for its GPU resource. +- **`catalog/pihole.yaml`:** `address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}` added to + `FTLCONF_misc_dnsmasq_lines`. Required, not cosmetic: measured 2026-10-04, + `plex-vip.${BASE_DOMAIN}` resolved to **`192.168.2.20`** (the Gateway VIP) via the zone wildcard, + and a connect to `:32400` there returned **`http=000`**. +- **`clusters/base/cluster-vars.yaml` + `clusters/prod/kustomization.yaml`:** `PLEX_ADVERTISE_URL` + **deleted** — the hostname makes it unnecessary, which removes the cert-hash fragility outright. + `PLEX_L4_VIP`'s note now records its second consumer (pihole). +- **`catalog/render.py`:** `plex-cert` EXTRA_UNITS entry; **`apps/wave-0/kustomization.yaml`:** + `plex-cert.yaml` hand-added (NOT via `--emit-waves`, which would also onboard `keycloak` and + `vm-demo` — the pre-existing drift recorded on 2026-10-03). + +**A correction to my own earlier reasoning, recorded because it was wrong in the handoff log.** The +previous entry asserted a bare-IP URL "can NEVER validate" and treated a SAN match as a +*requirement*. The operator pointed out it is not: `httpsRequired=0` and `secureConnections` default +to Preferred, so clients can fall back to cleartext. The measurements still stand (a validating +client did fail, and real clients did send `certificate unknown`) — but the correct framing is that a +matching certificate keeps the remote leg **encrypted**, not that it is required for Plex to +function. That is why this branch obtains a real certificate for the advertised name rather than +choosing the name to suit Plex's built-in one. + +**SENSITIVE PATHS — touches `clusters/**` (two files), so `ci/approval-policy.sh` will BLOCK until an +approving review (or the operator's `/approve-override ` posted as a PR *review* of type +"Comment"). Deliberately not self-posted: CLAUDE.md §1.** + +## What I verified + +Live prod, 2026-10-04, read-only. Secrets per CLAUDE.md §6: prefs were read by name-scoped grep, +the server token was used inside the pod and never printed, and no `Preferences.xml` was dumped. + +- **The LAN-classification root cause, directly:** `LanNetworksBandwidth=`, and request + tagging of **468 × `Allowed Network (WAN)` vs 0 × `(LAN)`** — including `10.0.2.33` (Seerr), an + in-cluster pod. This is the measurement the whole hook rests on. +- **All four prefs the hook sets EXIST on this build**, read from the server's own `/:/prefs` + (token-authenticated, output filtered to ids): `LanNetworksBandwidth`, `customCertificatePath`, + `customCertificateKey`, `customCertificateDomain`, all `type="text"`. I had earlier been unable to + confirm this (an unauthenticated `GET /:/prefs` returns **401**) and said so rather than assuming. +- **There is NO env path for these settings** — the reason the hook exists. Grep of the image's + entire init surface: `setPref` is called for exactly six prefs (`MachineIdentifier`, + `ProcessedMachineIdentifier`, `PlexOnlineToken`, `customConnections`, `allowedNetworks`, + `TranscoderTempDirectory`) and the only `PLEX_*` env it reads are CLAIM/CLAIM_FILE/UID/GID/ + UPDATE_CHANNEL and five INFO/HOME vars. No `PLEX_PREFERENCE*` of any kind; zero files mention + `LanNetworks`. +- **The DNS gap was real:** `plex-vip.${BASE_DOMAIN}` resolved to `192.168.2.20` (Gateway VIP), and + `https://192.168.2.20:32400` returned `http=000` (nothing listening). PiHole's wildcard is why. +- **`openssl` is present in the image** (`/usr/bin/openssl`, 3.0.13) — the hook's PEM→PKCS#12 + conversion has a tool to run. +- **The issuer exists and works:** ClusterIssuer `letsencrypt-prod` `READY=True` (28d), and the + existing `gateway/wildcard-home` Certificate is `Ready`/not-expired, which is what makes a DNS01 + issuance for a new name in this zone a reasonable expectation. +- **The renderer's substitution guard caught a real defect in my own work**, which is recorded + because it is the second time this class of bug has appeared here: my hook's explanatory comment + contained brace-wrapped names, and `assert_substitution_safe` refused to emit + (`'PX' is not a declared cluster-vars key — envsubst will replace it with the EMPTY STRING`). + Reworded; the rendered hook now contains exactly one brace-wrapped name, `BASE_DOMAIN`, verified + by parsing the emitted manifest. +- **A `set -u` bug I introduced and then removed.** With `optional: true` the TLS env vars are + genuinely UNSET when no certificate has issued; under `set -u` the first `[ -n ... ]` would abort + the hook, and a postStart hook exiting non-zero makes kubelet KILL the container — the exact + failure `optional` exists to prevent. Verified by parsing the rendered manifest that no `set -u` + **command** remains (only two mentions in comments). +- **Toolbox gate** (`./infra/devcontainer/dev.sh run bash -c './ci/gate.sh'`): **rc 3**, with + `spec-immutability`, `decisions-log`, `lint`, `l0`, `render-golden`, `l1` and `approval-policy` all + **pass**; the rc 3 is the two known environmental skips (`gitea-settings` — no local + `SETTINGS_API_TOKEN`; `conformance` L2/L3 — `CLUSTER_AVAILABLE` unset). **Not green. Reported as + rc 3.** `l0` passing is the specific proof the schema edit admits `optional` and still validates + every catalog file; `render-golden` passing is the proof catalog and committed goldens agree. +- **Only plex and pihole goldens moved.** No other service's manifest changed, and no service was + silently onboarded. + +## What I did NOT verify + +- **THE OUTCOME, ENTIRELY. Nothing in this branch has been observed working.** It is unreconciled: + no Certificate exists in prod, `plex-vip` still resolves to the Gateway VIP, `LanNetworksBandwidth` + is still absent, and the hook has NEVER RUN. Every claim above is about the inputs, not the result. +- **Whether the certificate will actually ISSUE.** DNS01 for a new name in this zone is inferred from + the wildcard's success, not demonstrated. If the Cloudflare credential or solver is scoped in a way + that fails for this name, Plex will start (by design) with no TLS and the advertised `https://` URL + will be unservable — i.e. **this branch could leave the advertised URL broken in a new way**. First + reconcile must be watched. +- **Whether `openssl pkcs12 -export` produces a bundle PLEX ACCEPTS.** OpenSSL 3 defaults to AES-256 + + SHA-256, which some consumers reject. Untested against Plex. If it is rejected, Plex may fail to + load its TLS listener — the hook guards against a *missing* file, NOT against a present-but- + unreadable one. **This is the single largest unverified risk in the change.** +- **Whether the certificate prefs take effect without a restart.** Plex may need to re-initialise its + listener after `customCertificatePath` changes. If so, the first reconcile applies the prefs and + the SECOND start serves the cert. Converges either way, but the first start may look wrong. +- **Whether `LanNetworksBandwidth` actually flips the Remote badge.** It is the correct setting by + Plex's own documentation and the measured absence explains the symptom exactly, but I did not set it + live and observe a client reclassify. +- **SNAT makes the LAN definition coarser than it looks.** With `externalTrafficPolicy: Cluster`, Plex + sees the ingress NODE's address for everything arriving via the VIP, so once `10.0.0.0/8` and + `192.168.0.0/16` are LAN, GENUINE REMOTE clients arriving that way will ALSO be classified local + and remote bandwidth limits will not bind on them. This is the metering consequence already + accepted on 2026-10-03, now made stronger rather than weaker. Not measured. +- **The public Cloudflare record for `plex-vip` is off-repo and absent from this branch**, as is the + WAN tcp/32400 port-forward. Remote access cannot work until the operator creates both; nothing here + can verify either, and this repo has no off-LAN vantage point. +- **The private key is exposed in the container's environment** (`/proc/1/environ`) because the schema + and renderer support no secret VOLUME — measured: `storage` accepts only + [class, export, mountPath, name, readOnly, size, subPath] with `additionalProperties: false`, and + no rendered app mounts a Secret. A volume would be the better shape; adding secret-volume support + is the clean follow-up and I did not do it. +- **The PKCS#12 passphrase is a hardcoded literal** (`plexvip`) in the rendered manifest. It protects + nothing — Plex needs it in cleartext in `customCertificateKey` and it sits beside the bundle on the + same PVC — but it is worth knowing it is not a secret and was never treated as one. +- **Four places must agree and nothing enforces it:** `ADVERTISE_IP`, the hook's + `customCertificateDomain`, `plex-cert.yaml`'s `dnsNames`, and PiHole's answer. No gate stage or + conformance test checks that they match, or that PiHole's `plex-vip` answer still equals + `${PLEX_L4_VIP}`. diff --git a/schemas/servicedefinition.schema.json b/schemas/servicedefinition.schema.json index 34f5543..1d6f30a 100644 --- a/schemas/servicedefinition.schema.json +++ b/schemas/servicedefinition.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json", "title": "ServiceDefinition", - "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.10.0 (adds optional `availability` for the container kind, one member: `availability.replicas`. It is AVAILABILITY INTENT, not a scheduling primitive -- it names no Kubernetes object, no topology key and no budget. 1 (the default) renders a Deployment exactly as before, so every pre-existing app stays byte-identical; >1 means the service must survive the loss of a single node, and the renderer satisfies that with a StatefulSet whose longhorn volumes become per-replica volumeClaimTemplates (a longhorn PVC is ReadWriteOnce and cannot attach on two nodes, so a multi-replica Deployment is UNSCHEDULABLE, not merely suboptimal), REQUIRED podAntiAffinity on kubernetes.io/hostname (preferred would let every replica land on one node and deliver none of the intended tolerance while reporting healthy), podManagementPolicy Parallel, a PodDisruptionBudget of minAvailable 1, and an `up` SLI that follows the rendered kind (a Deployment-shaped rule against a StatefulSet absent()s forever and can never fire). Added for pihole, where LAN DNS -- a critical network service -- was one pod on one node: measured 2026-10-04, losing the node running it meant roughly 6-10 minutes with no LAN resolution (tolerationSeconds 300 plus node-NotReady detection plus a Longhorn RWO force-detach). Deliberately NOT usable until a service's state is reproducible from git, because a replica with a fresh volume otherwise comes up serving but unconfigured -- for pihole that prerequisite was done first (decisions/2026-10-04-pihole-declare-state.md). Capped at 5: a count above the node count can never schedule under the anti-affinity this implies. Mechanism documented in schemas/renderer-contract.md `### availability`). 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", + "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.11.0 (adds optional `optional` to env.valueFrom.secretKeyRef and env.valueFrom.configMapKeyRef -- the k8s EnvVarSource field, emitted verbatim. Without it a secretKeyRef was ALWAYS hard: a missing Secret meant the pod would not start at all. That is the right default and stays the default, but it cannot express a value the workload is designed to run without. Added for plex, whose TLS material is issued by cert-manager INTO ITS OWN NAMESPACE and therefore cannot exist before the app wave that creates that namespace -- the Certificate and the Deployment land in the same reconcile, so on a cold bootstrap the Secret is briefly absent while a DNS01 challenge completes. Hard refs there would have made the media server unstartable until a certificate issued, and `apps-wave-0` is `wait: true`, so a failed or slow issuance would have STALLED THE WHOLE APP TIER -- the same hazard catalog/plex.yaml already documents for its GPU resource. With `optional: true` Plex starts regardless, its postStart hook logs that the certificate is not installed, and the next start after issuance converges. Renders nothing when absent, so every existing app stays byte-identical). 1.10.0 (adds optional `availability` for the container kind, one member: `availability.replicas`. It is AVAILABILITY INTENT, not a scheduling primitive -- it names no Kubernetes object, no topology key and no budget. 1 (the default) renders a Deployment exactly as before, so every pre-existing app stays byte-identical; >1 means the service must survive the loss of a single node, and the renderer satisfies that with a StatefulSet whose longhorn volumes become per-replica volumeClaimTemplates (a longhorn PVC is ReadWriteOnce and cannot attach on two nodes, so a multi-replica Deployment is UNSCHEDULABLE, not merely suboptimal), REQUIRED podAntiAffinity on kubernetes.io/hostname (preferred would let every replica land on one node and deliver none of the intended tolerance while reporting healthy), podManagementPolicy Parallel, a PodDisruptionBudget of minAvailable 1, and an `up` SLI that follows the rendered kind (a Deployment-shaped rule against a StatefulSet absent()s forever and can never fire). Added for pihole, where LAN DNS -- a critical network service -- was one pod on one node: measured 2026-10-04, losing the node running it meant roughly 6-10 minutes with no LAN resolution (tolerationSeconds 300 plus node-NotReady detection plus a Longhorn RWO force-detach). Deliberately NOT usable until a service's state is reproducible from git, because a replica with a fresh volume otherwise comes up serving but unconfigured -- for pihole that prerequisite was done first (decisions/2026-10-04-pihole-declare-state.md). Capped at 5: a count above the node count can never schedule under the anti-affinity this implies. Mechanism documented in schemas/renderer-contract.md `### availability`). 1.9.0 (adds optional `auth.apiRoutes` -- a per-service list of anchored path regexes the forward-auth oauth2-proxy sidecar answers with 401 instead of a 302 to Keycloak, rendered as one repeatable `--api-route` per entry. The `auth` block stays `additionalProperties: false`; this is one named, documented key, not an opened door. Without it there was NO per-app override hook at all, so the only way to answer an app's XHRs with 401 was a renderer-wide constant -- and the path shape is a property of the app: jackett's is `/api/` (33 x 302 vs 2 x 401 measured 2026-09-30, 4 of 6 POSTs answered by the proxy and never reaching the app), radarr's is `/signalr/` (232 x 302 measured 2026-10-01, every one a POST from a left-open browser tab that is never re-prompted), and pihole serves no `/api/` at all. Each entry must be anchored at `^/` and continue with literal path characters, because a too-broad regex turns a plain page load into a 401 and the service looks permanently broken to a logged-out user. Enforced on every gate run by ci/l0.sh's schema stage over catalog/*.yaml, by catalog/render.py refusing to render an unanchored entry, and by conformance/tests/l0-static/oauth2-proxy-api-routes-per-app)). 1.8.0 (adds optional `hardware` - a host HARDWARE CAPABILITY declaration for the container kind, currently one member: `hardware.gpu: intel-quicksync`, meaning the workload needs the host iGPU render device for hardware video transcode. It is an ENUM OF INTENT, not a device path: the renderer satisfies it today with just two things -- a /dev/dri hostPath volume (`DirectoryOrCreate`) and a read-write /dev/dri volumeMount on the app container -- and a later move to the Intel GPU device plugin is then a render.py change with no catalog edit. It sets NO podSecurityContext.supplementalGroups: measured in-container on prod 2026-09-30, the pms-docker cont-init hook adds the app user to the device's group inside the container and `s6-setuidgid` then REPLACES the process group list from /etc/group, discarding anything Kubernetes supplied, so such an entry would be inert and misleading. Renders NOTHING when absent, so every existing app stays byte-identical. Emits no privileged securityContext (Kyverno restrict-host-namespaces would reject it and it is not needed), no nodeSelector (node placement is a separate concern with its own justification) and no resource requests). 1.7.0 (no field changes; a MAPPING change to `exposure`, which had never had an effect on DNS. `exposure: internal` now renders ExternalDNS's own controller annotation onto every HTTPRoute it emits, set to a value other than `dns-controller`, so external-dns (v0.21.0, source/gateway.go -> annotations.IsControllerMismatch) skips the route before resolving any hostname and the service is INCAPABLE of producing a public record. Both docs/09 R-DNS-03 and schemas/renderer-contract.md already said `internal` renders no public DNS record; nothing enforced it, and six internal services -- jackett, pihole, qbittorrent, radarr, semaphore, sonarr -- were publishing `.home.keithjackson.dev` into the PUBLIC Cloudflare zone answering with the RFC1918 Gateway VIP, which published the internal subnet and a full service inventory and made each name a DNS-rebind primitive. Enforced on every gate run by conformance/tests/l0-static/internal-services-publish-no-public-dns). 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", "type": "object", "additionalProperties": false, "required": ["name", "source"], @@ -69,7 +69,8 @@ "required": ["name", "key"], "properties": { "name": { "type": "string", "minLength": 1 }, - "key": { "type": "string", "minLength": 1 } + "key": { "type": "string", "minLength": 1 }, + "optional": { "type": "boolean", "description": "Tolerate the key being ABSENT: the env var is simply unset instead of the pod refusing to start (kubelet CreateContainerConfigError). Default false, which is the pre-1.11.0 behaviour. Use ONLY where the workload degrades gracefully without the value, and say so at the call site -- a silently-empty credential is usually worse than a pod that will not start." } } }, "configMapKeyRef": { @@ -78,7 +79,8 @@ "required": ["name", "key"], "properties": { "name": { "type": "string", "minLength": 1 }, - "key": { "type": "string", "minLength": 1 } + "key": { "type": "string", "minLength": 1 }, + "optional": { "type": "boolean", "description": "Tolerate the key being ABSENT: the env var is simply unset instead of the pod refusing to start (kubelet CreateContainerConfigError). Default false, which is the pre-1.11.0 behaviour. Use ONLY where the workload degrades gracefully without the value, and say so at the call site -- a silently-empty credential is usually worse than a pod that will not start." } } } }