I broke the app tier with #48. Two faults, both mine, both measured after merge: 1. KYVERNO BLOCKS cert-manager IN APP NAMESPACES. The temporary private-key Secrets (plex-vip-<random>) were denied by secret-provenance/secrets-must-be-flux-managed, which matches every Secret in a namespace labelled homelab.keithjackson.dev/app-namespace=true and excludes only those labelled secret-managed-by=controller or carrying a Flux label. `plex` HAS that label; `gateway` -- where the working wildcard cert lives -- does NOT. That asymmetry is why the existing Certificate was never affected and mine was, and I did not check it. secretTemplate cannot fix it: those Secrets come from the keymanager controller, not the Certificate template. So no CertificateRequest, Order or Challenge was ever created. 2. FLUX HEALTH-CHECKS THE CERTIFICATE. `optional: true` did its job -- the pod rolled and ran fine without the Secret -- but apps-wave-0 is wait:true, so Flux waits on the Certificate OBJECT: timeout waiting for: [Certificate/plex/plex-vip status: 'InProgress'] apps-wave-0 never went Ready; wave-1/-2 and everything downstream of platform-config sat in DependencyNotReady behind it. I guarded the pod and missed the object. That distinction is the whole bug. Third, independent: the advertised URL was https with no servable certificate -- Plex answers 32400 with its *.plex.direct cert, which cannot cover plex-vip -- the exact failure mode of the outage earlier the same day. So: delete the Certificate (pruning it clears the health gate) and advertise http://plex-vip:32400, which is MEASURED to work today -- PiHole answers the name with the Plex VIP (verified against pihole-0) and Plex serves 32400 there. http is explicitly temporary and only tolerable because remote access does not work yet; the restore condition is written at the call site. DELETED, not merely unregistered: my first attempt kept the file as "tracked but not onboarded" and the gate rejected it -- tls-lifecycle-automated (R-TLS-02) fails any Certificate no Flux path reconciles. Legal for a Deployment, not for a TLS object. The re-enable recipe (Kyverno exemption + a wait:false Kustomization + the https flip, as ONE change) is recorded in catalog/plex.yaml. Kept and verified working from #48: the hostname design, PiHole's split-horizon answer, and the postStart hook -- LanNetworksBandwidth is now set live. Gate: rc 3, all substantive stages pass (first attempt was rc 1 BLOCKED). Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
343 lines
13 KiB
YAML
343 lines
13 KiB
YAML
---
|
|
# Namespace (trust zone; default-deny + policies apply)
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: plex
|
|
labels:
|
|
homelab.keithjackson.dev/app-namespace: 'true'
|
|
gateway-access: shared
|
|
annotations:
|
|
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
|
---
|
|
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: default-deny
|
|
namespace: plex
|
|
labels:
|
|
homelab.keithjackson.dev/baseline: default-deny
|
|
spec:
|
|
podSelector: {}
|
|
policyTypes:
|
|
- Ingress
|
|
- Egress
|
|
ingress: []
|
|
egress:
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: kube-system
|
|
ports:
|
|
- protocol: UDP
|
|
port: 53
|
|
- protocol: TCP
|
|
port: 53
|
|
---
|
|
# PersistentVolume plex-media - static external NFS (bulk tier)
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata:
|
|
name: plex-media
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
capacity:
|
|
storage: 9Ti
|
|
accessModes:
|
|
- ReadOnlyMany
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: ''
|
|
mountOptions:
|
|
- nfsvers=4.1
|
|
- hard
|
|
- noatime
|
|
- timeo=600
|
|
- retrans=2
|
|
nfs:
|
|
server: ${BULK_NFS_SERVER}
|
|
path: /mnt/Storage/plex
|
|
readOnly: true
|
|
claimRef:
|
|
namespace: plex
|
|
name: plex-media
|
|
---
|
|
# PVC plex-config (R-STATE)
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: plex-config
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 8Gi
|
|
---
|
|
# PVC plex-media (R-STATE)
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: plex-media
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
accessModes:
|
|
- ReadOnlyMany
|
|
storageClassName: ''
|
|
volumeName: plex-media
|
|
resources:
|
|
requests:
|
|
storage: 9Ti
|
|
---
|
|
# Deployment (R-PLAT-02)
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: plex
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: plex
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
homelab.keithjackson.dev/egress-class: internet
|
|
homelab.keithjackson.dev/auth-mode: native
|
|
spec:
|
|
containers:
|
|
- name: plex
|
|
image: plexinc/pms-docker:latest
|
|
ports:
|
|
- containerPort: 32400
|
|
env:
|
|
- name: PLEX_UID
|
|
value: '1000'
|
|
- name: PLEX_GID
|
|
value: '1000'
|
|
- name: TZ
|
|
value: Etc/UTC
|
|
- name: ALLOWED_NETWORKS
|
|
value: 10.0.0.0/8,192.168.0.0/16
|
|
- name: PLEX_CLAIM
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: plex-claim
|
|
key: PLEX_CLAIM
|
|
- name: ADVERTISE_IP
|
|
value: http://plex-vip.${BASE_DOMAIN}:32400
|
|
- name: PLEX_TLS_CRT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: plex-vip-tls
|
|
key: tls.crt
|
|
optional: true
|
|
- name: PLEX_TLS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: plex-vip-tls
|
|
key: tls.key
|
|
optional: true
|
|
resources:
|
|
limits:
|
|
gpu.intel.com/i915: 1
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /config
|
|
- name: media
|
|
mountPath: /media
|
|
readOnly: true
|
|
lifecycle:
|
|
postStart:
|
|
exec:
|
|
command:
|
|
- /bin/bash
|
|
- -c
|
|
- "# NOTE \u2014 LOAD-BEARING: SHELL VARIABLES HERE ARE WRITTEN BARE, NEVER BRACE-WRAPPED.\n# This whole script is rewritten by Flux postBuild envsubst BEFORE it ever runs, so a\n# brace-wrapped name is read as a cluster-vars knob: an undeclared one is a HARD\n# Kustomization failure under StrictPostBuildSubstitutions, and a declared one would be\n# replaced with its value. Shell vars are therefore `$PX`, `$TOKEN`, `$P12` \u2014 bare. Only\n# BASE_DOMAIN below is brace-wrapped, because it IS a knob and IS meant to be\n# substituted. Command substitution `$(cmd)` is left alone and needs no escaping. A shell\n# variable written with braces would be blanked or would fail the whole Kustomization \u2014\n# this repo has already lost a gitea OIDC bootstrap to exactly that mistake, and\n# catalog/render.py now refuses to emit it (which is how this very comment got caught).\n# DELIBERATELY NO `set -u`. The TLS env vars are declared `optional: true`, so when the\n# certificate has not issued yet they are genuinely UNSET, not empty \u2014 and under `set -u`\n# the first `[ -n \"$PLEX_TLS_CRT\" ]` would abort the script, which for a postStart hook\n# means kubelet KILLS THE CONTAINER. That is the precise failure the `optional` flag exists\n# to prevent, so turning it back on here would defeat the whole design. (The alternative,\n# a brace-wrapped `:-` default, would need envsubst escaping \u2014 the trap the note above\n# describes.) Every value below is explicitly guarded instead.\nPX=\"/config/Library/Application Support/Plex Media Server/Preferences.xml\"\nCERTDIR=/config/certs\nP12=$CERTDIR/plex-vip.p12\n# The PKCS#12 passphrase protects nothing and is deliberately not a secret: Plex needs it\n# in cleartext in customCertificateKey, and it and the bundle sit on the same PVC. It is a\n# container-format requirement, not an access control.\nP12PASS=plexvip\nLANNETS='10.0.0.0/8,192.168.0.0/16'\nCERTDOMAIN=plex-vip.${BASE_DOMAIN}\nAPI=http://127.0.0.1:32400\n\n# (1) Build the PKCS#12 bundle from the PEM the Secret supplies. Rebuilt on every start so\n# a cert-manager RENEWAL is picked up with no extra moving part.\nif [ -n \"$PLEX_TLS_CRT\" ] && [ -n \"$PLEX_TLS_KEY\" ]; then\n mkdir -p $CERTDIR && chmod 700 $CERTDIR\n printf '%s' \"$PLEX_TLS_CRT\" > $CERTDIR/tls.crt\n printf '%s' \"$PLEX_TLS_KEY\" > $CERTDIR/tls.key\n chmod 600 $CERTDIR/tls.key\n if openssl pkcs12 -export -out $P12.tmp -inkey $CERTDIR/tls.key \\\n -in $CERTDIR/tls.crt -passout pass:$P12PASS 2>/dev/null; then\n mv -f $P12.tmp $P12\n chown 1000:1000 $P12 2>/dev/null || true\n chmod 600 $P12\n echo \"[plex-settings] PKCS#12 bundle written\"\n else\n rm -f $P12.tmp\n echo \"[plex-settings] WARN: pkcs12 export failed; keeping any previous bundle\" >&2\n fi\n rm -f $CERTDIR/tls.key\nelse\n echo \"[plex-settings] WARN: TLS env empty; certificate not installed this start\" >&2\nfi\n\n# (2) Wait, bounded (~90s), for the local API. /identity needs no auth.\ni=0\nwhile [ \"$i\" -lt 45 ]; do\n if [ \"$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 $API/identity 2>/dev/null)\" = \"200\" ]; then\n break\n fi\n i=$(expr $i + 1); sleep 2\ndone\n\n# (3) The prefs API needs the server token. It already lives in Preferences.xml on this\n# PVC; it is never echoed here, and Plex redacts tokens in its own request log. An\n# UNCLAIMED server has none \u2014 a normal state on a fresh /config \u2014 so skip, do not fail.\nTOKEN=$(grep -o 'PlexOnlineToken=\"[^\"]*\"' \"$PX\" 2>/dev/null | head -1 | cut -d'\"' -f2)\nif [ -z \"$TOKEN\" ]; then\n echo \"[plex-settings] not claimed yet \u2014 skipping prefs, will retry next start\" >&2\n exit 0\nfi\n\n# Percent-encode what our values actually contain. Plex takes prefs as QUERY parameters on\n# a PUT, so a raw '/' or ',' would be read as structure rather than data.\nenc() {\n printf '%s' \"$1\" | sed -e 's|%|%25|g' -e 's|/|%2F|g' -e 's|,|%2C|g' \\\n -e 's|:|%3A|g' -e 's| |%20|g'\n}\nsetpref() {\n code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X PUT\
|
|
\ \\\n -H \"X-Plex-Token: $TOKEN\" \"$API/:/prefs?$1=$(enc \"$2\")\" 2>/dev/null)\n if [ \"$code\" = \"200\" ]; then\n echo \"[plex-settings] set $1\"\n else\n echo \"[plex-settings] WARN: $1 -> HTTP $code\" >&2\n fi\n}\n\n# (4) THE LAN DEFINITION \u2014 the reason this hook was written. Without it Plex has NO notion\n# of a local network and tags EVERY client remote. Measured 2026-10-04: 468 requests\n# logged \"Allowed Network (WAN)\" and ZERO \"(LAN)\", including Seerr from inside the\n# cluster, which is why the server showed as Remote to LAN clients. Note allowedNetworks\n# (the ALLOWED_NETWORKS env) only waives AUTH \u2014 it does NOT make a client local. Two\n# different settings; only one of them has an env hook.\nsetpref LanNetworksBandwidth \"$LANNETS\"\n\n# (5) THE CERTIFICATE, which is what makes the `https` in ADVERTISE_IP servable. Set only\n# if the bundle exists \u2014 pointing Plex at a missing file would break its TLS listener.\nif [ -s \"$P12\" ]; then\n setpref customCertificatePath \"$P12\"\n setpref customCertificateKey \"$P12PASS\"\n setpref customCertificateDomain \"$CERTDOMAIN\"\nelse\n echo \"[plex-settings] WARN: no bundle; leaving certificate prefs untouched\" >&2\nfi\nexit 0\n"
|
|
volumes:
|
|
- name: config
|
|
persistentVolumeClaim:
|
|
claimName: plex-config
|
|
- name: media
|
|
persistentVolumeClaim:
|
|
claimName: plex-media
|
|
---
|
|
# Service
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: plex
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
selector:
|
|
app.kubernetes.io/name: plex
|
|
ports:
|
|
- name: port-32400
|
|
port: 32400
|
|
targetPort: 32400
|
|
---
|
|
# CiliumNetworkPolicy reciprocal ingress allow (R-NET-02): plex-allow-radarr-32400
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: plex-allow-radarr-32400
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: plex
|
|
ingress:
|
|
- fromEndpoints:
|
|
- matchLabels:
|
|
app.kubernetes.io/name: radarr
|
|
k8s:io.kubernetes.pod.namespace: radarr
|
|
toPorts:
|
|
- ports:
|
|
- port: '32400'
|
|
protocol: TCP
|
|
---
|
|
# CiliumNetworkPolicy reciprocal ingress allow (R-NET-02): plex-allow-seerr-32400
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: plex-allow-seerr-32400
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: plex
|
|
ingress:
|
|
- fromEndpoints:
|
|
- matchLabels:
|
|
app.kubernetes.io/name: seerr
|
|
k8s:io.kubernetes.pod.namespace: seerr
|
|
toPorts:
|
|
- ports:
|
|
- port: '32400'
|
|
protocol: TCP
|
|
---
|
|
# CiliumNetworkPolicy reciprocal ingress allow (R-NET-02): plex-allow-sonarr-32400
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: plex-allow-sonarr-32400
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: plex
|
|
ingress:
|
|
- fromEndpoints:
|
|
- matchLabels:
|
|
app.kubernetes.io/name: sonarr
|
|
k8s:io.kubernetes.pod.namespace: sonarr
|
|
toPorts:
|
|
- ports:
|
|
- port: '32400'
|
|
protocol: TCP
|
|
---
|
|
# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): plex-allow-gateway-ingress
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: plex-allow-gateway-ingress
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: plex
|
|
ingress:
|
|
- fromEntities:
|
|
- ingress
|
|
toPorts:
|
|
- ports:
|
|
- port: '32400'
|
|
protocol: TCP
|
|
---
|
|
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): plex-allow-internet-egress
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: plex-allow-internet-egress
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: plex
|
|
egress:
|
|
- toEntities:
|
|
- world
|
|
---
|
|
# HTTPRoute plex-32400 (R-TLS/R-DNS)
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: plex-32400
|
|
namespace: plex
|
|
labels:
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
parentRefs:
|
|
- name: shared
|
|
namespace: gateway
|
|
sectionName: https
|
|
hostnames:
|
|
- plex.${BASE_DOMAIN}
|
|
rules:
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /
|
|
backendRefs:
|
|
- name: plex
|
|
port: 32400
|
|
---
|
|
# PrometheusRule SLI alerts (R-OBS-01)
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: plex-slis
|
|
namespace: plex
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
app.kubernetes.io/name: plex
|
|
spec:
|
|
groups:
|
|
- name: plex.slis
|
|
rules:
|
|
- alert: plex-down
|
|
expr: kube_deployment_status_replicas_available{namespace="plex", deployment="plex"} == 0 or absent(kube_deployment_status_replicas_available{namespace="plex", deployment="plex"})
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: plex
|
|
annotations:
|
|
summary: plex is down
|
|
description: 'plex: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).'
|