apps/wave-0/kustomization.yaml

@@ -17,6 +17,7 @@ resources:
 - pihole-dns.yaml
 - pihole-oauth2-proxy-credentials.sops.yaml
 - pihole.yaml
+- plex-claim-secret.sops.yaml
 - plex.yaml
 - qbittorrent-bootstrap-secret.sops.yaml
 - qbittorrent-oauth2-proxy-credentials.sops.yaml

apps/wave-0/plex-claim-secret.sops.yaml

@@ -0,0 +1,69 @@
+# Plex plex.tv server-claim token (WS-APPS). SOPS-encrypted (age).
+# Consumed by catalog/plex.yaml as env PLEX_CLAIM via valueFrom.secretKeyRef.
+#
+# NOTE ON THESE COMMENTS: they carry NO dollar-brace expressions, deliberately. This file is applied
+# by the apps-wave-0 Flux Kustomization, which runs envsubst over the built output, and kustomize
+# preserves body comments into that stream (see conformance/tests/l0-static/flux-postbuild-
+# substitution). A braced token quoted from the upstream shell script would be a live hazard here,
+# not prose, so the upstream conditions below are described in words.
+#
+# THE VALUE SHIPPED WITH THIS FILE IS A PLACEHOLDER, DELIBERATELY — the literal string
+# `replace-me-with-a-fresh-plex-tv-claim-token`, not a credential. A https://plex.tv/claim token has
+# a ~4-minute TTL, far shorter than commit -> push -> Flux reconcile -> pod-roll, so no token
+# committed here is ever still valid when the pod starts. This file exists to make the Secret — and
+# therefore the env reference — ALWAYS RESOLVABLE IN THE FLUX GRAPH, not to carry a working token.
+# The real token is injected by the operator inside that 4-minute window: runbooks/plex-claim.md.
+#
+# WHY SHIPPING A PLACEHOLDER IS SAFE (verified against plexinc/pms-docker branch `master`, the
+# s6-overlay v2 script root/etc/cont-init.d/40-plex-first-run, cross-checked byte-identical against
+# the same file extracted from the running `plexinc/pms-docker:latest` image):
+#   - line 106 guards the claim exchange on TWO conditions: PLEX_CLAIM non-empty AND the
+#     PlexOnlineToken pref read out of /config empty. Once the server is claimed that block is
+#     skipped entirely, so whatever this Secret holds is a NO-OP on every later start. Upstream
+#     README: "If server is already logged in, this parameter is ignored."
+#   - on a still-UNCLAIMED /config, an invalid or expired token makes plex.tv answer 403 with an
+#     empty body. curl's exit status is never checked, the sed for the authentication-token element
+#     yields the empty string, and the success branch at line 120 simply does not fire. There is no
+#     else branch and no non-zero exit; the script has no `set -e` and ends by touching
+#     /.firstRunComplete and echoing, so it exits 0. The server comes up UNCLAIMED and serving.
+#     It does NOT crashloop.
+#   - the image never sets S6_BEHAVIOUR_IF_STAGE2_FAILS, whose default is 0 ("continue silently even
+#     if a cont-init script failed"), so even a hypothetical non-zero exit would not halt startup.
+# A MISSING Secret would NOT be safe: the ServiceDefinition schema's valueFrom.secretKeyRef shape
+# has no `optional` field, so an absent Secret is a hard CreateContainerConfigError. That is exactly
+# why this file is committed and listed in apps/wave-0/kustomization.yaml instead of being created
+# by hand at claim time.
+#
+# INJECTION / ROTATION (operator, not agents) — runbooks/plex-claim.md has the timed procedure:
+#   infra/devcontainer/dev.sh run sops apps/wave-0/plex-claim-secret.sops.yaml
+# Never decrypt this file into a shell, a log, or a shared terminal.
+apiVersion: v1
+kind: Secret
+metadata:
+    name: plex-claim
+    namespace: plex
+    labels:
+        app.kubernetes.io/name: plex
+type: Opaque
+stringData:
+    PLEX_CLAIM: ENC[AES256_GCM,data:8O9nvFw+6OR+53mXvHfe321aSmYUQcZuAqLKZpcSlUwNyeU+Nbp/tRPBlg==,iv:EbvvOsXlXjAFOewnWbK28ErmUdWcJJN0Q5Gm53El3ZM=,tag:wXaInGFQ+wqUt765vvjCDA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2dWtqOVNQalZ2QlRxOXJa
+            cDc1MTdtWENkdWIxS1JId3dIU1JDSmk0ODNrCk1YUzhFWXR3bUxyV2NNcmFOakdz
+            THl1Q284dFJxTy9Rd1h0eXJJaVZtdmcKLS0tIERuN0ZFeW9zdi9aNFRYNWxmUS9s
+            QUtYdHEvN042ajNoNktOLzlYQmQrc1kK4NHozMbXVzNGYbUBRbK6a9Uw/sVPAfe6
+            RW5KWzZ4I/hloqjN3zzVISoSMieMkBGDnF5qZS5N6FqIc6/HrsDD8A==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-09-29T03:25:20Z"
+    mac: ENC[AES256_GCM,data:u5uVmOdXN78UaKUMkPZzqeW84222+tcG57rFKIKcvRKwJkzMTOuSFOeQpKh4eK4KZXVKK7+Q3mFf5G8a+jjuKjYbjSCpR38AF3TJv3xrj8fhnqR+8QHRp8tsgA0jJZBS55N7KKDyOztLFo49sI7mNnlrUwhWTQUkbLgN1LxF5MA=,iv:P4W9G2mI/3VTkvSrDZvnkJ0Qxb1nFmXcF35CrSNhako=,tag:yVsg7xsUALUoCJH0cHfQjw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

apps/wave-0/plex.yaml

@@ -132,6 +132,11 @@ spec:
           value: Etc/UTC
         - name: ALLOWED_NETWORKS
           value: 10.0.0.0/8,192.168.0.0/16
+        - name: PLEX_CLAIM
+          valueFrom:
+            secretKeyRef:
+              name: plex-claim
+              key: PLEX_CLAIM
         volumeMounts:
         - name: config
           mountPath: /config

catalog/plex.yaml

@@ -32,11 +32,40 @@ env:
   # (10.43.0.0/16); 192.168.0.0/16 covers the home LAN.
   - name: ALLOWED_NETWORKS
     value: 10.0.0.0/8,192.168.0.0/16
-  # PLEX_CLAIM is deliberately NOT set here: a https://plex.tv/claim token has a ~4-min TTL, far
-  # shorter than the commit -> push -> Flux reconcile -> pod-roll cycle, so any token committed to git
-  # is already expired by the time the pod starts. Claiming is a one-time LIVE step (UI sign-in, or a
-  # single API POST) against the running server; the claim then persists on the /config PVC and
-  # survives pod rolls (including the roll this env change causes), so it never belongs in git.
+  # PLEX_CLAIM — the ONE-SHOT plex.tv server claim, sourced from the SOPS Secret `plex-claim`
+  # (apps/wave-0/plex-claim-secret.sops.yaml, same wave => applied in the same reconcile as this
+  # workload). Plex cannot create libraries over the mounted bulk share while the server is
+  # UNCLAIMED, so this is wired; it was previously left out on the reasoning below, and only one
+  # half of that reasoning survived contact with the upstream source.
+  #
+  # THE 4-MINUTE TTL IS STILL TRUE, and it is now the OPERATIONAL constraint rather than a reason to
+  # abstain. A https://plex.tv/claim token expires in ~4 minutes — far shorter than commit -> push ->
+  # Flux reconcile -> pod-roll if that cycle is left to its own timers. So the committed value is a
+  # documented PLACEHOLDER and the live token is injected by the operator inside that window, with an
+  # explicit `flux reconcile` to collapse the wait: runbooks/plex-claim.md is the timed procedure.
+  #
+  # ONE-SHOT, AND SAFE TO SIT HERE EXPIRED. Verified from plexinc/pms-docker's own first-run script
+  # (branch `master`, root/etc/cont-init.d/40-plex-first-run, confirmed byte-identical to the file in
+  # the running `:latest` image), NOT assumed:
+  #   - the exchange (line 106) is guarded on TWO conditions: PLEX_CLAIM non-empty AND the
+  #     PlexOnlineToken pref in /config empty. Once claimed, the block is SKIPPED — this env is a
+  #     no-op on every subsequent start, whatever it holds. (Upstream README: "If server is already
+  #     logged in, this parameter is ignored.") The claim itself persists on the /config PVC as before.
+  #   - an invalid/expired token on a still-unclaimed /config gets a 403 with an empty body; curl's
+  #     status is unchecked, the token `sed` yields "", the success branch at line 120 does not fire,
+  #     there is no `else` and no `exit 1`, and the script (no `set -e`) exits 0. It comes up UNCLAIMED and
+  #     SERVING — it does NOT crashloop. The image also leaves S6_BEHAVIOUR_IF_STAGE2_FAILS at its
+  #     default 0, which continues startup even when a cont-init script fails.
+  # So the earlier "a placeholder claim Secret would crashloop the pod" note was WRONG about the
+  # placeholder; it was right only about an ABSENT Secret. secretKeyRef here has no `optional` field,
+  # so the Secret must always exist in the Flux graph — which is why it is committed, encrypted, and
+  # listed in apps/wave-0/kustomization.yaml rather than hand-created at claim time. No schema change
+  # is needed: `optional` would only matter if a missing Secret had to be tolerated, and it doesn't.
+  - name: PLEX_CLAIM
+    valueFrom:
+      secretKeyRef:
+        name: plex-claim
+        key: PLEX_CLAIM
   # ADVERTISE_IP ("Custom server access URLs") is DEFERRED, not accidentally omitted. It tells Plex
   # which URL to advertise to plex.tv for REMOTE clients. The only candidate is
   # https://plex.home.keithjackson.dev, but that host resolves via split-horizon DNS to the INTERNAL
@@ -56,10 +85,12 @@ storage:
                               # container's EPHEMERAL layer -> unclaimed + all state lost on restart. The image inits as
                               # root (s6-svscan PID 1), chowns /config to the plex user, then drops -> no fsGroup needed
                               # (verified in-cluster: PID1=root, "Plex Media Server"=plex). Persisting /config here also
-                              # makes the claim survive restarts once claimed. To auto-claim on first boot instead of the
-                              # UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token
-                              # (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`,
-                              # so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
+                              # makes the claim survive restarts once claimed — which is what makes the PLEX_CLAIM env
+                              # above a ONE-SHOT: the token is exchanged for a PlexOnlineToken written here, and the
+                              # exchange is then skipped on every later start. (The older note here claimed a
+                              # placeholder claim Secret "would crashloop the pod"; upstream's first-run script says
+                              # otherwise — see the PLEX_CLAIM comment in `env` above. An ABSENT Secret is the hard
+                              # failure, and that is why one is committed.)
   # THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies,
   # /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`:
   # downloads/ movies/ photos/ tv/).

catalog/render.py

@@ -1844,6 +1844,10 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
     convention:
       - `<svc>.yaml`                        -> wave(svc)
       - `<svc>-bootstrap-secret.sops.yaml`  -> wave(svc)   (consumed by svc's env, same apply)
+      - `<svc>-claim-secret.sops.yaml`      -> wave(svc)   (a one-shot upstream-service claim token
+        consumed by svc's env — plex's plex.tv claim. Same wave as the app for the same reason the
+        bootstrap secret is: an env `secretKeyRef` has no `optional`, so the Secret must be applied
+        in the SAME reconcile as the workload or the pod is CreateContainerConfigError.)
       - `<svc>-oauth2-proxy-credentials.sops.yaml` -> wave(svc)  (forward-auth sidecar OIDC creds)
       - `<svc>-db.yaml`                     -> wave(svc-db) (a strictly-lower wave; CNPG DB)
       - `<svc>-db-app.sops.yaml`            -> wave(svc-db) (CNPG initdb owner secret)
@@ -1870,6 +1874,11 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
         w = waves[name]
         add(w, f"{name}.yaml")
         add(w, f"{name}-bootstrap-secret.sops.yaml")
+        # A one-shot claim token for an UPSTREAM service (plex's plex.tv claim). Same wave as the
+        # app: it is read by the app's env at pod start, and an env secretKeyRef has no `optional`,
+        # so a Secret applied in a later wave would leave the pod in CreateContainerConfigError.
+        # Placed only when the file exists, so services without one are unaffected.
+        add(w, f"{name}-claim-secret.sops.yaml")
         # forward-auth (Option B): the per-app oauth2-proxy sidecar's OIDC client + cookie secret.
         # Same wave as the app (the sidecar reads it at pod start); placed only when the file exists,
         # so non-forward-auth apps are unaffected. Mirrors the bootstrap-secret naming convention.
@@ -2548,7 +2557,8 @@ def _selftest() -> int:
     # 7a) plan_apps_waves: sidecars inherit their unit's wave; not-onboarded SDs are skipped;
     #     EXTRA_UNITS anchor strictly above; a file absent from the tree is never placed.
     present = {
-        "a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
+        "a.yaml", "a-claim-secret.sops.yaml",
+        "svc.yaml", "svc-bootstrap-secret.sops.yaml",
         "svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
         "svc-db-backup-credentials-prod.sops.yaml",
         "gitea.yaml",
@@ -2559,6 +2569,10 @@ def _selftest() -> int:
     plan = plan_apps_waves(plan_sds, present)
     flat = {f: wv for wv, fs in plan.items() for f in fs}
     check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
+    check(flat.get("a-claim-secret.sops.yaml") == 0,
+          "a `<svc>-claim-secret.sops.yaml` (plex's plex.tv claim) must be placed in the SAME wave "
+          "as its app — an env secretKeyRef has no `optional`, so a later wave means "
+          "CreateContainerConfigError")
     check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
           and flat.get("svc-db-backup-credentials.sops.yaml") == 0
           and flat.get("svc-db-backup-credentials-prod.sops.yaml") == 0,

runbooks/README.md

@@ -6,7 +6,7 @@ Every alert rule in this platform carries an annotation of the form
 
     runbook_url: "https://${BASE_DOMAIN}/runbooks/<name>"
 
-46 of them, over the 35 names below. Until this directory existed, that path returned **HTTP 404** on the
+46 of them, over 35 names. Until this directory existed, that path returned **HTTP 404** on the
 live cluster: every alert that reached a human handed them a dead link at the exact moment they needed
 help. These pages are what makes the link resolve.
 
@@ -108,6 +108,11 @@ Each affected runbook says so on its own page and gives the by-hand equivalent.
 - **`keycloak-client-secrets`** — the client/URL reconciler failing or silent. The 2026-09-06 lockout that
   every green signal missed.
 
+### Operator procedures (no alert links here)
+- **`plex-claim`** — claiming the Plex Media Server. The plex.tv claim token lives ~4 minutes, so this
+  is a timed sequence: token → `sops` → push → **forced** `flux reconcile` → pod roll. Includes why a
+  changed Secret does not restart anything, and why an expired token is harmless.
+
 ### Virtualisation
 - **`kubevirt-control-plane`** — VM lifecycle operations will fail; running VMs keep running.
 - **`kubevirt-launch-failed`** — a VMI reached the Failed phase. The three k3s-specific launch traps.

runbooks/kustomization.yaml

@@ -81,6 +81,11 @@ configMapGenerator:
       - node-io=node-io.md
       - node-memory=node-memory.md
       - node-memory-swap=node-memory-swap.md
+      # Operator procedure, not an alert page (no alert links to it — the
+      # alert-runbooks-resolvable check treats that as advisory). It is served
+      # because the 4-minute plex.tv claim window is not a thing to go looking
+      # for in git: /runbooks/plex-claim has to resolve the moment it is needed.
+      - plex-claim=plex-claim.md
       - pvc-capacity=pvc-capacity.md
       - targets-down=targets-down.md
       - update-rollback=update-rollback.md

runbooks/plex-claim.md

@@ -0,0 +1,185 @@
+# plex-claim — claiming the Plex Media Server inside the 4-minute token window
+
+**Alert:** none. This is an **operator procedure**, not an incident page — it is indexed here because
+it is the page you need at the moment you need it, with a stopwatch running.
+**Files:** `catalog/plex.yaml` (env `PLEX_CLAIM`), `apps/wave-0/plex-claim-secret.sops.yaml`
+(the SOPS Secret `plex-claim`, key `PLEX_CLAIM`), `apps/wave-0/plex.yaml` (the rendered golden).
+
+## What this means here
+
+Plex will not let you **create libraries** over the mounted bulk media share while the server is
+**unclaimed** — the 8.33 T NFS tree at `/media` is there, readable, and unusable. Claiming binds the
+server to a plex.tv account. It is a **one-shot** operation: the token is exchanged once for a durable
+`PlexOnlineToken` written into `Preferences.xml` on the `plex-config` Longhorn PVC, and from then on
+the `PLEX_CLAIM` env is **ignored on every start** (upstream gate:
+`[ ! -z "${PLEX_CLAIM}" ] && [ -z "${token}" ]` in `root/etc/cont-init.d/40-plex-first-run`).
+
+**A `https://plex.tv/claim` token lives ~4 minutes.** That is shorter than commit → push → Flux's
+10-minute reconcile interval → pod roll if you let those run on their own timers, which is why this
+page exists: every step below is written to be run back-to-back, with the reconcile **forced** rather
+than waited for. Budget: token in hand at T+0, pod restarted by roughly T+2:00.
+
+**An expired or wrong token does not break anything.** Verified against the image's own first-run
+script (and the copy inside `plexinc/pms-docker:latest`): a failed exchange gets a 403 with an empty
+body, `curl`'s status is never checked, the token `sed` yields empty, the `if` does not fire, there is
+no `else` and no `exit 1`, and the script (no `set -e`) exits 0. The pod comes up **unclaimed and
+serving**. So a burnt token costs you one more attempt, not an outage — and the repo deliberately
+ships a **placeholder** in that Secret for exactly this reason (the env `secretKeyRef` has no
+`optional`, so the Secret must always exist even when its value is useless).
+
+## Running the commands in this runbook
+
+`kubectl` does not work on the operator workstation. The toolchain lives only in the pinned dev
+container, and the host's `k3s-oidc` context hangs on a browser prompt once its token expires — that
+hang is not the cluster being down. Every command below runs as:
+
+    infra/devcontainer/dev.sh run bash -c 'kubectl --context k3s-prod get nodes'
+
+Always pass `--context k3s-prod` explicitly. `dev.sh` merges BOTH clusters into one kubeconfig and
+picks a current-context per invocation, and that default has already changed once (k3s-dev until dev
+was decommissioned on 2026-09-13, k3s-prod now). Do not run two `dev.sh` invocations at once.
+
+Mutating `kubectl` and `flux reconcile` are blocked for agents here. **This whole page is operator
+work** — the reconcile and the pod roll below are deliberate operator actions, and the durable change
+is still a git change.
+
+## First commands
+
+**Before spending a token, find out whether the server is already claimed.** This prints only the
+attribute NAME, never its value — do not widen it to print the token:
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex exec deploy/plex -- sh -c \
+     "grep -o PlexOnlineToken= \"/config/Library/Application Support/Plex Media Server/Preferences.xml\" | head -1"'
+```
+
+`PlexOnlineToken=` printed ⇒ **already claimed**; stop, there is nothing to do and no token to burn.
+Empty output ⇒ unclaimed, continue.
+
+Is the workload healthy enough to claim at all, and is the wave reconciling?
+
+```
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n plex get deploy,pod,pvc'
+infra/devcontainer/dev.sh run bash -c \
+  'kubectl --context k3s-prod -n flux-system get kustomization apps-wave-0'
+```
+
+## The 4-minute procedure
+
+Have this page open and do not stop between steps.
+
+1. **T+0 — get the token.** Sign in to plex.tv as the account that should OWN this server, open
+   <https://plex.tv/claim>, copy the `claim-…` string. The clock starts now.
+
+2. **T+0:15 — put it in the Secret**, in place, encrypted, without it ever touching a plaintext file:
+
+   ```
+   infra/devcontainer/dev.sh run sops apps/wave-0/plex-claim-secret.sops.yaml
+   ```
+
+   Replace the `PLEX_CLAIM:` value (the placeholder `replace-me-with-a-fresh-plex-tv-claim-token`, or
+   a previous expired token) and save. `sops` re-encrypts on write to the committed age recipient.
+
+3. **T+0:45 — commit and push.** `git push origin main` **is** the deploy; there is no other apply
+   step.
+
+   ```
+   git add apps/wave-0/plex-claim-secret.sops.yaml
+   git commit -m 'chore(plex): fresh plex.tv claim token'
+   git push origin main
+   ```
+
+4. **T+1:00 — force the reconcile immediately. This is the one command that saves the window** — it
+   pulls the new commit and re-applies wave 0 instead of waiting up to 10 minutes for the interval:
+
+   ```
+   infra/devcontainer/dev.sh run bash -c \
+     'flux --context k3s-prod -n flux-system reconcile kustomization apps-wave-0 --with-source'
+   ```
+
+   `--with-source` is the load-bearing flag: it reconciles the `flux-system` GitRepository first, so
+   the Kustomization applies **your commit** and not the previously-fetched revision.
+
+5. **T+1:30 — roll the pod.** This step is **mandatory and easy to forget**: a changed Secret does
+   **not** restart anything. `PLEX_CLAIM` is injected by `secretKeyRef` and read only at container
+   start, and the Deployment's pod template is byte-identical across this change, so Flux applies the
+   new Secret and the running pod keeps the old value forever.
+
+   ```
+   infra/devcontainer/dev.sh run bash -c \
+     'kubectl --context k3s-prod -n plex rollout restart deployment/plex'
+   ```
+
+6. **T+2:00 — confirm the exchange happened**, in the new pod's init log:
+
+   ```
+   infra/devcontainer/dev.sh run bash -c \
+     'kubectl --context k3s-prod -n plex logs deploy/plex --since=5m | grep -i -e claim -e token'
+   ```
+
+   * `Attempting to obtain server token from claim token` **and** `Token obtained successfully`
+     ⇒ **claimed.** Re-run the First-commands `PlexOnlineToken=` check to confirm it persisted, then
+     create the libraries over `/media/movies`, `/media/tv`, `/media/photos`.
+   * `Attempting to obtain server token from claim token` with **no** success line ⇒ the token was
+     expired or invalid. Nothing is broken; go back to step 1 with a new token.
+   * **Neither line** ⇒ the exchange was skipped, which means `/config` already had a
+     `PlexOnlineToken` (already claimed) or the env did not reach the container. Check the pod actually
+     has the variable, by NAME only:
+
+     ```
+     infra/devcontainer/dev.sh run bash -c \
+       'kubectl --context k3s-prod -n plex get pod -l app.kubernetes.io/name=plex \
+          -o jsonpath="{.items[0].spec.containers[0].env[*].name}"'
+     ```
+
+## Likely causes, most likely first
+
+When the claim does not take, in order of how often it is the answer here:
+
+1. **The token expired** — you spent more than ~4 minutes, usually in step 3 or waiting on step 4.
+   Symptom: the "Attempting…" line with no success line. Remedy: a new token, and force the reconcile
+   rather than waiting for it.
+2. **The pod was never restarted** (step 5 skipped). Symptom: no claim log lines at all from a pod
+   whose `AGE` predates your push. The Secret is correct in git and in the cluster and the running
+   container still holds the placeholder.
+3. **Flux had not applied the commit yet** when you rolled the pod — step 5 run before step 4
+   finished, so the new pod read the OLD Secret. Check
+   `kubectl --context k3s-prod -n flux-system get kustomization apps-wave-0 -o jsonpath='{.status.lastAppliedRevision}'`
+   against your commit, then roll again.
+4. **Already claimed** — the exchange is gated on `PlexOnlineToken` being absent from `/config`, so a
+   server claimed earlier ignores every later token silently. This is the benign case.
+5. **`plex.tv` unreachable.** The upstream `curl` has no `--max-time`/`--connect-timeout`, so a stalled
+   connection can leave the container sitting in cont-init with **nothing listening on 32400** — the
+   one genuine hang risk in this path. Symptom: pod Running but never Ready, log stops after
+   "Attempting…". Check egress (`plex` is `egress: internet`) rather than re-rolling repeatedly.
+6. **Wrong plex.tv account.** The claim succeeds and binds the server to whoever was signed in at
+   step 1. Unclaiming is a plex.tv-side operation on that account — see "escalate or stop".
+
+## When to escalate or stop
+
+- **Stop after two failed token attempts** and fix the timing, not the token. The third rushed attempt
+  is how people start editing live Secrets by hand, which leaves the cluster holding a value that is
+  in no git commit — precisely the drift `qbittorrent-bootstrap` had to be recovered from.
+- **Never `kubectl edit`/`patch` the `plex-claim` Secret in the cluster, and never
+  `kubectl create secret` it.** Everything durable goes through the SOPS file; a live-only Secret is
+  reverted (or adopted with the wrong value) on the next reconcile.
+- **Never `sops -d` this file into a terminal, a log, or a shared screen**, and never widen the
+  `PlexOnlineToken=` check into one that prints the value. The token is a plex.tv account credential.
+- **If the server got claimed to the wrong account, stop and get the operator.** Reversing it means
+  removing the server from that account on plex.tv and clearing `PlexOnlineToken` from
+  `Preferences.xml` on the PVC — a destructive edit of app state on the replicated tier, not a git
+  change.
+- **Do not delete the `plex-config` PVC to "start clean".** It holds the library database, metadata
+  and watch state (`policy: daily-offcluster`), and it is expensive to rebuild. The claim is a
+  two-line pref, not a reason to lose the DB.
+- After a successful claim, **leave the expired token in the SOPS file**. It is inert (the exchange is
+  skipped once claimed) and the Secret must keep existing, because the env `secretKeyRef` has no
+  `optional` — deleting it would put the pod into `CreateContainerConfigError`.
+
+---
+Source of truth: `runbooks/plex-claim.md`. Index: `/runbooks/`
+Upstream behaviour cited here is from `plexinc/pms-docker` branch `master`,
+`root/etc/cont-init.d/40-plex-first-run` (s6-overlay v2), cross-checked against the same file inside
+the running `plexinc/pms-docker:latest` image.