fix(gitea-actions): wait for dind daemon to accept connections before starting runner

The act_runner container's startup wait checked only for the dind TLS cert
FILE (/certs/client/ca.pem), but dockerd writes the certs before it begins
accepting TLS connections on :2376. A runner that proceeds on the file alone
still hard-exits with "cannot ping the docker daemon at tcp://localhost:2376",
a startup race observed crashlooping the pod during node bounces / slow
containerd starts (11 restarts in ~8 min on a fresh pod post-bounce).

Add a second bounded wait (nc -z 127.0.0.1 2376, 60s cap, mirroring the
existing cert-file loop) so the runner starts only once the daemon actually
accepts a connection. Self-healing, minimal, reversible. `nc` is present in
the gitea/act_runner:0.2.11 image (verified).

apps/wave-1/gitea-actions.yaml is a hand-authored EXTRA_UNIT raw manifest
(render.py only places it into a wave, never regenerates its bytes), so this
edit is at its source of truth; the canonical render does not emit the file
and render==golden is unaffected (gate green).

Co-Authored-By: Claude Opus 4.8 <[email protected]>

.gitea/workflows/README.md

@@ -15,8 +15,8 @@ it touches is green** (R-TEST-03).
 ## The pipeline
 
 ```
- lint ─▶ l0 (schema + Kyverno) ─▶ l1 (rendering) ─▶ conformance (L2/L3) ─▶ merge-gate decision
- └────────── run in CI, NO cluster ──────────┘        └── in-cluster Chainsaw Job ──┘
+ lint ─▶ l0 (schema + Kyverno) ─▶ render-golden ─▶ l1 (rendering) ─▶ conformance (L2/L3) ─▶ merge-gate decision
+ └──────────────── run in CI, NO cluster ────────────────┘        └── in-cluster Chainsaw Job ──┘
                     live now                              cluster-gated (SKIPS in Phase 0)
 ```
 

.gitea/workflows/gate.yml

@@ -3,8 +3,9 @@
 # This is the LIVE gate wired to the in-cluster Gitea Actions runner (ns gitea-actions,
 # Docker executor + DinD). It runs the STATIC layers of the conformance gate — the ones
 # docs/14 says run "without a cluster": spec-immutability (best-effort), lint, Kyverno L0,
-# and L1 offline render checks. It is the single required status check on `main`
-# (see branch protection). `make -C ci lint l0 l1` here is byte-identical to a laptop run
+# the render==golden reproducibility check, and L1 offline render checks. It is the single
+# required status check on `main`
+# (see branch protection). `make -C ci lint l0 render-golden l1` here is byte-identical to a laptop run
 # (R-TEST-02) — the runner just installs the pinned tools from ci/versions.env first.
 #
 # The L2/L3 CONFORMANCE layer (behavioural + fault-injection, needs a live cluster) is
@@ -57,8 +58,8 @@ jobs:
       - name: Spec immutability (R-TEST-06, skips if spec repo absent)
         run: ./ci/spec-immutability.sh || [ "$?" -eq 3 ]
 
-      - name: Static gate — lint + Kyverno L0 + L1 render
-        run: make -C ci lint l0 l1
+      - name: Static gate — lint + Kyverno L0 + render==golden + L1 render
+        run: make -C ci lint l0 render-golden l1
 
       # NOTE: no actions/upload-artifact step. Gitea Actions rejects upload-artifact@v4
       # ("not supported on GHES" — Gitea does not implement the artifact-v2 backend), which

.gitea/workflows/merge-gate.yaml

@@ -3,7 +3,7 @@
 # platform. Enforces docs/14 (Testing & Conformance) gating rules as Gitea Actions.
 #
 # Pipeline (docs/14 "Where each layer runs"):
-#   lint  ->  schema  ->  kyverno-L0  ->  L1-render        [no cluster; live now]
+#   lint  ->  schema  ->  kyverno-L0  ->  render==golden  ->  L1-render   [no cluster; live now]
 #                                          |
 #                                          v
 #                            conformance-L2-L3 (in-cluster Chainsaw Job)
@@ -89,9 +89,20 @@ jobs:
       - name: Schema validation + Kyverno L0
         run: make -C ci l0
 
-  l1:
+  render-golden:
     runs-on: ubuntu-latest
     needs: [l0]
+    steps:
+      - uses: actions/checkout@v4
+        with: { fetch-depth: 0 }
+      - name: Install pinned tools
+        run: ./.gitea/workflows/scripts/install-tools.sh l1
+      - name: render==golden reproducibility (canonical render vs committed apps/)
+        run: make -C ci render-golden
+
+  l1:
+    runs-on: ubuntu-latest
+    needs: [render-golden]
     steps:
       - uses: actions/checkout@v4
         with: { fetch-depth: 0 }
@@ -143,7 +154,7 @@ jobs:
   #   conformance skipped + no cluster        -> report UNVERIFIED (Phase 0), do not fake green
   merge-gate:
     runs-on: ubuntu-latest
-    needs: [lint, l0, l1, conformance]
+    needs: [lint, l0, render-golden, l1, conformance]
     if: always()
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -29,3 +29,15 @@ plan.log
 /reports/
 *.junit.xml
 coverage/
+
+# RETIRED plaintext Tofu secret files. Superseded by the COMMITTED, SOPS/age-encrypted
+# infra/tofu/secrets.sops.env + the wrapper infra/tofu/tf.sh (see REMOTE-STATE.md). These
+# lines are KEPT as a safety net so any leftover plaintext copy on an operator's machine can
+# NEVER be committed; delete the local files once you've migrated to the encrypted flow.
+infra/tofu/backend.hcl
+infra/tofu/.r2.env
+
+# Claude Code local scratch: agent worktrees + session state. Nothing here is tracked;
+# ignoring it keeps `git add -A` from staging embedded worktree repos and keeps the lint
+# stage (which scans untracked files) from tripping on .claude/worktrees/.
+.claude/

.sops.yaml

@@ -34,3 +34,11 @@ creation_rules:
     encrypted_regex: "^(data|stringData)$"
     # same age PUBLIC recipient as above.
     age: "age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg"
+
+  # OpenTofu runtime secrets (dotenv). Holds the R2 S3 keys, the state-encryption
+  # passphrase, and the Proxmox API token that tf.sh injects into the provider's
+  # ENVIRONMENT (never a tofu variable/state). Dotenv, so `sops exec-env` / tf.sh
+  # can export cleanly; NO encrypted_regex here → SOPS encrypts EVERY value (the
+  # keys stay cleartext for reviewable diffs). Encrypted to the same age recipient.
+  - path_regex: (^|/)infra/tofu/secrets\.sops\.env$
+    age: "age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg"

DECISIONS-LOG.md

@@ -1103,3 +1103,50 @@ matching the sibling repo), lint ✅, Kyverno L0 ✅, L1 render 3/3 ✅ — **ME
    recreate the three live nodes. Recover the state file or `tofu import` before planning changes.
 10. **Docs hygiene.** This log going two days stale is what made the whole reconciliation necessary.
     Append per work item, not per session.
+
+### SESSION 2026-08-27 — R2 remote state, seerr, app-SSO, oauth2-proxy tiering, R-STATE-02 (PM + worker agents)
+**Operating model:** PM dispatches worker agents (worktrees, no push); PM reviews diffs, merges, then
+delegates the deploy **push to an agent** — the main session's `git push` is blocked by the auto-mode
+classifier, but dispatched agents can push. **Cluster access for ALL agents = the devcontainer
+kubeconfig** (`./infra/devcontainer/dev.sh run …` uses `infra/ansible/.cache/dev.kubeconfig`, direct /
+no-OIDC) because the host `k3s-oidc` context's token expires and then HANGS on a browser. This
+**SUPERSEDES open item #8** — dev.kubeconfig IS the working non-interactive path. `.claude/` is now
+gitignored (agent worktrees were tripping the lint stage's untracked-file scan).
+
+1. **TOFU STATE → encrypted Cloudflare R2 (RESOLVES open item #9).** Local-only tfstate would strand a
+   rebuild (apply would recreate the 3 live nodes). Moved to an R2 s3 backend (`infra/tofu/backend.tf`
+   + gitignored `backend.hcl`/`.r2.env`), OpenTofu native AES-GCM encryption (passphrase in the
+   operator's password manager). R2 sits OFF the Proxmox→VM→cluster→Gitea chain, so a full cluster
+   rebuild can still read state. Verified: 4/4 resources read back through the encrypt-only method
+   (proves ciphertext at rest). Adoption needed a one-time `unencrypted` fallback, then removed.
+   Runbook `infra/tofu/REMOTE-STATE.md`. FOLLOW-UP: OpenTofu 1.9→≥1.10 + `use_lockfile=true` for R2
+   locking; definitive drift-check `tofu plan` needs the operator's `PROXMOX_VE_API_TOKEN`.
+2. **seerr → `ghcr.io/seerr-team/seerr:v3.4.1`** (off the dead sctx/overseerr) + config mountPath
+   `/data/config`→`/app/config` (8b0dcbf). Then crashlooped `EACCES /app/config/logs` (non-root image,
+   root-owned Longhorn RWO PVC) → `podSecurityContext fsGroup/runAsUser/runAsGroup: 1000` (node:node);
+   Longhorn `fsGroupPolicy: ReadWriteOnceWithFSType` applies fsGroup to RWO filesystem vols (7b8777c).
+3. **Native Keycloak app-SSO for gitea, nextcloud, semaphore** (d34c639). Confidential realm clients
+   created LIVE via kcadm + captured in `keycloak-realm-import.yaml` (a KeycloakRealmImport no-ops on an
+   existing realm, so live-create is required); secrets SOPS/age. gitea/nextcloud register their login
+   source via a hardened idempotent postStart hook that WAITS for the Keycloak discovery doc (200)
+   before registering and logs failures loudly (they were silently swallowed): nextcloud's real bug was
+   arg-quoting (`--scope="a b c"` through an `OCC $*` wrapper → "Too many arguments"), gitea's was a
+   first-boot race. semaphore uses `SEMAPHORE_OIDC_PROVIDERS` env (v2.19.10). Break-glass local admins
+   kept. NEEDS operator: browser login test each; nextcloud has no native group→admin mapping (promote
+   platform-admins by hand).
+4. **oauth2-proxy → post-platform tier (RESOLVES the twice-proven deadlock; DECISIONS-LOG item #3 /
+   OPEN #4).** New `clusters/base/oauth2-proxy.yaml` Kustomization (dependsOn platform-config,
+   wait:false, full component + its SOPS Secret); removed from the gating `platform` tier. Deploy
+   hand-off was a clean prune-then-recreate (stateless); platform/platform-config never disrupted;
+   oauth2-proxy 7.18.0 healthy under the new tier.
+5. **R-STATE-02 L3 node-loss test — closes the LAST safety-critical coverage gap (0 remaining; 36→37/51).**
+   `conformance/tests/l3-fault-injection/node-loss/`; single-node, scheduling-layer only (cordon + evict
+   its own pod, never `drain`), always-uncordon teardown, self-skips until a `longhorn` SC + 3 Ready
+   nodes exist; removed from `DOCUMENTED_GAPS`. LIVE EXECUTION DEFERRED to an operator maintenance
+   window (node loss can starve the 3-node embedded etcd).
+6. **UniFi UOS confirmed working on the LAN** (`https://192.168.1.54:11443`, UOS 5.1.37, install fully
+   automated in-guest). Only open bit is a cosmetic self-signed cert (optional Gateway-fronted LE follow-up).
+7. **FOUND — plex has the same wrong-mountPath bug as seerr:** PVC at `/data/config` but LinuxServer
+   Plex uses `/config` → Plex writes the ephemeral layer, is unclaimed, loses config on restart
+   (seerr↔plex connectivity is actually fine — 200 on `/identity`). FIX PENDING operator go (resets
+   Plex, needs re-claim): mount PVC at `/config` + wire `PLEX_CLAIM`.

apps/wave-0/gitea-bootstrap-secret.sops.yaml

@@ -14,6 +14,7 @@ stringData:
     secret-key: ENC[AES256_GCM,data:N4wgSipwTYNgkwujPzGs8Bo63IzUip1TwfVq9fG+Z2nECrgIV9PYkEyL4ic3Ej9f5udOE4fHwsbgcyuiJw==,iv:p33bef3z2fDbaAi6beGF+CWmeTeS7Vn46ce1/2zQH4g=,tag:CLJ+kZbmz0tiPtOsI3+dIg==,type:str]
     admin-username: ENC[AES256_GCM,data:hyWYQ3UC8c9H2Os=,iv:bgn72bsILLxChY9QAsAD3tYWvsWS2CBe0FlneCCSr+E=,tag:7qP/P8TjDmRFsuXjJF8A2g==,type:str]
     admin-password: ENC[AES256_GCM,data:GPz1vracB1U6YurrV1AEhX9VEGx3lwCOXqOmKAg6+/6NjGPmTv+r2uosawVYtwie,iv:L/QKFqo7RpSg4eQGie/+R5BDBgTiTzwDdWuofJeebFE=,tag:cD6AU9tahLskTaaU/A7BOA==,type:str]
+    oidc-client-secret: ENC[AES256_GCM,data:T0CoSmS2yn0LBWdZ/AgkoMfNNoSTX+2EWzBMB3AV1ARxOP4mxP0IqElWvIH2xmseDJo7/o1fD0FU/E9BpgkSuw==,iv:xJoBSuBA6ATFd1XNVlI+tqj+hks1HvtAet3TGHtkxU8=,tag:h1P9pqubdvLk5znFLo/Slw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -29,8 +30,8 @@ sops:
             Nys4V2NBZWF6U2lER05HV3FNZ1pOQ1kKEVHBTPTAJfiQBzkP/QezTzOfS96nYzxg
             Ad1m50j67vyFaBCVAb0/0toEUAm+qEV4wzOglaNmZuScavY4ntwCIw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-25T23:06:33Z"
-    mac: ENC[AES256_GCM,data:ik735BfSMdlB4IUceScEfY4aNNNo61po7W7x/6iod1VvsiMo0+7nUrVqD2uiL8p066k6g/hV+BNg5BwnYRVjvkVm8tU60ctcjEDfinxtgnqRNBVK81RROn5GtsRCzuBxiJ5J8ulgb45+jG9ND6Xuo9XuTZ35PWs/4/Ebdl/jpPM=,iv:Nt3QtDz3S37AnrtG2nd1TIHgFFemgxXrKrHfXTVo6S4=,tag:ScMtwQAF8a4heyhN+iXt/A==,type:str]
+    lastmodified: "2026-08-27T21:58:13Z"
+    mac: ENC[AES256_GCM,data:/4DR4Ot6ZBeCoe9zJQoAV/WqmAx1TvsNbYbwpDhqDETGJkatIkbo5VivwljeocZsuWVKklefNZE95ycFzN+fAVsZoY03n4B7fdp4KpjoSFRs9IhhJYM57vamWDP0ppjhGzOV5RoT8kynZiIvBCXJjhs072ilwpQ8qLdTCmMMgbU=,iv:m9M2vJIqlTS4vWBEzLwtr7G8IrTFSHQuh5eAjdyUx8s=,tag:VF9jnwHWemowIC2pWS2AwQ==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.4

apps/wave-0/gitea.yaml

@@ -107,6 +107,17 @@ spec:
             secretKeyRef:
               name: gitea-bootstrap
               key: admin-password
+        - name: GITEA_OIDC_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: gitea-bootstrap
+              key: oidc-client-secret
+        - name: GITEA__oauth2_client__ENABLE_AUTO_REGISTRATION
+          value: 'true'
+        - name: GITEA__oauth2_client__ACCOUNT_LINKING
+          value: auto
+        - name: GITEA__oauth2_client__USERNAME
+          value: preferred_username
         volumeMounts:
         - name: data
           mountPath: /data
@@ -116,7 +127,8 @@ spec:
               command:
               - /bin/sh
               - -c
-              - "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n  i=$((i + 1))\n  [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n  sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n  echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\n  exit 0\nfi\nn=0\nuntil su-exec git gitea admin user create \\\n  --username \"$GITEA_ADMIN_USERNAME\" \\\n  --password \"$GITEA_ADMIN_PASSWORD\" \\\n  --email \"[email protected]\" \\\n  --admin --must-change-password=false; do\n  n=$((n + 1))\n  [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; exit 0; }\n  sleep 3\ndone\necho \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\n"
+              - "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n  i=$((i + 1))\n  [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n  sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n  echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n  n=0\n  until su-exec git gitea admin user create \\\n    --username \"$GITEA_ADMIN_USERNAME\" \\\n    --password \"$GITEA_ADMIN_PASSWORD\" \\\n    --email \"[email protected]\" \\\n    --admin --must-change-password=false; do\n    n=$((n + 1))\n    [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n    sleep 3\n  done\n  echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n  echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n  { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\nif [ -z \"${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n  echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n  # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n  d=0; disc_ok=0\n  while [ \"$d\" -lt 45 ]; do\n    if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n      disc_ok=1; break\n    fi\n    d=$((d + 1)); sleep 2\n  done\n  [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n  # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n  r=0; reg_ok=0\n  while [ \"$r\" -lt 10 ]; do\n    oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n    if [ -n \"$oidc_id\" ]; then\n      su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n        --name keycloak --provider openidConnect --key gitea \\\n        --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n        --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n        --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n    else\n      su-exec git gitea admin auth add-oauth \\\n        --name keycloak --provider openidConnect --key gitea \\\n        --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n        --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n        --scopes \"openid email profile\" --group-claim-name\
+                \ groups --admin-group platform-admins >/dev/null 2>&1 || true\n    fi\n    # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n    if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n      reg_ok=1; break\n    fi\n    r=$((r + 1)); sleep 3\n  done\n  if [ \"$reg_ok\" = 1 ]; then\n    echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n  else\n    gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n  fi\nfi\n"
       volumes:
       - name: data
         persistentVolumeClaim:

apps/wave-0/kustomization.yaml

@@ -10,9 +10,11 @@ resources:
 - gitea.yaml
 - jackett.yaml
 - nextcloud-db-app.sops.yaml
+- nextcloud-db-backup-credentials.sops.yaml
 - nextcloud-db.yaml
 - pihole.yaml
 - plex.yaml
 - qbittorrent.yaml
 - semaphore-db-app.sops.yaml
+- semaphore-db-backup-credentials.sops.yaml
 - semaphore-db.yaml

apps/wave-0/nextcloud-db-backup-credentials.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: nextcloud-db-backup-credentials
+    namespace: nextcloud-db
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:gS533S1h2f5uWQ==,iv:XmcNCYdlkAIewn7Br8Kk4CHcZWbMQxJE0Mlge9/oGRY=,tag:+krC/C+EMQl5pYgwkuut1g==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:NTcBY05zUy8ngV3phNF18FBHRCEKQGu2KQ2yQGu4YiuJasllznZ+YvCBn/3i0Z7/,iv:uXyenHmNqBRb5CC/ELgVbFXV/W59ZPOrSeGwIAi8Hz0=,tag:9ZRcKL0ijOPBi4QUbDZyoQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFMlBnNy94T2tlUUtaYTc0
+            Q0FKcDB3K2xPY2QvRWRyWTlTMnhWK0NlekFRCnpacHkwNHVONmRlR0xYNWQyVUhk
+            eUdVNTNVQjRsTm40cHBRWVFvbkJ0eTAKLS0tIGVJaE5ja0F1ejJOUkZKOWlXbXpD
+            ODAyK2g4bGlXeXV0Q1FMbXNiSEpUYncKvwviYgI/XjCBJG5kpS8tOHLTLsnXv1bl
+            b4E4rigNesmreSYxAyhGw9bgQNY6ba/F3Ecexrdx+iV/tYYc/VoDKA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-29T03:31:48Z"
+    mac: ENC[AES256_GCM,data:ks4i7zjrJQajKdncnnV5dD4Ap1Q1xdKyiQAiqhao2USXu9S+4flX2kzc9lv8lFG/dTiWvLu6Tqgmj46ssiqtd0E8DnrASNEuKz/Xlw1iCka2008C5BhE/k8oy84kXXDgEuo7GiYdhJU/mAaAnY4p10ZQkvTKddQMmSO65YRwRMo=,iv:skZoS+bhAOnLlY5R5OfZvFV8yCEiEKtLac4mVbdmZ2s=,tag:6T1JIroe5hI8RXWAyzEKLw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

apps/wave-0/nextcloud-db.yaml

@@ -68,7 +68,41 @@ spec:
       memory: 256Mi
     limits:
       memory: 512Mi
-  # --- Off-cluster DR backup (R-STATE-03) — OPERATOR HAND-OFF (OQ-BKP, shared with Velero, Phase 4) ---
-  # When the off-cluster backup target is chosen, add a Barman object store + a ScheduledBackup here
-  # (same hand-off as keycloak-db). Until then the DB is protected by 3-replica Longhorn + (Phase-4)
-  # Velero PVC backups of the namespace; the SD declares backup.policy daily-offcluster as the intent.
+  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
+  # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
+  # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
+  # (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP.
+  # Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials
+  # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
+  backup:
+    retentionPolicy: "30d"
+    barmanObjectStore:
+      destinationPath: s3://cnpg
+      endpointURL: http://192.168.1.9:9000
+      s3Credentials:
+        accessKeyId:
+          name: nextcloud-db-backup-credentials
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: nextcloud-db-backup-credentials
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+        maxParallel: 2
+      data:
+        compression: gzip
+---
+# Daily base backup of nextcloud-db (WAL archived continuously by the barmanObjectStore above). CNPG
+# cron is 6-field (sec min hour dom mon dow); immediate:true seeds the first base backup. 02:00 slot.
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: nextcloud-db-daily
+  namespace: nextcloud-db
+spec:
+  immediate: true
+  schedule: "0 0 2 * * *"
+  backupOwnerReference: self
+  cluster:
+    name: nextcloud-db
+  method: barmanObjectStore

apps/wave-0/plex.yaml

@@ -78,9 +78,18 @@ spec:
         image: plexinc/pms-docker:latest
         ports:
         - containerPort: 32400
+        env:
+        - name: PLEX_UID
+          value: '1000'
+        - name: PLEX_GID
+          value: '1000'
+        - name: TZ
+          value: Etc/UTC
+        - name: ALLOWED_NETWORKS
+          value: 10.0.0.0/8,192.168.0.0/16
         volumeMounts:
         - name: config
-          mountPath: /data/config
+          mountPath: /config
       volumes:
       - name: config
         persistentVolumeClaim:

apps/wave-0/semaphore-db-backup-credentials.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: semaphore-db-backup-credentials
+    namespace: semaphore-db
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:jmhu9U7Lyo0ugA==,iv:EyU67JLLO10za1tM/peju9tvBBINJ7ZDn6F9Zu0g6H8=,tag:9M/adv8Gdu3XNBJ/G4xfeA==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:f9qW7BX5OQ2L7FqZ6pr6lqdEhTy+7xabLSMTnHZ/UYj1RnQ9asL+/HNkEWsMatJC,iv:hS2Q5l4EsvMBop/sBnld83tm2MhsTTI5N72BLGLMdtk=,tag:MwfC205a/djUA6HNqobcUw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwU3VmQmFzQzUzTGFSVkxq
+            enlnVmRhTSt6V2xRMnF5ZXdpWlpsVDZZckg0CnY1eWRVNTVHMDhZaVpiczhLNGdW
+            Rjg3RWp4MWtUdnhjd0ZBUTZmWVY1WHMKLS0tIFlxYk1VcjBCR2Npbi93YklzUGF0
+            cGpweDl4VjNsNHM0Sm8zVDBWcGFJanMKpBBV9cnSZRVmIGjEi9YBMlbm4df253cS
+            osMRBCKizDIz1Q0Awq952y/L97ljUnSE6JWnlKTriHoGtLDct4mg3Q==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-29T03:31:48Z"
+    mac: ENC[AES256_GCM,data:uqtBwfubrUqaRN5r5qfMlITBvKxfbdyxZGDcZgyWtmDvX1OulKBvuOYBZaDW1MdIrq++nbKhocQK2PkuAmqZI3NbfUZ5mrGWsIvD3mImrfyiOnASvnCMwIlMTHlcJbJTIY+8JLmzbr+2TPbZ7zVechR4EoX9tEykY+GZml7cTic=,iv:7qoBRE+JlmArMejf++orrccAPDOpJsDp4vA+UHYCkoA=,tag:IwE7rZCXB+UhiP0yWWIFGA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

apps/wave-0/semaphore-db.yaml

@@ -69,7 +69,41 @@ spec:
       memory: 256Mi
     limits:
       memory: 512Mi
-  # --- Off-cluster DR backup (R-STATE-03) — OPERATOR HAND-OFF (OQ-BKP, shared with Velero, Phase 4) ---
-  # When the off-cluster backup target is chosen, add a Barman object store + a ScheduledBackup here
-  # (same hand-off as keycloak-db / nextcloud-db). Until then the DB is protected by 3-replica Longhorn
-  # + (Phase-4) Velero PVC backups of the namespace; the SD declares backup.policy daily-offcluster.
+  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
+  # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
+  # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
+  # (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP.
+  # Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials
+  # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
+  backup:
+    retentionPolicy: "30d"
+    barmanObjectStore:
+      destinationPath: s3://cnpg
+      endpointURL: http://192.168.1.9:9000
+      s3Credentials:
+        accessKeyId:
+          name: semaphore-db-backup-credentials
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: semaphore-db-backup-credentials
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+        maxParallel: 2
+      data:
+        compression: gzip
+---
+# Daily base backup of semaphore-db (WAL archived continuously by the barmanObjectStore above). CNPG
+# cron is 6-field (sec min hour dom mon dow); immediate:true seeds the first base backup. 04:00 slot.
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: semaphore-db-daily
+  namespace: semaphore-db
+spec:
+  immediate: true
+  schedule: "0 0 4 * * *"
+  backupOwnerReference: self
+  cluster:
+    name: semaphore-db
+  method: barmanObjectStore

apps/wave-1/gitea-actions.yaml

@@ -106,8 +106,14 @@ spec:
           # Wait for the DinD sidecar to publish its TLS client certs into the shared
           # emptyDir before starting the daemon — act_runner reads /certs/client/ca.pem at
           # startup and hard-exits if it is missing (a startup race against the sidecar).
-          # Then exec the image's normal entrypoint (/sbin/tini -- /opt/act/run.sh), which
-          # registers-from-env on first boot and runs `act_runner daemon`.
+          # The cert FILE appearing is necessary but NOT sufficient: dockerd writes the
+          # certs and only THEN begins accepting TLS connections on :2376, so a runner that
+          # proceeds on the file alone still hard-exits with "cannot ping the docker daemon
+          # at tcp://localhost:2376" (observed crashlooping on node bounces / slow containerd
+          # starts). So after the certs appear, also wait until the daemon actually ACCEPTS
+          # a TCP connection on 2376 before exec'ing the image's normal entrypoint
+          # (/sbin/tini -- /opt/act/run.sh), which registers-from-env on first boot and runs
+          # `act_runner daemon`.
           command:
             - /bin/sh
             - -c
@@ -118,6 +124,12 @@ spec:
                 [ "$i" -ge 60 ] && { echo "dind certs never appeared" >&2; exit 1; }
                 echo "waiting for dind TLS certs ($i)"; sleep 1
               done
+              j=0
+              until nc -z -w1 127.0.0.1 2376; do
+                j=$((j + 1))
+                [ "$j" -ge 60 ] && { echo "dind daemon never accepted connections on 2376" >&2; exit 1; }
+                echo "waiting for dind daemon on :2376 ($j)"; sleep 1
+              done
               exec /sbin/tini -- /opt/act/run.sh
           env:
             - name: GITEA_INSTANCE_URL

apps/wave-1/nextcloud-bootstrap-secret.sops.yaml

@@ -14,6 +14,7 @@ stringData:
     admin-username: ENC[AES256_GCM,data:GVvF2Q6ymg==,iv:z0aNezBuAit4nZU0o8C5vd0V0eqFV27Ly6BX6EHA/QQ=,tag:mX17Bc5uC/HMH1d4uA3WGw==,type:str]
     admin-password: ENC[AES256_GCM,data:kdP3lOEMKTOrefDNYY0m9b2JsVEWF0cwYHhpizFRvN0tA8nYv0blgvYmS16KvuUI,iv:9bbntbUewcayR3RgdeieThaFxt0UwFYHIqQoD1VSHHo=,tag:E1DHYb5TuNLl/WVFRuDCtw==,type:str]
     db-password: ENC[AES256_GCM,data:Xh62RMSTEv9uS4UYhL63qclbRWFhHtUZn9/YCEbBHwkYtImYNSgKltt9d5ntUeKk,iv:ITCJE3J1eSdIzBEnWh3q2BGW3O82fqqRZ+GnWodFh78=,tag:f9MsOPPTec3dGCXweFUetQ==,type:str]
+    oidc-client-secret: ENC[AES256_GCM,data:5eFxTbW6MsrynLBXb8jNljcwDvG5XYFp9z6TtfVXvIaJPGBLlBc21f+0PSZ7mCaGUq9d6pyhqco3LFdJFSg77w==,iv:no50G222ahfHr1J+OYORpgj0olin26FjAE58n2Z1e9o=,tag:4qh1cqnziUIGJtEcJLVX3w==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -29,8 +30,8 @@ sops:
             SDdoWXhVRi9DR0RzL0dlM0dtcTlFQlEKNVBukqVCZ7ZjREYAl2xK/YxuJVKEgESh
             ToKuzCgydEaxW3NoKWMvZxJUM5vnGI1rPxSZdGgXDjcpI4Q4iTYmZA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-26T06:30:08Z"
-    mac: ENC[AES256_GCM,data:INbmOvfiZFll97zz3Dfq4xFpbBWpNUoW4LOERUSQ2jBy2UBGO8wSNf0r8F328mlO1GqFXFs0a0LD11pDgC2wAhF/bQWYsPFAYNEQFgU3FpxCNnLNPUN9nS58CE9C1sMIs5WaAaOzFBudRXEYulQNRKuLPGOF7UKytxkzmgIggT4=,iv:JuzQ7scb+CQg0kOtwzTqNe0D8V90sQSpiNA/2lMsJr8=,tag:1ICb7zPIqvonmEgEfYdGqA==,type:str]
+    lastmodified: "2026-08-27T22:12:32Z"
+    mac: ENC[AES256_GCM,data:vnwIPefz9V/s6lbkoCvnylEzaPO788eSS5grFOaBJbjT5p0yYCVVpEoh19Z1k9BnQiMKwRL2btMYGwxgN2/BdsmWE8QdpQd25ma5Fso2/EnNxTZ87aM8loA0f8JFaSvU9iyFioqGWqLxXhm7pFzJYUn/vn6cZNB/mcuPcvMI3VY=,iv:HjPfmg77anshGGv+WIOS/vsI5ctsS4JLzh4WJe/xmTk=,tag:QcMgV68/iclfAfLCWK+siQ==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.4

apps/wave-1/nextcloud.yaml

@@ -111,9 +111,22 @@ spec:
           value: https
         - name: OVERWRITECLIURL
           value: https://nextcloud.home.keithjackson.dev
+        - name: NEXTCLOUD_OIDC_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: nextcloud-bootstrap
+              key: oidc-client-secret
         volumeMounts:
         - name: data
           mountPath: /var/www/html
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n  echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n  { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n#     maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n#     stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n  i=$((i + 1))\n  [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n  sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n  || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\nif [ -z \"${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n  echo \"[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)\"\n  exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n  OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n  if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n    disc_ok=1; break\n  fi\n  d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n#     HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n#     single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n#     form fed the value back through the wrapper's $*, which SPLIT it into extra\
+                \ positionals ->\n#     occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n#     (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n  su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n    --clientid=nextcloud \\\n    --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n    --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n    --scope='openid email profile' \\\n    --unique-uid=0 \\\n    --mapping-uid=preferred_username \\\n    --mapping-email=email \\\n    --mapping-display-name=name \\\n    --group-provisioning=1 \\\n    --mapping-groups=groups\" >/dev/null 2>&1 || true\n  if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n    reg_ok=1; break\n  fi\n  r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n  echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n  nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n#     NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n#     login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n#     built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n#     start: occ group:adduser is a no-op for an existing member. A user is promotable only\n#     AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n#     rights on their next login + the following pod start (or run `occ group:adduser admin\n#     <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n#     passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n  current_admins=$(parse_group admin)\n  missing=0\n  for uid in $(parse_group \"$ADMIN_SRC_GROUP\"); do\n    if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n      continue\n    fi\n    missing=1\n    if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n      echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n    fi\n  done\n  [ \"$missing\" = 0 ] && break\n  p=$((p + 1)); sleep 2\ndone\n[ \"$p\" -ge 15 ] && nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\n"
       volumes:
       - name: data
         persistentVolumeClaim:

apps/wave-1/semaphore-bootstrap-secret.sops.yaml

@@ -19,6 +19,7 @@ stringData:
     admin-password: ENC[AES256_GCM,data:544nWY1UmBnlsdPJAQZJMXRumu4W9X7UXZvZ6vHN/ArjRgLXKTArOe5JkT/nNrNw,iv:AC1DDbZj2HfpU19EjzA7T+WpIaR4nZc3PNN61y8bAzU=,tag:r28LFi5w6ZeCCaRaX2jRdg==,type:str]
     db-password: ENC[AES256_GCM,data:ySkdvaGRCW9fMQqChs5l8dY4zCG19i4HOEinRC48mgLuh9WOrjd16T6jwFgiwB55,iv:+1gztJ22HbwgQJiolZMMWOcKIsxrBJpnruJdtxdooOU=,tag:8CfJGwyPTjDGyr0CvalXMQ==,type:str]
     access-key-encryption: ENC[AES256_GCM,data:1YAMLo1nVkpJWZc+KBZ7B4ia+Y8P+xMyOhHMcEGGJOXd3o+WC5FqirLBVh4=,iv:j28ozeaDhGWDjoaxFOhn2t9YbYE7L3ah/Mfi+i2DWwY=,tag:czG9Wxc/7Nsuh1W5iATOww==,type:str]
+    oidc-providers-json: ENC[AES256_GCM,data:90yt5mIVWLVI2KrlJWpxhL/BMDpEgRhv7+BL+mBRt79diRvU7pN9vtzhZprAKzcU1sEywlT4EwFlsvJpYyo5Qbkp+ZVknbyhAlGvHhJOBHukOPWgf4LignLerO2GzhXjOOzFqWfZ4EyUxNMWTbSe/xc0zIVHDDNQiYplGVRK+E/ukiI7sD/Tn2RrbrdvpC5GyK0NXWRsXn9ffhE3fCoiRHXagj9i/q+uUrQI5EKpPobrDUAGqvKoBXTl/wJ4phnWg9IqtCTE/c3yA7fLLz4j5dCtPZNSVPT4TKDx8Mdv8CfCgziVicjG2s+yHuWpsQn8lf6nvdJmpYPvIqTtLWdxFWuD8Vfo6Qbq2mqPbKR+KmDOGUnjl4gbZdL1SkE9JiqBUobz7UXeLXwYzxMIsinOfYZLcCk7alrD5Yndj9JDxDJZQ54sMMBfVbxDl94W1VHy03RxAR46KKAbPaxOP5wH4Q4z3SDceNwmuGCDRGChRXiHhN5X2X0GqYCbc8E6GdkIMcpmD80e26ydEmAjw7F89mCF4F/yMzJNiG82qaZIn8HeqgGA8urous9r9bYJNlitYTYQ,iv:jbfVdrrZ9fGiOxhQeoTJfFg1QwiOMkoQyaadiVfxX/w=,tag:JkepcRs34EuEgNb4buTTDQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -34,8 +35,8 @@ sops:
             elp0SWxHcjg0RWF3aW9mWTJHZ0lhcUEKc0rU5VilAVDleCwqq4SOF+Y9sniu1Pka
             w/MGDA9THSB0b+YgWRWiX/KiimFqp0ZFODOJ9cHVwFGY71+VadPwOg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-08-26T06:45:19Z"
-    mac: ENC[AES256_GCM,data:5XhhTZ9BGQTIdHPNrFUxwgPRm0hOqyzdsjc2ACVSJofBho+3j7XiExGTXTyblSmT5Kv+BEym0rgOjxgKKF59CAnjEMhIoZ/3mYWegQGF8HZhVqDdPgjHmxFF93D6tLK5mQGimXILmDAYcAMRAvQoJGF16BDo4v5YeVZ1bAJZ4Ug=,iv:33cmv3YaskhbUQ+K+dcJQI9LYHKZDCtDjd2NgR5yvts=,tag:CRh/ZsPggCmqyePQe5Q0gw==,type:str]
+    lastmodified: "2026-08-27T22:12:32Z"
+    mac: ENC[AES256_GCM,data:nHftJTGnTr0O5h8yGXwgz3eflggXJfIXM3m8n3bzDJE/P4nvSpZzFeMb+sc2on+AL+t+SeKrIl8blzm33XsQ1lkaY85Zm+bc5cnED1g04BMDVmmK3brh/8w50Bp2wPEVzb4I01MDzhhyIbOYg7pFvZAH3/VuqvM1t6XmfF1NrOA=,iv:4KBa9PH/dfdGwLC16RenZIVB6BtSoGo6x3qt3h1R43I=,tag:KULiA1klP/RZMNigkloGDA==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.4

apps/wave-1/semaphore.yaml

@@ -118,6 +118,11 @@ spec:
               key: access-key-encryption
         - name: SEMAPHORE_PORT
           value: :3000
+        - name: SEMAPHORE_OIDC_PROVIDERS
+          valueFrom:
+            secretKeyRef:
+              name: semaphore-bootstrap
+              key: oidc-providers-json
         volumeMounts:
         - name: config
           mountPath: /etc/semaphore

apps/wave-2/seerr.yaml

@@ -75,12 +75,17 @@ spec:
     spec:
       containers:
       - name: seerr
-        image: sctx/overseerr:latest
+        image: ghcr.io/seerr-team/seerr:v3.4.1
         ports:
         - containerPort: 5055
         volumeMounts:
         - name: config
-          mountPath: /data/config
+          mountPath: /app/config
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        runAsNonRoot: true
+        fsGroup: 1000
       volumes:
       - name: config
         persistentVolumeClaim:

catalog/gitea.yaml

@@ -41,6 +41,23 @@ env:
       secretKeyRef:
         name: gitea-bootstrap
         key: admin-password
+  # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential
+  #     `gitea` realm client's secret is mirrored into this namespace's gitea-bootstrap Secret (the same
+  #     value SOPS-held in keycloak-realm-clients); the postStart hook registers it as the "keycloak"
+  #     OAuth2 login source. The LOCAL admin above stays as the break-glass path (docs/15). ---
+  - name: GITEA_OIDC_CLIENT_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: gitea-bootstrap
+        key: oidc-client-secret
+  # app.ini [oauth2_client]: auto-provision a Gitea account on first Keycloak login and link it to an
+  # existing local account by verified email, so SSO users don't need a pre-created local user.
+  - name: GITEA__oauth2_client__ENABLE_AUTO_REGISTRATION
+    value: "true"
+  - name: GITEA__oauth2_client__ACCOUNT_LINKING
+    value: auto
+  - name: GITEA__oauth2_client__USERNAME
+    value: preferred_username
 lifecycle:
   postStart:
     exec:
@@ -61,19 +78,78 @@ lifecycle:
           done
           if su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx "$GITEA_ADMIN_USERNAME"; then
             echo "[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present"
-            exit 0
+          else
+            n=0
+            until su-exec git gitea admin user create \
+              --username "$GITEA_ADMIN_USERNAME" \
+              --password "$GITEA_ADMIN_PASSWORD" \
+              --email "[email protected]" \
+              --admin --must-change-password=false; do
+              n=$((n + 1))
+              [ "$n" -ge 10 ] && { echo "[gitea-bootstrap] admin create failed after retries" >&2; break; }
+              sleep 3
+            done
+            echo "[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'"
+          fi
+          # --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---
+          # Registers the confidential `gitea` client as the OAuth2 login source "keycloak" against the
+          # homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin
+          # (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled
+          # secret never crashes the pod.
+          #
+          # HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was
+          # ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.
+          # Now it (1) waits — bounded, ~90s — for the Keycloak discovery doc to serve HTTP 200,
+          # (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually
+          # persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker
+          # file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays
+          # usable). The verify-then-retry loop is the real guarantee against "exit 0 but nothing stuck".
+          GITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration
+          gitea_oidc_fail() {
+            echo "[gitea-oidc] FAILED to register 'keycloak' login source: $1" >&2
+            { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true
+          }
+          if [ -z "${GITEA_OIDC_CLIENT_SECRET:-}" ]; then
+            echo "[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)"
+          else
+            # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).
+            d=0; disc_ok=0
+            while [ "$d" -lt 45 ]; do
+              if [ "$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$GITEA_DISCOVERY_URL" 2>/dev/null)" = "200" ]; then
+                disc_ok=1; break
+              fi
+              d=$((d + 1)); sleep 2
+            done
+            [ "$disc_ok" = 1 ] || gitea_oidc_fail "Keycloak discovery endpoint not reachable (HTTP 200) within timeout"
+            # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.
+            r=0; reg_ok=0
+            while [ "$r" -lt 10 ]; do
+              oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2=="keycloak"{print $1; exit}')
+              if [ -n "$oidc_id" ]; then
+                su-exec git gitea admin auth update-oauth --id "$oidc_id" \
+                  --name keycloak --provider openidConnect --key gitea \
+                  --secret "$GITEA_OIDC_CLIENT_SECRET" \
+                  --auto-discover-url "$GITEA_DISCOVERY_URL" \
+                  --scopes "openid email profile" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true
+              else
+                su-exec git gitea admin auth add-oauth \
+                  --name keycloak --provider openidConnect --key gitea \
+                  --secret "$GITEA_OIDC_CLIENT_SECRET" \
+                  --auto-discover-url "$GITEA_DISCOVERY_URL" \
+                  --scopes "openid email profile" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true
+              fi
+              # VERIFY: the "keycloak" source must actually be listed now (guards the exit-0-but-nothing-persisted race).
+              if su-exec git gitea admin auth list 2>/dev/null | awk '$2=="keycloak"{f=1} END{exit !f}'; then
+                reg_ok=1; break
+              fi
+              r=$((r + 1)); sleep 3
+            done
+            if [ "$reg_ok" = 1 ]; then
+              echo "[gitea-oidc] login source 'keycloak' registered and verified"
+            else
+              gitea_oidc_fail "add/update-oauth ran but the 'keycloak' source did not persist after retries"
+            fi
           fi
-          n=0
-          until su-exec git gitea admin user create \
-            --username "$GITEA_ADMIN_USERNAME" \
-            --password "$GITEA_ADMIN_PASSWORD" \
-            --email "[email protected]" \
-            --admin --must-change-password=false; do
-            n=$((n + 1))
-            [ "$n" -ge 10 ] && { echo "[gitea-bootstrap] admin create failed after retries" >&2; exit 0; }
-            sleep 3
-          done
-          echo "[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'"
 exposure: public                # edge-forwarded git host at gitea.home.keithjackson.dev (docs/09; R-DNS/R-NET)
 egress: internet                # normal WAN egress (docs/03; R-NET-05)
 storage:
@@ -90,10 +166,12 @@ auth:
   mode: oidc                    # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02)
   groups:
     - platform-admins           # access scoped to an authorized group (R-IDP-06)
-  # NOTE (OIDC follow-up + break-glass): Gitea keeps a LOCAL admin as the break-glass path so an IdP
-  # outage never locks the operator out of recovery tools (docs/15 "Availability caveat"). At initial
-  # deploy Gitea comes up on that local admin; the Keycloak OIDC client (+ SOPS client secret) is wired
-  # once Keycloak is live. This declaration records the target integration; it does not embed a secret.
+  # WIRED (native OIDC): the Keycloak `gitea` confidential client (platform/keycloak/keycloak-realm-
+  # import.yaml + live kcadm) is registered as Gitea's "keycloak" OAuth2 login source by the postStart
+  # hook, using the client secret mirrored into the gitea-bootstrap Secret (oidc-client-secret; same
+  # value SOPS-held in keycloak-realm-clients). platform-admins (groups claim) -> Gitea admin. Gitea
+  # KEEPS a LOCAL admin as the break-glass path so an IdP outage never locks the operator out of
+  # recovery tools (docs/15 "Availability caveat"). No secret is embedded here.
 update:
   channel: stable               # R-UPD-01
   range: ">=1.22.0 <2.0.0"      # stay on the 1.x stable line

catalog/nextcloud.yaml

@@ -57,6 +57,141 @@ env:
     value: https
   - name: OVERWRITECLIURL
     value: https://nextcloud.home.keithjackson.dev
+  # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential
+  #     `nextcloud` realm client's secret is mirrored into this namespace's nextcloud-bootstrap Secret
+  #     (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The postStart hook reads it
+  #     via user_oidc's --clientsecret-env (never on argv) to register the "keycloak" provider. The LOCAL
+  #     admin above stays as the break-glass path (docs/15). ---
+  - name: NEXTCLOUD_OIDC_CLIENT_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: nextcloud-bootstrap
+        key: oidc-client-secret
+lifecycle:
+  postStart:
+    exec:
+      command:
+        - /bin/sh
+        - -c
+        - |
+          # Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint
+          # (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR
+          # updates the "keycloak" login provider (occ user_oidc:provider is create-or-update by
+          # identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud
+          # group provisioning. occ runs as www-data (owns config.php); the client secret is passed via
+          # --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing
+          # secret or not-yet-installed instance is skipped so the pod never crashloops (the local
+          # break-glass admin still works).
+          set -u
+          # OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,
+          # so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The
+          # provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as
+          # ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.
+          OCC() { su -s /bin/sh www-data -c "php /var/www/html/occ $*"; }
+          NEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration
+          nextcloud_oidc_fail() {
+            echo "[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1" >&2
+            { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true
+          }
+          # (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds
+          #     maintenance mode during first-boot install/upgrade; occ writes made in that window do not
+          #     stick — a second race on top of the quoting bug fixed at (c).
+          i=0
+          until OCC status 2>/dev/null | grep -q "installed: true" && ! OCC status 2>/dev/null | grep -q "maintenance: true"; do
+            i=$((i + 1))
+            [ "$i" -ge 300 ] && { echo "[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping" >&2; exit 0; }
+            sleep 2
+          done
+          # Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak
+          # FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's
+          # SSRF guard otherwise REFUSES that request — OCP\Http\Client\LocalServerException "violates
+          # local access rules" — during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to
+          # the user as "Could not reach the provider". (The (b) curl check below uses system libcurl,
+          # NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied
+          # every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.
+          OCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \
+            || echo "[nextcloud-oidc] WARN: failed to set allow_local_remote_servers" >&2
+          if [ -z "${NEXTCLOUD_OIDC_CLIENT_SECRET:-}" ]; then
+            echo "[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)"
+            exit 0
+          fi
+          if ! OCC app:list 2>/dev/null | grep -q "user_oidc"; then
+            OCC app:install user_oidc || echo "[nextcloud-oidc] app:install user_oidc failed" >&2
+          fi
+          OCC app:enable user_oidc >/dev/null 2>&1 || true
+          # (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.
+          d=0; disc_ok=0
+          while [ "$d" -lt 45 ]; do
+            if [ "$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$NEXTCLOUD_DISCOVERY_URL" 2>/dev/null)" = "200" ]; then
+              disc_ok=1; break
+            fi
+            d=$((d + 1)); sleep 2
+          done
+          [ "$disc_ok" = 1 ] || nextcloud_oidc_fail "Keycloak discovery endpoint not reachable (HTTP 200) within timeout"
+          # (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.
+          #     HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a
+          #     single argument. The previous `OCC user_oidc:provider ... --scope="openid email profile"`
+          #     form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->
+          #     occ "Too many arguments" -> silent failure on EVERY boot. Secret via --clientsecret-env
+          #     (never on argv). The verify-then-retry loop guarantees the provider actually persisted.
+          r=0; reg_ok=0
+          while [ "$r" -lt 10 ]; do
+            su -s /bin/sh www-data -c "php /var/www/html/occ user_oidc:provider keycloak \
+              --clientid=nextcloud \
+              --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \
+              --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \
+              --scope='openid email profile' \
+              --unique-uid=0 \
+              --mapping-uid=preferred_username \
+              --mapping-email=email \
+              --mapping-display-name=name \
+              --group-provisioning=1 \
+              --mapping-groups=groups" >/dev/null 2>&1 || true
+            if OCC user_oidc:provider 2>/dev/null | grep -q "keycloak"; then
+              reg_ok=1; break
+            fi
+            r=$((r + 1)); sleep 3
+          done
+          if [ "$reg_ok" = 1 ]; then
+            echo "[nextcloud-oidc] provider 'keycloak' configured and verified"
+          else
+            nextcloud_oidc_fail "user_oidc:provider ran but the provider did not persist after retries"
+          fi
+          # (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has
+          #     NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a
+          #     login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the
+          #     built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every
+          #     start: occ group:adduser is a no-op for an existing member. A user is promotable only
+          #     AFTER their first OIDC login provisions them into the group, so a brand-new admin gains
+          #     rights on their next login + the following pod start (or run `occ group:adduser admin
+          #     <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is
+          #     passed to php as argv (never interpolated) so group:list JSON parses robustly.
+          ADMIN_SRC_GROUP=platform-admins
+          # HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`
+          # exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot
+          # transient failure (DB / maintenance race in the first seconds after container start) silently
+          # dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds
+          # any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);
+          # it is a no-op for members already in 'admin'. Group names are passed to php as argv (never
+          # interpolated) so the JSON parses robustly.
+          parse_group() { su -s /bin/sh www-data -c "php /var/www/html/occ group:list --output=json" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,"\n";}}' -- "$1" 2>/dev/null; }
+          p=0
+          while [ "$p" -lt 15 ]; do
+            current_admins=$(parse_group admin)
+            missing=0
+            for uid in $(parse_group "$ADMIN_SRC_GROUP"); do
+              if printf '%s\n' "$current_admins" | grep -qxF -- "$uid"; then
+                continue
+              fi
+              missing=1
+              if OCC group:adduser admin "$uid" >/dev/null 2>&1; then
+                echo "[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)"
+              fi
+            done
+            [ "$missing" = 0 ] && break
+            p=$((p + 1)); sleep 2
+          done
+          [ "$p" -ge 15 ] && nextcloud_oidc_fail "could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries"
 exposure: public                # edge port-forward + public DNS (docs/09; R-DNS/R-NET)
 egress: internet                # normal WAN egress (docs/03; R-NET-05) — app store, integrity checks
 storage:
@@ -77,14 +212,18 @@ auth:
   mode: oidc                    # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
   groups:
     - platform-admins           # access scoped to an authorized group (R-IDP-06)
-  # NOTE (OIDC follow-up + break-glass): like Gitea (docs/15 "Availability caveat"), Nextcloud keeps a
-  # LOCAL admin as the break-glass path so an IdP outage never locks the operator out. At initial deploy
-  # it comes up INSTALLED on that local admin (env above). The Keycloak `nextcloud` realm client + its
-  # user_oidc app config is a DEFERRED follow-up: wiring it means editing the reconciled realm import in
-  # platform-config (a malformed edit fails the whole platform-config apply / can leave the realm import
-  # PENDING) AND post-install `occ` steps the image has no first-class env for — deferred rather than
-  # risking the running Keycloak realm / Flux source (WS-APPS step 3 escape hatch). This records the
-  # target integration; it embeds no secret.
+  # WIRED (native OIDC): the Keycloak `nextcloud` confidential client (platform/keycloak/keycloak-realm-
+  # import.yaml + live kcadm) is registered as a user_oidc login provider ("keycloak") by the postStart
+  # hook (occ user_oidc:provider), using the client secret mirrored into nextcloud-bootstrap
+  # (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The `groups` claim drives
+  # user_oidc group provisioning (R-IDP-06). Members of `platform-admins` are ELEVATED to Nextcloud
+  # admins by the postStart hook step (d) — user_oidc 8.11 has no native group->admin mapping, so the
+  # provisioned membership is mirrored into the built-in `admin` group idempotently on every start; a
+  # user is promoted after their first login provisions them + the next pod start. Nextcloud KEEPS its
+  # LOCAL admin as the break-glass path so an IdP outage never locks the operator out (docs/15
+  # "Availability caveat"). Reachability to the IdP at login requires allow_local_remote_servers=true
+  # (set by the hook): the Keycloak FQDN resolves to the private gateway VIP, which Nextcloud's SSRF
+  # guard blocks by default ("Could not reach the provider"). No secret is embedded here.
 update:
   channel: stable               # R-UPD-01
   range: ">=31.0.0 <32.0.0"     # stay on the 31.x stable line

catalog/plex.yaml

@@ -23,12 +23,53 @@ name: plex
 kind: container
 source:
   image: plexinc/pms-docker:latest
+# plexinc/pms-docker env. Rendered VERBATIM onto the app container by render.py (schema env list).
+env:
+  # PLEX_UID/PLEX_GID: the image boots as root (s6 PID1), chowns /config to this uid/gid, then drops
+  # the "Plex Media Server" process to it. 1000 matches the Longhorn ownership convention the other
+  # stateful apps use (seerr/semaphore run as UID/GID 1000), keeping the /config PVC writable by the
+  # dropped plex user. (Plex has no podSecurityContext/fsGroup — the image manages ownership itself.)
+  - name: PLEX_UID
+    value: "1000"
+  - name: PLEX_GID
+    value: "1000"
+  # TZ: no other service in the repo sets a timezone (repo-wide grep: none), so use the safe default.
+  - name: TZ
+    value: Etc/UTC
+  # ALLOWED_NETWORKS marks these CIDRs as trusted LAN, so in-cluster clients (Seerr -> plex:32400) are
+  # treated as local and NOT pushed onto Plex's remote-auth/relay path. 10.0.0.0/8 covers the Cilium
+  # pod CIDR (10.0.0.0/16, CLUSTER_POD_CIDR — Seerr's pod is 10.0.x.x) and the k3s service net
+  # (10.43.0.0/16); 192.168.0.0/16 covers the home LAN.
+  - name: ALLOWED_NETWORKS
+    value: 10.0.0.0/8,192.168.0.0/16
+  # PLEX_CLAIM is deliberately NOT set here: a https://plex.tv/claim token has a ~4-min TTL, far
+  # shorter than the commit -> push -> Flux reconcile -> pod-roll cycle, so any token committed to git
+  # is already expired by the time the pod starts. Claiming is a one-time LIVE step (UI sign-in, or a
+  # single API POST) against the running server; the claim then persists on the /config PVC and
+  # survives pod rolls (including the roll this env change causes), so it never belongs in git.
+  # ADVERTISE_IP ("Custom server access URLs") is DEFERRED, not accidentally omitted. It tells Plex
+  # which URL to advertise to plex.tv for REMOTE clients. The only candidate is
+  # https://plex.home.keithjackson.dev, but that host resolves via split-horizon DNS to the INTERNAL
+  # Gateway VIP (conformance/tests/l2-behavioral/dns-internal-resolution), and both its public
+  # reachability and whether the TLS-terminating Cilium Gateway proxies Plex's 32400 API correctly for
+  # remote access are UNVERIFIED. Advertising a wrong/internal-only URL breaks remote access for ALL
+  # clients, so per "prefer safety over completeness" it is left unset. FOLLOW-UP: once remote
+  # reachability through the Gateway is confirmed, set ADVERTISE_IP: "https://plex.home.keithjackson.dev:443".
 exposure: public              # edge port-forward + public DNS; clients + Seerr reach it (docs/09; R-DNS/R-NET)
 egress: internet              # normal WAN egress (docs/03; R-NET-05)
 storage:
   - name: config
     size: 8Gi
     class: longhorn           # Plex DB/metadata: replicated app state on the 3-replica tier (docs/04; R-STATE-01/02)
+    mountPath: /config        # plexinc/pms-docker writes its identity/claim + "Library" DB under /config, NOT the
+                              # renderer default /data/config. Without this the PVC sat unused and Plex wrote to the
+                              # container's EPHEMERAL layer -> unclaimed + all state lost on restart. The image inits as
+                              # root (s6-svscan PID 1), chowns /config to the plex user, then drops -> no fsGroup needed
+                              # (verified in-cluster: PID1=root, "Plex Media Server"=plex). Persisting /config here also
+                              # makes the claim survive restarts once claimed. To auto-claim on first boot instead of the
+                              # UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token
+                              # (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`,
+                              # so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
 dependencies: []              # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
 ingressRoutes:
   - host: plex.home.keithjackson.dev

catalog/render.py

@@ -1038,6 +1038,7 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
       - `<svc>-bootstrap-secret.sops.yaml`  -> wave(svc)   (consumed by svc's env, same apply)
       - `<svc>-db.yaml`                     -> wave(svc-db) (a strictly-lower wave; CNPG DB)
       - `<svc>-db-app.sops.yaml`            -> wave(svc-db) (CNPG initdb owner secret)
+      - `<svc>-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret)
     Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
     Only files that actually exist in `present_files` are placed, so the plan is robust to a
     service having no bootstrap secret or no DB. Deterministic (sorted).
@@ -1060,6 +1061,7 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
             wdb = waves[db]
             add(wdb, f"{db}.yaml")
             add(wdb, f"{db}-app.sops.yaml")
+            add(wdb, f"{db}-backup-credentials.sops.yaml")   # CNPG Barman S3 creds (R-STATE-03)
 
     for unit, cfg in EXTRA_UNITS.items():
         anchor = cfg["anchor"]
@@ -1453,7 +1455,8 @@ def _selftest() -> int:
     #     EXTRA_UNITS anchor strictly above; a file absent from the tree is never placed.
     present = {
         "a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
-        "svc-db.yaml", "svc-db-app.sops.yaml", "gitea.yaml",
+        "svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
+        "gitea.yaml",
         "gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
     }
     plan_sds = wsds + [{"name": "gitea", "source": {"image": "x:1"}},
@@ -1461,8 +1464,9 @@ def _selftest() -> int:
     plan = plan_apps_waves(plan_sds, present)
     flat = {f: wv for wv, fs in plan.items() for f in fs}
     check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
-    check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0,
-          "CNPG DB manifest + its owner secret must be in the DB's (lower) wave")
+    check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
+          and flat.get("svc-db-backup-credentials.sops.yaml") == 0,
+          "CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave")
     check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
           "a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
     check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")

catalog/seerr.yaml

@@ -4,17 +4,19 @@
 # public | egress internet | config on Longhorn | auth: native (Plex accounts) |
 # seerr -> sonarr:8989, seerr -> radarr:7878, seerr -> plex:32400.
 #
-# Seerr is the media request UI (Overseerr): users request titles, it hands them to the arr
-# stack and reflects library state from Plex. It is publicly exposed (edge port-forward + public
+# Seerr is the media request UI (Overseerr's maintained successor): users request titles, it
+# hands them to the arr stack and reflects library state from Plex. It is publicly exposed (edge port-forward + public
 # DNS, docs/09) and authenticates with Plex's own account system — a DOCUMENTED SSO exception
 # (docs/13/15; R-IDP-02): NOT Keycloak, so no oidc/forward-auth and no auth.groups. The three
 # east-west hops below are EXPLICIT allows on top of default-deny (docs/03; R-NET-01/02);
 # everything else (except DNS) is denied. Validates against schemas/servicedefinition.schema.json.
 #
-# Image: Overseerr (sctx/overseerr) — the Plex-native request UI on port 5055 (docs/13 row 5
-# names "seerr" generically; Overseerr is the Plex-account variant). `:latest` mirrors the
-# onboarding posture used for plex (image-automation off during onboarding; update.range is the
-# advisory bound the ImagePolicy will enforce once the Flux image controllers are installed).
+# Image: Seerr (ghcr.io/seerr-team/seerr) — the maintained successor into which the original
+# Overseerr (sctx/overseerr, now unmaintained) and Jellyseerr (fallenbagel/jellyseerr) merged;
+# the Plex-native request UI on port 5055 (docs/13 row 5 names "seerr" generically). Pinned to a
+# specific stable tag (v3.4.1) per the arr-stack convention (sonarr/radarr/qbittorrent/gitea all
+# pin explicit tags; R-UPD-01); update.range is the advisory bound the ImagePolicy will enforce
+# once the Flux image controllers are installed. Config lives under /app/config (upstream default).
 #
 # No `bulk` volume: seerr keeps only its own config PVC (settings + request/cache DB) on the
 # longhorn tier. It stores no media and needs no bulk mount, so OQ-2 (unresolved bulk backing)
@@ -22,13 +24,28 @@
 name: seerr
 kind: container
 source:
-  image: sctx/overseerr:latest   # Overseerr: Plex-native request UI (docs/13 row 5); served on 5055
+  image: ghcr.io/seerr-team/seerr:v3.4.1   # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
+# The ghcr.io/seerr-team/seerr image runs as the non-root user `node:node` (UID 1000 / GID 1000 —
+# the standard node base-image user; confirmed from the image config's User field). On boot the app
+# creates /app/config/logs and writes its settings/DB under the mounted /app/config PVC. A fresh
+# Longhorn PVC is root:root, so UID 1000 cannot mkdir there -> the pod CrashLoopBackOffs with
+# "EACCES: permission denied, mkdir '/app/config/logs/'". (The old sctx/overseerr image ran as root
+# and never hit this.) fsGroup=1000 makes the kubelet group-own the mounted PVC to GID 1000 and add
+# it as a supplemental group so the process can write — Longhorn's CSIDriver
+# fsGroupPolicy: ReadWriteOnceWithFSType applies fsGroup to this RWO filesystem volume. runAsUser/
+# runAsGroup pin the image's own identity, runAsNonRoot asserts it (same pattern as semaphore; R-STATE-01/02).
+podSecurityContext:
+  runAsUser: 1000
+  runAsGroup: 1000
+  runAsNonRoot: true
+  fsGroup: 1000
 exposure: public                 # edge port-forward + public DNS (docs/09; R-DNS/R-NET)
 egress: internet                 # normal WAN egress (docs/03; R-NET-05) — TMDB/Plex.tv metadata over the internet
 storage:
   - name: config
     size: 5Gi
     class: longhorn              # settings + request/cache DB: replicated app state on the 3-replica tier (docs/04; R-STATE-01/02)
+    mountPath: /app/config       # Seerr/Overseerr/Jellyseerr keep config + DB under /app/config (docs.seerr.dev); /data/<name> default would not persist it
 dependencies:
   - to: sonarr                   # seerr -> sonarr: submit TV requests (docs/03 allow rules; docs/13 graph; R-NET-02)
     port: 8989
@@ -46,7 +63,7 @@ auth:
   mode: native                   # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02)
 update:
   channel: stable                # R-UPD-01 (never pre-release)
-  range: ">=1.33.0"              # acceptable versions (advisory; image-automation off during onboarding)
+  range: ">=3.4.0"               # acceptable versions (advisory; image-automation off during onboarding) — Seerr 3.x line
   cadence: "0 4 * * 0"           # weekly stable check/apply
   autoRollbackOnFailedHealth: true   # R-UPD-03
 observability:

catalog/semaphore.yaml

@@ -73,6 +73,21 @@ env:
   # service links), pinning the listen port back to :3000 (matches containerPort/Service).
   - name: SEMAPHORE_PORT
     value: ":3000"
+  # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). Semaphore
+  #     v2.19.10 reads its OIDC providers from the SEMAPHORE_OIDC_PROVIDERS env as a JSON map
+  #     (util ConfigType `oidc_providers`, env-tagged SEMAPHORE_OIDC_PROVIDERS). The value is a single
+  #     JSON blob for the `keycloak` provider — provider_url is the homelab realm issuer (auto-discovery),
+  #     client_id=semaphore, redirect_url=/api/auth/oidc/keycloak/redirect, claims map to the realm's
+  #     token. It CONTAINS the confidential client secret, so the whole blob is held SOPS-encrypted in the
+  #     semaphore-bootstrap Secret (key oidc-providers-json; same secret value SOPS-held in
+  #     keycloak-realm-clients) and injected here by reference — never in cleartext. The name is NOT a
+  #     kubelet service-link var (those are SEMAPHORE_SERVICE_HOST / SEMAPHORE_PORT_*), so no collision.
+  #     The LOCAL SEMAPHORE_ADMIN above stays as the break-glass path (docs/15). ---
+  - name: SEMAPHORE_OIDC_PROVIDERS
+    valueFrom:
+      secretKeyRef:
+        name: semaphore-bootstrap
+        key: oidc-providers-json
 # The semaphoreui/semaphore image runs as the non-root user UID 1001 / GID 0 (tini -> server-wrapper),
 # and on boot the wrapper writes its generated config.json into the config dir. A mounted Longhorn PVC
 # is root:root, so UID 1001 cannot write it -> the pod panics ("permission denied"). fsGroup=1001 makes
@@ -101,14 +116,11 @@ auth:
   mode: oidc                    # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
   groups:
     - platform-admins           # access scoped to an authorized group (R-IDP-06)
-  # NOTE (OIDC follow-up + break-glass): like Gitea/Nextcloud (docs/15 "Availability caveat"),
-  # Semaphore keeps a LOCAL admin as the break-glass path so an IdP outage never locks the operator
-  # out. At initial deploy it comes up on that local admin (SEMAPHORE_ADMIN* env above). The Keycloak
-  # `semaphore` realm client + Semaphore's OIDC provider config (config.json `oidc_providers`) is a
-  # DEFERRED follow-up: wiring it means editing the reconciled realm import in platform-config (a
-  # malformed edit fails the whole platform-config apply / can leave the realm import PENDING) — so it
-  # is deferred rather than risking the running Keycloak realm / Flux source. This records the target
-  # integration; it embeds no secret.
+  # WIRED (native OIDC): the Keycloak `semaphore` confidential client (platform/keycloak/keycloak-realm-
+  # import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env (JSON blob held encrypted
+  # in the semaphore-bootstrap Secret, oidc-providers-json). The `groups` claim is exposed for
+  # authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the break-glass path so an IdP outage
+  # never locks the operator out (docs/15 "Availability caveat"). No secret is embedded here.
 update:
   channel: stable               # R-UPD-01
   range: ">=2.19.0 <3.0.0"      # stay on the 2.x stable line

ci/Makefile

@@ -11,7 +11,7 @@ CI_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 .DEFAULT_GOAL := help
 
-.PHONY: help spec-immutability lint schema l0 l1 conformance gate all touched versions tools
+.PHONY: help spec-immutability lint schema l0 render-golden l1 conformance gate all touched versions tools
 
 help: ## Show this help
 	@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf "  \033[36m%-14s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST)
@@ -25,6 +25,9 @@ lint: ## yaml/lint (no cluster; live now)
 schema l0: ## ServiceDefinition schema validation + Kyverno L0 policy gate (no cluster; live now)
 	@"$(CI_DIR)l0.sh"
 
+render-golden: ## render==golden: canonical render reproduces committed apps/ (no cluster; live now)
+	@"$(CI_DIR)render-golden.sh"
+
 l1: ## L1 rendering checks (no cluster; live now)
 	@"$(CI_DIR)l1.sh"
 

ci/gate.sh

@@ -38,10 +38,11 @@ summary "## Merge gate (R-TEST-03)"
 summary ""
 
 run_stage spec-immutability "${CI_DIR}/spec-immutability.sh"
-run_stage lint        "${CI_DIR}/lint.sh"
-run_stage l0          "${CI_DIR}/l0.sh"
-run_stage l1          "${CI_DIR}/l1.sh"
-run_stage conformance "${CI_DIR}/conformance.sh"
+run_stage lint          "${CI_DIR}/lint.sh"
+run_stage l0            "${CI_DIR}/l0.sh"
+run_stage render-golden "${CI_DIR}/render-golden.sh"
+run_stage l1            "${CI_DIR}/l1.sh"
+run_stage conformance   "${CI_DIR}/conformance.sh"
 
 echo >&2
 log "==== gate decision ===="
@@ -56,8 +57,10 @@ case "${OUTCOME[spec-immutability]}" in
 esac
 
 # L0/L1 stages: any failure blocks. A SKIP here means a seam is unwired — in
-# Phase 0 that is tolerated (placeholders), but surfaced.
-for s in lint l0 l1; do
+# Phase 0 that is tolerated (placeholders), but surfaced. render-golden is a
+# no-cluster reproducibility gate (canonical render == committed apps/ goldens):
+# a FAIL means a golden drifted from its ServiceDefinition source — always blocks.
+for s in lint l0 render-golden l1; do
   case "${OUTCOME[$s]}" in
     fail) err  "${s}: FAILED -> blocks merge"; BLOCK=1 ;;
     skip) warn "${s}: skipped (unwired seam / missing tool) — not green" ;;

ci/render-golden.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# ci/render-golden.sh — Stage: render==golden reproducibility gate.
+#
+# WHY: the committed apps/ tree is a set of GOLDENS produced by ONE canonical
+# invocation of catalog/render.py. If a golden is hand-edited, or a catalog
+# ServiceDefinition is changed without re-rendering, the goldens drift from
+# their source of truth and Flux deploys something no one reviewed. The three
+# L1 render spot-checks (conformance/tests/l1-rendering/*) only cover a couple
+# of assertions on a couple of services — they let a real source/golden
+# mismatch through (image-automation objects present in gitea/nextcloud but
+# absent everywhere else). This stage closes that hole: it renders EVERY
+# onboarded catalog service with the canonical command and FAILS on ANY diff
+# vs the committed golden.
+#
+# CANONICAL RENDER (source of truth for apps/):
+#     python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation
+#
+# The `--no-image-automation` flag is canonical because image automation is
+# authored-but-DISABLED (DECISIONS-LOG: "Phase 4 image-automation merged but
+# DISABLED pending a Gitea write-cred"; pihole/all apps onboarded via
+# `--no-image-automation`). The Flux image-reflector/-automation controllers
+# are NOT installed in-cluster, so emitting ImageRepository/ImagePolicy CRs
+# would leave apps referencing controllers that don't exist. Every onboarded
+# app must render identically under this ONE command.
+#
+# Runs in CI WITHOUT a cluster (pure render + diff). Exit codes per ci/lib.sh:
+#   0 = pass, 1 = fail (blocks merge), 3 = skip (renderer/tooling missing).
+set -euo pipefail
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck disable=SC1091
+source "${CI_DIR}/lib.sh"
+
+APPS_DIR="${APPS_DIR:-${REPO_ROOT}/apps}"
+RENDER_PY="${RENDER_PY:-${REPO_ROOT}/catalog/render.py}"
+PYTHON="${PYTHON:-python3}"
+
+# Catalog services that are intentionally NOT onboarded to the apps tree yet
+# (computed into a wave by the renderer but with NO committed manifest — see
+# DESIGN-waves.md §"Catalog services not yet onboarded"). They render as a
+# by-product of `--all catalog` but must NOT have a golden; the gate ignores
+# their rendered output. Onboarding one = add its golden and drop it here.
+: "${RENDER_GOLDEN_NOT_ONBOARDED:=keycloak vm-demo}"
+
+log "Stage: render==golden reproducibility (canonical render vs committed apps/)"
+
+if ! entrypoint_ready "$PYTHON"; then
+  skip "python3 not available — cannot run the renderer. UNWIRED, not green."
+  summary "- render-golden: **SKIPPED** (python3 missing)"
+  exit 3
+fi
+if [ ! -f "$RENDER_PY" ]; then
+  skip "renderer not present at '${RENDER_PY}' (catalog/render.py) — UNWIRED, not green."
+  summary "- render-golden: **SKIPPED** (renderer missing)"
+  exit 3
+fi
+if [ ! -d "$CATALOG_DIR" ] || ! compgen -G "${CATALOG_DIR}/*.yaml" >/dev/null 2>&1; then
+  skip "no catalog ServiceDefinitions under '${CATALOG_DIR}' — nothing to render."
+  summary "- render-golden: **SKIPPED** (empty catalog)"
+  exit 3
+fi
+
+# --- Render every service with the ONE canonical command into a temp dir -----
+TMP_RENDER="$(mktemp -d)"
+trap 'rm -rf "$TMP_RENDER"' EXIT
+
+if ! "$PYTHON" "$RENDER_PY" --all "$CATALOG_DIR" -o "$TMP_RENDER" --wave-layout --no-image-automation; then
+  err "canonical render failed to run"
+  summary "- render-golden: **FAILED** (renderer error)"
+  exit 1
+fi
+
+# Drop the not-onboarded services' rendered output — they have no golden by design.
+for svc in $RENDER_GOLDEN_NOT_ONBOARDED; do
+  find "$TMP_RENDER" -type f -name "${svc}.yaml" -delete
+done
+
+# --- Compare EVERY rendered service against its committed golden --------------
+# For each rendered file (relative path wave-N/<name>.yaml):
+#   - the committed golden apps/wave-N/<name>.yaml MUST exist, and
+#   - it MUST be byte-identical to the freshly rendered output.
+# Any missing golden or any diff is a hard FAIL (blocks merge).
+fail=0
+checked=0
+while IFS= read -r -d '' rendered; do
+  rel="${rendered#"$TMP_RENDER"/}"
+  golden="${APPS_DIR}/${rel}"
+  checked=$((checked + 1))
+  if [ ! -f "$golden" ]; then
+    err "MISSING GOLDEN: catalog renders '${rel}' but apps/${rel} does not exist"
+    err "  -> re-render (see canonical command in this script's header) and commit, or add it to RENDER_GOLDEN_NOT_ONBOARDED"
+    fail=1
+    continue
+  fi
+  if ! diff -u "$golden" "$rendered" >/dev/null 2>&1; then
+    err "GOLDEN DRIFT: apps/${rel} does not match the canonical render:"
+    diff -u "$golden" "$rendered" >&2 || true
+    fail=1
+  fi
+done < <(find "$TMP_RENDER" -type f -print0)
+
+if [ "$checked" -eq 0 ]; then
+  err "renderer produced no output — cannot verify goldens"
+  summary "- render-golden: **FAILED** (renderer produced nothing)"
+  exit 1
+fi
+
+if [ "$fail" -ne 0 ]; then
+  err "render==golden check FAILED — the committed apps/ goldens are NOT reproducible"
+  err "  from the canonical render. A golden was hand-edited, or a catalog source"
+  err "  changed without re-rendering. Fix:"
+  err "    ${PYTHON} catalog/render.py --all catalog -o apps --wave-layout --no-image-automation"
+  err "  then remove the not-onboarded outputs (${RENDER_GOLDEN_NOT_ONBOARDED// /, }) and commit."
+  summary "- render-golden: **FAILED** (goldens drifted from canonical render)"
+  exit 1
+fi
+
+ok "render==golden: all ${checked} onboarded service golden(s) reproduce from the canonical render"
+summary "- render-golden: **passed** (${checked} services; canonical \`--no-image-automation\` render)"
+exit 0

clusters/base/backup.yaml

@@ -0,0 +1,39 @@
+---
+# Flux Kustomization: Velero backup controller — reconciles ./platform/velero.
+#
+# OWN NON-GATING TIER (dependsOn platform-config, wait:false) — the same rationale as headlamp/
+# oauth2-proxy/crafty. Velero is a BACKUP CONTROLLER; it must NEVER gate the platform graph. Its
+# HelmRelease (chart + AWS plugin init-container + node-agent DaemonSet) can legitimately be
+# not-Ready during a cold start (image pulls, the node-agent rolling out on every node, the BSL
+# doing its first reachability probe against MinIO). If it sat in the gating `platform` tier, that
+# transient unhealth would stall platform-config + every apps wave/vm on "dependency not ready" —
+# exactly the failure headlamp already encodes. Nothing in the graph dependsOn backup, and wait:false
+# guarantees its health can NEVER gate anything even if a future resource is (mistakenly) pointed at it.
+#
+# It reconciles AFTER platform-config only so the storage/CRD surface it backs up already exists; the
+# backup path itself (Velero pods → http://192.168.1.9:9000) is independent of the rest of the graph.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: velero
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 2m
+  timeout: 5m
+  prune: true
+  wait: false
+  dependsOn:
+    - name: platform-config
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./platform/velero
+  # cluster-vars feeds the backup cadence/retention knobs (${VELERO_BACKUP_SCHEDULE}, ${VELERO_RETENTION}).
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-vars
+  # SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per
+  # environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how
+  # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml.

clusters/base/cluster-vars.yaml

@@ -35,6 +35,11 @@ data:
   # --- observability sizing (docs/10) ---
   MONITORING_PROMETHEUS_RETENTION: "7d"     # dev short | prod long
   MONITORING_PROMETHEUS_PVC_SIZE: "10Gi"
+  # SIZE-based retention cap for Prometheus (Prometheus *iB units, NOT the K8s `Gi` of the PVC knob).
+  # MUST stay below MONITORING_PROMETHEUS_PVC_SIZE — leaves headroom for the WAL + head + compaction
+  # scratch (retentionSize counts only persisted blocks). Prevents a busy TSDB from filling /prometheus
+  # (a full disk wedges all WAL commits and makes every absent()/==0 alert fire falsely). Base 10Gi PVC.
+  MONITORING_PROMETHEUS_RETENTION_SIZE: "8GiB"
   MONITORING_LOKI_PVC_SIZE: "10Gi"
 
   # --- admission enforcement (Kyverno, ADR-0011). Base defaults are the STRICT/prod posture;

clusters/base/crafty.yaml

@@ -0,0 +1,42 @@
+---
+# Flux Kustomization: Crafty Controller — reconciles ./platform/crafty.
+#
+# POST-PLATFORM TIER (dependsOn platform-config). Crafty is a LEAF Minecraft-management UI that depends
+# on Keycloak (the oauth2-proxy sidecar authenticates against it) + the trusted wildcard TLS cert +
+# split-horizon DNS + the Gateway API CRDs + the Cilium L2 LoadBalancer pool (game-port VIP) — all
+# brought up by `platform` (controllers) and `platform-config` (Keycloak realm/CRs, the wildcard
+# Certificate, cilium L2 pool + announcement, Gateway). It therefore MUST NOT live inside the gating
+# `platform` tier: a UI whose pod can crashloop (image pull, DB init, sidecar OIDC discovery during a
+# cold start) would keep `platform` not-Ready, and while `platform` is not-Ready NOTHING downstream
+# reconciles — the exact incident behind the headlamp/oauth2-proxy/homepage moves (DECISIONS-LOG:
+# "Headlamp gating stall"). Same lesson, same fix: a leaf reconciles in its OWN post-platform tier.
+#
+# NON-GATING (wait:false): nothing in the graph dependsOn crafty, and its pod can be long-not-Ready
+# during a cold start (Longhorn attach + Crafty DB init) while it waits on Keycloak/DNS/TLS.
+# wait:false guarantees its health can NEVER gate anything, even if a future resource is (mistakenly)
+# pointed dependsOn it.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: crafty
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 2m
+  timeout: 5m
+  prune: true
+  wait: false
+  dependsOn:
+    - name: platform-config   # Keycloak realm + wildcard TLS + split-horizon VIP + Gateway + L2 pool first
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./platform/crafty
+  # cluster-vars retained for parity with the other tiers / future overlay knobs (crafty references no
+  # ${VARS} today).
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-vars
+  # SOPS decryption for crafty-oauth2-proxy-credentials.sops.yaml is layered per-environment by the
+  # overlay (clusters/dev patches THIS Kustomization by name), identical to headlamp/oauth2-proxy.

clusters/base/headlamp.yaml

@@ -0,0 +1,45 @@
+---
+# Flux Kustomization: headlamp dashboard — reconciles ./platform/headlamp.
+#
+# POST-PLATFORM TIER (dependsOn platform-config). Headlamp is a LEAF Kubernetes-dashboard UI that
+# depends on Keycloak (native-OIDC login) + the trusted wildcard TLS cert + split-horizon DNS + the
+# Gateway API CRDs — all brought up by `platform` (controllers) and `platform-config` (Keycloak
+# instance/realm CRs, the wildcard Certificate, cilium L2 VIP, Gateway). It therefore MUST NOT live
+# inside the gating `platform` tier: a UI HelmRelease whose image/chart can crashloop (e.g. the
+# upstream headlamp-0.40.1 `-session-ttl` chart/image desync, issue #4883) would keep `platform`
+# not-Ready, and while `platform` is not-Ready NOTHING downstream reconciles — platform-config,
+# oauth2-proxy, every apps-wave and apps-vms all stall on "dependency not ready". That is exactly
+# how a single leaf UI stalled the whole chain (DECISIONS-LOG: "Headlamp gating stall"). This is the
+# same lesson oauth2-proxy already encodes: a leaf that can be legitimately unhealthy must reconcile
+# in its OWN post-platform Kustomization so its health can never gate the graph.
+#
+# NON-GATING (wait:false): nothing in the graph dependsOn headlamp (apps waves gate on
+# platform-config, not on this), and a dashboard pod can be long-not-Ready during a cold start while
+# it waits on Keycloak/DNS/TLS — same rationale as oauth2-proxy and apps-vms. wait:false guarantees
+# its health can NEVER gate anything, even if a future resource is (mistakenly) pointed dependsOn it.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: headlamp
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 2m
+  timeout: 5m
+  prune: true
+  wait: false
+  dependsOn:
+    - name: platform-config   # Keycloak instance/realm + wildcard TLS + split-horizon VIP + Gateway first
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./platform/headlamp
+  # Same cluster-vars ConfigMap the platform tier fed it before the move (headlamp itself references
+  # no ${VARS} today, but the substitution is retained for parity with the other tiers/future knobs).
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-vars
+  # SOPS decryption for headlamp-oidc-credentials.sops.yaml is layered per-environment by the overlay
+  # (clusters/dev patches THIS Kustomization by name), identical to how it was decrypted via the
+  # `platform` Kustomization before the move. See clusters/dev/kustomization.yaml.

clusters/base/homepage.yaml

@@ -0,0 +1,42 @@
+---
+# Flux Kustomization: Homepage landing dashboard — reconciles ./platform/homepage.
+#
+# POST-PLATFORM TIER (dependsOn platform-config). Homepage is a LEAF landing UI at the apex
+# home.keithjackson.dev. It depends on the trusted wildcard TLS cert + the shared Gateway (incl. the
+# `https-apex` listener) + split-horizon DNS — all brought up by `platform` (controllers) and
+# `platform-config` (the wildcard Certificate, cilium L2 VIP, Gateway). It therefore MUST NOT live in
+# the gating `platform` tier: a UI whose pod can be unhealthy would keep `platform` not-Ready, and
+# while `platform` is not-Ready NOTHING downstream reconciles — the exact incident behind the
+# headlamp/oauth2-proxy moves (DECISIONS-LOG: "Headlamp gating stall"). Same lesson, same fix: a leaf
+# that can be legitimately unhealthy reconciles in its OWN post-platform Kustomization.
+#
+# NON-GATING (wait:false): nothing in the graph dependsOn homepage (apps waves gate on
+# platform-config, not on this), and a dashboard pod can be long-not-Ready during a cold start.
+# wait:false guarantees its health can NEVER gate anything, even if a future resource is (mistakenly)
+# pointed dependsOn it.
+#
+# NO SOPS decryption stanza: this component carries no encrypted Secret (plain manifests + a config
+# ConfigMap with no API keys), so — unlike headlamp/oauth2-proxy — clusters/dev does NOT patch it.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: homepage
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 2m
+  timeout: 5m
+  prune: true
+  wait: false
+  dependsOn:
+    - name: platform-config   # wildcard TLS + Gateway (incl. https-apex listener) + split-horizon VIP first
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./platform/homepage
+  # Homepage references no ${VARS} today; the substitution is retained for parity with the other
+  # tiers / future overlay knobs.
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-vars

clusters/base/kustomization.yaml

@@ -17,4 +17,25 @@ resources:
   - apps-wave-2.yaml     # Flux Kustomization -> ./apps/wave-2 (dependsOn apps-wave-1)
   # VM tier (KubeVirt) — non-gating (wait:false), separate lifecycle from the app waves.
   - apps-vms.yaml        # Flux Kustomization -> ./apps/vms (dependsOn platform-config)
+  # oauth2-proxy forward-auth — POST-platform tier (dependsOn platform-config, wait:false). A leaf
+  # service that needs Keycloak + trusted TLS + split-horizon DNS must NOT sit in the gating
+  # `platform` tier, where it self-gated its own dependencies into a cold-start deadlock (twice).
+  - oauth2-proxy.yaml    # Flux Kustomization -> ./platform/oauth2-proxy (dependsOn platform-config)
+  # headlamp dashboard — POST-platform tier (dependsOn platform-config, wait:false). A leaf UI whose
+  # HelmRelease can crashloop (upstream headlamp-0.40.1 `-session-ttl` chart/image desync) must NOT
+  # sit in the gating `platform` tier, where its unhealth stalled platform-config + oauth2-proxy +
+  # every apps wave/vm on "dependency not ready" (DECISIONS-LOG: "Headlamp gating stall").
+  - headlamp.yaml        # Flux Kustomization -> ./platform/headlamp (dependsOn platform-config)
+  # Homepage apex landing dashboard — POST-platform tier (dependsOn platform-config, wait:false).
+  # A leaf UI at home.keithjackson.dev; same non-gating rationale as headlamp/oauth2-proxy (a UI must
+  # never gate the platform chain). Carries no SOPS secret, so clusters/dev adds no decryption patch.
+  - homepage.yaml        # Flux Kustomization -> ./platform/homepage (dependsOn platform-config)
+  # Crafty Controller — Minecraft management UI, web UI Keycloak-gated by a per-pod oauth2-proxy
+  # reverse-proxy sidecar (Option B). POST-platform tier (dependsOn platform-config, wait:false); same
+  # non-gating leaf-UI rationale as headlamp/homepage/oauth2-proxy (a UI must never gate the platform).
+  - crafty.yaml          # Flux Kustomization -> ./platform/crafty (dependsOn platform-config)
+  # Velero backup controller — POST-platform tier (dependsOn platform-config, wait:false). A backup
+  # controller must NEVER gate the platform chain; same non-gating rationale as headlamp/oauth2-proxy.
+  # Carries velero-cloud.sops.yaml, so clusters/dev adds a SOPS decryption patch for it (by name).
+  - backup.yaml          # Flux Kustomization -> ./platform/velero (dependsOn platform-config)
   - cluster-vars.yaml    # overlay knob surface (base defaults)

clusters/base/oauth2-proxy.yaml

@@ -0,0 +1,44 @@
+---
+# Flux Kustomization: oauth2-proxy forward-auth — reconciles ./platform/oauth2-proxy.
+#
+# POST-PLATFORM TIER (dependsOn platform-config). oauth2-proxy is a LEAF forward-auth service that
+# depends on Keycloak + the trusted wildcard TLS cert + split-horizon DNS — all of which are brought
+# up by `platform` (controllers) and `platform-config` (Keycloak instance/realm CRs, the wildcard
+# Certificate, cilium L2 VIP). It therefore MUST NOT live inside the gating `platform` tier: on a
+# cold start it Fails until its dependencies exist, and while it Failed it kept `platform` not-Ready,
+# so `platform-config` (which carries oauth2-proxy's own httproute/servicemonitor AND the Keycloak
+# instance it needs) could never reconcile — a self-gating deadlock that bit TWICE (DECISIONS-LOG
+# Session 7 + Session 9; roadmap follow-up "oauth2-proxy -> post-platform tier"). Moving it below
+# platform-config breaks the cycle: platform reaches Ready without it, platform-config brings up
+# Keycloak + TLS + DNS, THEN oauth2-proxy reconciles against dependencies that already exist.
+#
+# NON-GATING (wait:false): nothing in the graph dependsOn oauth2-proxy (apps waves gate on
+# platform-config, not on this), and a forward-auth pod can be legitimately long-not-Ready during a
+# cold start while it waits on Keycloak/DNS/TLS — same rationale as apps-vms. wait:false guarantees
+# its health can NEVER gate anything, even if a future resource is (mistakenly) pointed dependsOn it.
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: oauth2-proxy
+  namespace: flux-system
+spec:
+  interval: 10m
+  retryInterval: 2m
+  timeout: 5m
+  prune: true
+  wait: false
+  dependsOn:
+    - name: platform-config   # Keycloak instance/realm + wildcard TLS + split-horizon VIP exist first
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./platform/oauth2-proxy
+  # R-ENV-02: resolves ${OAUTH2_PROXY_REPLICAS} (dev 1 / prod 2) in the HelmRelease. Same cluster-vars
+  # ConfigMap the platform tier fed it before the move.
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-vars
+  # SOPS decryption for oauth2-proxy-credentials.sops.yaml is layered per-environment by the overlay
+  # (clusters/dev patches THIS Kustomization by name), identical to how it was decrypted via the
+  # `platform` Kustomization before the move. See clusters/dev/kustomization.yaml.

clusters/dev/kustomization.yaml

@@ -63,6 +63,62 @@ patches:
           provider: sops
           secretRef:
             name: sops-age
+  # oauth2-proxy moved to its own post-platform Flux Kustomization (clusters/base/oauth2-proxy.yaml).
+  # It carries oauth2-proxy-credentials.sops.yaml, previously decrypted via the `platform`
+  # Kustomization; now that this tier owns the encrypted Secret it needs its own decryption stanza.
+  - patch: |-
+      apiVersion: kustomize.toolkit.fluxcd.io/v1
+      kind: Kustomization
+      metadata:
+        name: oauth2-proxy
+        namespace: flux-system
+      spec:
+        decryption:
+          provider: sops
+          secretRef:
+            name: sops-age
+  # headlamp moved to its own post-platform Flux Kustomization (clusters/base/headlamp.yaml).
+  # It carries headlamp-oidc-credentials.sops.yaml, previously decrypted via the `platform`
+  # Kustomization; now that this tier owns the encrypted Secret it needs its own decryption stanza.
+  - patch: |-
+      apiVersion: kustomize.toolkit.fluxcd.io/v1
+      kind: Kustomization
+      metadata:
+        name: headlamp
+        namespace: flux-system
+      spec:
+        decryption:
+          provider: sops
+          secretRef:
+            name: sops-age
+  # crafty has its own post-platform Flux Kustomization (clusters/base/crafty.yaml). It carries
+  # crafty-oauth2-proxy-credentials.sops.yaml, so it needs its own SOPS decryption stanza (same as
+  # headlamp/oauth2-proxy). The `sops-age` Secret is created out-of-band (name only).
+  - patch: |-
+      apiVersion: kustomize.toolkit.fluxcd.io/v1
+      kind: Kustomization
+      metadata:
+        name: crafty
+        namespace: flux-system
+      spec:
+        decryption:
+          provider: sops
+          secretRef:
+            name: sops-age
+  # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml).
+  # It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS
+  # decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only).
+  - patch: |-
+      apiVersion: kustomize.toolkit.fluxcd.io/v1
+      kind: Kustomization
+      metadata:
+        name: velero
+        namespace: flux-system
+      spec:
+        decryption:
+          provider: sops
+          secretRef:
+            name: sops-age
   - patch: |-
       apiVersion: source.toolkit.fluxcd.io/v1
       kind: GitRepository
@@ -106,9 +162,17 @@ patches:
       - op: replace
         path: /data/MONITORING_PROMETHEUS_RETENTION
         value: "3d"
+      # 5Gi was too tight: 3d of this cluster's series volume (~1.2Gi/day of blocks) plus a WAL that
+      # ballooned past 1Gi overran the disk, wedging WAL commits and firing every absent()/==0 alert
+      # falsely. Bumped to 8Gi (still lighter than base 10Gi) so the size-cap below has real headroom.
       - op: replace
         path: /data/MONITORING_PROMETHEUS_PVC_SIZE
-        value: "5Gi"
+        value: "8Gi"
+      # Size-cap kept well below the 8Gi PVC (headroom for WAL + head + compaction scratch) so
+      # Prometheus prunes oldest blocks by size and never fills the disk on this young cluster.
+      - op: replace
+        path: /data/MONITORING_PROMETHEUS_RETENTION_SIZE
+        value: "5GiB"
       - op: replace
         path: /data/MONITORING_LOKI_PVC_SIZE
         value: "5Gi"

clusters/prod/kustomization.yaml

@@ -31,6 +31,11 @@ patches:
       - op: replace
         path: /data/MONITORING_PROMETHEUS_PVC_SIZE
         value: "50Gi"
+      # Size-cap below the 50Gi PVC (headroom for WAL + head + compaction scratch): prod may hit the
+      # size cap before 30d if series volume is high, keeping the disk from filling either way.
+      - op: replace
+        path: /data/MONITORING_PROMETHEUS_RETENTION_SIZE
+        value: "45GiB"
       - op: replace
         path: /data/MONITORING_LOKI_PVC_SIZE
         value: "50Gi"

conformance/README.md

@@ -30,9 +30,12 @@ behavioral / L3 fault-injection** (this harness).
 - `--requirements R-…` — **merge gate**: selects the tagged tests to run **and** asserts
   those touched requirements are covered (a listed safety-critical one must have an L3 test).
 - `--require-all` — **phase-exit gate**: every registry requirement must be covered.
-- Documented out-of-scope gaps (e.g. `R-STATE-02` node-loss, owned by WS-STORAGE) are
-  exempted from the completeness gates via `--allow-gap`, wired in `run.sh` as
-  `DOCUMENTED_GAPS` so the exemption is explicit and reviewable.
+- Documented out-of-scope gaps (e.g. `R-ENV-02` dev→prod promote and `R-GITOPS-01`
+  state-in-git, owned by WS-INFRA/WS-GITOPS and keyed wholesale by the coarse path map)
+  are exempted from the completeness gates via `--allow-gap`, wired in `run.sh` as
+  `DOCUMENTED_GAPS` so the exemption is explicit and reviewable. (`R-STATE-02` node-loss
+  used to be here; it now has a real L3 test — `tests/l3-fault-injection/node-loss` — and
+  is covered, not exempted.)
 
 `run.sh` is the **only** entrypoint. Local runs and the in-cluster CI Chainsaw Job call
 the *same* script with the same config, so results are identical.

conformance/incluster/rbac.yaml

@@ -88,13 +88,15 @@ rules:
   # namespaces policy is enforced, the `chainsaw-conf-*` test namespaces MUST be in
   # its exclusion set (documented, audited exception — R-SEC-02).
   #
-  # ---- R-STATE-02 (node loss): cordon/drain/uncordon a node  (NOT in WS-TEST scope yet)
-  # - apiGroups: [""]
-  #   resources: [nodes]
-  #   verbs: [get, list, patch]             # cordon = patch spec.unschedulable
-  # - apiGroups: [""]
-  #   resources: [pods/eviction]
-  #   verbs: [create]                       # drain
+  # ---- R-STATE-02 (node loss, tests/l3-fault-injection/node-loss): the fault is
+  # `kubectl cordon` + label of ONE node, reverted by `kubectl uncordon` + label removal
+  # in the test's `finally`. All three (cordon/uncordon/label) are a `patch` on the node
+  # object; `get`/`list` locate the victim + the finally cleanup selector. The test evicts
+  # only its OWN writer pod via `delete pod` (already granted above) — it deliberately does
+  # NOT `kubectl drain`, so no eviction rights and no reach over unrelated pods are needed.
+  - apiGroups: [""]
+    resources: [nodes]
+    verbs: [get, list, patch]             # cordon/uncordon/label = patch spec.unschedulable + metadata.labels
   #
   # ---- R-UPD-02 (bad-update auto-rollback): inject a bad image tag via automation
   # - apiGroups: [image.toolkit.fluxcd.io]

conformance/lib/coverage.py

@@ -30,7 +30,7 @@ Exit status / gating (R-TEST-03 gates are *scoped*, not "all 51 always"):
   --require-all    FULL completeness gate (phase-exit / final): fails if ANY registry
                    requirement is uncovered or any safety-critical one lacks an L3 test.
   --allow-gap IDs  exempt specific IDs from the completeness gates (documented, out-of-
-                   scope gaps such as R-STATE-02 node-loss owned by another workstream).
+                   scope gaps such as R-ENV-02/R-GITOPS-01 owned by another workstream).
 
 Pure standard library (no PyYAML) so the harness has no pip dependency: the registry and
 the test label/annotation lines we care about are parsed with line-oriented regexes, and

conformance/run.sh

@@ -49,7 +49,6 @@ FORCE_NO_CLUSTER=0
 # Documented, out-of-scope coverage gaps exempt from the completeness gate: requirements
 # owned by another workstream that WS-TEST does not yet scaffold. Listed here so the
 # exemption is explicit and reviewable, not silent.
-#   R-STATE-02 (node-loss)         — delivered with Longhorn/infra (WS-STORAGE).
 #   R-ENV-02   (dev->prod promote) — owned by WS-INFRA/WS-GITOPS; unscaffolded (a GAP in
 #                                    coverage.md). The coarse path->requirement map keys
 #                                    clusters/* AND ci/* to it and the R-ENV-02 doc token
@@ -71,8 +70,12 @@ FORCE_NO_CLUSTER=0
 # authoritative signal is the `conformance.home.k3s/R-*` label on a test, which still is.
 # The PATH-MAP half remains: ci/touched-requirements.sh keys clusters/* and ci/* to
 # R-ENV-02 + R-GITOPS-01 wholesale, which is why those two must stay exempted here. Tighten
-# the path map (or scaffold the two requirements) and this list shrinks to R-STATE-02.
-DOCUMENTED_GAPS="R-STATE-02 R-ENV-02 R-GITOPS-01"
+# the path map (or scaffold the two requirements) and this list empties.
+#
+# R-STATE-02 (node-loss) WAS listed here (owned by WS-STORAGE, "delivered with Longhorn/infra")
+# but now has a real verifying L3 test — tests/l3-fault-injection/node-loss — so it is no
+# longer a gap and has been removed from this list; the coverage report counts it as covered.
+DOCUMENTED_GAPS="R-ENV-02 R-GITOPS-01"
 
 usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }
 
@@ -154,7 +157,7 @@ select_dirs() {
 #     DIRECTORY basename (stable); override via CONF_HEAVY_TESTS to add/remove without code
 #     changes. Listed here even for tests still scaffolded (vm-lifecycle, dr-restore, ...) so
 #     the serialization is already correct the moment they grow real KubeVirt/Longhorn bodies.
-CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore}"
+CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore node-loss}"
 
 conf_is_heavy_dir() {
   # $1 = test dir; true if its basename is in the heavy set.

conformance/tests/l3-fault-injection/node-loss/chainsaw-test.yaml

@@ -0,0 +1,210 @@
+# =============================================================================
+# L3 FAULT-INJECTION — Volume survives node loss  (R-STATE-02)  [RUNNABLE / self-skips]
+#                      *SAFETY-CRITICAL*
+# Catalog: docs/14 "Node loss (R-STATE-02, L3)"
+# Spec:    docs/04-storage-and-state.md ("Longhorn — primary persistent storage";
+#          Default StorageClass = 3 replicas so any single node loss keeps data available)
+#
+# GIVEN a 3-replica Longhorn volume with written data, attached on some node N,
+# WHEN  node N is REMOVED from service (cordoned + the attached workload evicted off it,
+#       forcing the volume to detach from N),
+# THEN  the volume stays AVAILABLE — a fresh workload scheduled on a SURVIVING node
+#       re-attaches the same volume (served from its replicas on the other nodes) and
+#       reads the data intact — and the cluster reconverges.
+#
+# R-TEST-05 (fault-injection, not happy-path): the availability guarantee is proven by
+# actually taking a node out from under the attached volume and asserting the data is
+# still served from a different node — not by re-reading the still-attached volume.
+#
+# SAFE-BY-CONSTRUCTION (small 3-node embedded-etcd dev cluster):
+#   * Disruption is scoped to EXACTLY ONE node, and only at the SCHEDULING layer:
+#     `kubectl cordon` (spec.unschedulable=true) + eviction of THIS test's own workload
+#     pod. The node's kubelet/etcd member stay UP — etcd quorum is never touched, so this
+#     cannot starve embedded-etcd I/O the way a real power-off / NotReady node can. That
+#     harsher "node truly down" drill is an OPERATOR-run exercise for a maintenance window,
+#     NOT this automated test (see report).
+#   * We evict ONLY our own `writer` pod (delete), never `kubectl drain` — so no unrelated
+#     workload or system pod on that node is disturbed.
+#   * The victim node is labelled `conformance.home.k3s/cordoned-by=<this-test-namespace>`
+#     at cordon time; the `finally` step ALWAYS uncordons every node carrying THAT unique
+#     label and removes it — so the node is restored to schedulable even if the test fails,
+#     panics, or times out, and we never uncordon a node an operator cordoned deliberately.
+#
+# RUNNABLE / SELF-SKIPPING: skips CLEANLY (green) until BOTH a `longhorn` StorageClass
+# exists AND the cluster has >= 3 Ready nodes (a 3-replica volume + one-node-loss headroom
+# is not meaningful otherwise); runs the full node-loss survival check the moment it does —
+# no edit needed. Hermetic: PVC + pods live in the chainsaw per-test namespace and are torn
+# down with it; the only out-of-namespace mutation (the node cordon/label) is reverted in
+# `finally`.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: node-loss
+  labels:
+    conformance.home.k3s/R-STATE-02: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-STATE-02"
+    conformance.home.k3s/layer: "L3"
+    conformance.home.k3s/safety-critical: "true"
+    # executable => runs against a cluster; self-skips (green) until longhorn + 3 nodes exist.
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      Take one node out of service (cordon + evict the attached workload) under a 3-replica
+      Longhorn volume; a fresh workload on a surviving node re-attaches the volume and reads
+      the data intact — the volume stays available and the cluster reconverges. Skips cleanly
+      until a longhorn StorageClass and >= 3 Ready nodes exist.
+spec:
+  # Longhorn detach-from-victim + re-attach-on-survivor (rebuild/replica scheduling) is far
+  # slower than a same-node pod restart; give the eventual-consistency waits real headroom.
+  timeouts:
+    exec: 20m
+  steps:
+    - name: volume-stays-available-across-node-loss
+      try:
+        - script:
+            content: |
+              set -euo pipefail
+              NS="$NAMESPACE"
+
+              # -- Gate 1: skip cleanly (green) if the longhorn tier is not present yet. --
+              if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
+                echo "SKIP: 'longhorn' StorageClass absent — node-loss not applicable on this cluster yet"
+                echo NODE_LOSS_SKIPPED_OK
+                exit 0
+              fi
+
+              # -- Gate 2: a 3-replica volume + single-node-loss headroom needs >= 3 Ready nodes. --
+              READY="$(kubectl get nodes \
+                -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \
+                | grep -c '^True$' || true)"
+              echo "ready nodes: ${READY}"
+              if [ "${READY:-0}" -lt 3 ]; then
+                echo "SKIP: only ${READY:-0} Ready node(s) (<3) — node-loss resilience not meaningful yet"
+                echo NODE_LOSS_SKIPPED_OK
+                exit 0
+              fi
+
+              CANARY="node-loss-canary-$(date +%s%N)"
+
+              # -- GIVEN: a Longhorn PVC (platform default SC = 3 replicas across nodes) + a
+              #    writer pod that writes the canary and flushes it to the replicated volume. --
+              kubectl -n "$NS" apply -f - <<EOF
+              apiVersion: v1
+              kind: PersistentVolumeClaim
+              metadata:
+                name: survivor
+              spec:
+                accessModes: [ReadWriteOnce]
+                storageClassName: longhorn
+                resources:
+                  requests:
+                    storage: 1Gi
+              EOF
+
+              apply_pod() {
+                kubectl -n "$NS" apply -f - <<EOF
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: $1
+              spec:
+                restartPolicy: Never
+                containers:
+                  - name: app
+                    image: busybox:1.36.1
+                    command: ["sh", "-c", "sleep 3600"]
+                    volumeMounts:
+                      - name: data
+                        mountPath: /data
+                volumes:
+                  - name: data
+                    persistentVolumeClaim:
+                      claimName: survivor
+              EOF
+              }
+
+              apply_pod writer
+              kubectl -n "$NS" wait --for=condition=Ready pod/writer --timeout=240s
+              printf '%s' "$CANARY" | kubectl -n "$NS" exec -i writer -- sh -c 'cat > /data/canary && sync'
+              echo "wrote canary: $(kubectl -n "$NS" exec writer -- cat /data/canary)"
+
+              # The victim node is the one the attached volume currently lives on.
+              VICTIM="$(kubectl -n "$NS" get pod writer -o jsonpath='{.spec.nodeName}')"
+              [ -n "$VICTIM" ] || { echo "FAIL: could not determine writer's node" >&2; exit 1; }
+              echo "victim node (currently hosts the attached volume): $VICTIM"
+
+              # Best-effort precondition log: if the Longhorn Volume CR is readable, show the
+              # volume is genuinely 3-replica/healthy before we take a node. Non-fatal (the
+              # runner is least-privilege and may not have longhorn.io read rights) — the
+              # data-availability assertion below is the authoritative proof, not this.
+              PVNAME="$(kubectl -n "$NS" get pvc survivor -o jsonpath='{.spec.volumeName}' 2>/dev/null || true)"
+              if [ -n "$PVNAME" ]; then
+                kubectl -n longhorn-system get volumes.longhorn.io "$PVNAME" \
+                  -o jsonpath='pre-fault robustness={.status.robustness} replicas={.spec.numberOfReplicas}{"\n"}' \
+                  2>/dev/null || echo "(Longhorn Volume CR not readable — continuing; data read is the proof)"
+              fi
+
+              # -- WHEN: FAULT INJECTION — remove the victim node FROM SERVICE. --
+              #    Label it with THIS test's unique namespace so `finally` can always find and
+              #    uncordon exactly this node; then cordon it; then evict ONLY our own writer
+              #    pod, forcing the volume to detach from the victim. (No `drain` — unrelated
+              #    pods on the node are never touched.)
+              kubectl label node "$VICTIM" "conformance.home.k3s/cordoned-by=$NS" --overwrite
+              kubectl cordon "$VICTIM"
+              echo "cordoned + labelled victim node: $VICTIM"
+              kubectl -n "$NS" delete pod writer --wait=true --timeout=240s
+              echo "evicted writer off the victim node; volume detached from $VICTIM"
+
+              # -- THEN: a fresh workload re-binds the SAME volume. The victim is unschedulable,
+              #    so the scheduler MUST place the reader on a SURVIVING node; Longhorn re-attaches
+              #    the volume there from its replicas on the other nodes. --
+              apply_pod reader
+              kubectl -n "$NS" wait --for=condition=Ready pod/reader --timeout=360s
+              READER_NODE="$(kubectl -n "$NS" get pod reader -o jsonpath='{.spec.nodeName}')"
+              echo "reader scheduled on surviving node: $READER_NODE (victim was $VICTIM)"
+              if [ "$READER_NODE" = "$VICTIM" ]; then
+                echo "FAIL: reader landed back on the cordoned victim node — node was not removed from service" >&2
+                exit 1
+              fi
+
+              GOT="$(kubectl -n "$NS" exec reader -- cat /data/canary)"
+              echo "recovered='$GOT' expected='$CANARY'"
+              [ "$GOT" = "$CANARY" ] || { echo "FAIL: data not available after node loss" >&2; exit 1; }
+              echo "volume available + data intact from a surviving node"
+
+              # Best-effort reconvergence log: if the Longhorn Volume CR is readable, wait
+              # (bounded) for robustness to settle back to available (healthy|degraded — both
+              # serve I/O; a healthy 3-replica volume rebuilds a replica once headroom exists).
+              # Non-fatal for the same least-privilege reason as the pre-fault log.
+              if [ -n "$PVNAME" ]; then
+                for i in $(seq 1 30); do
+                  ROB="$(kubectl -n longhorn-system get volumes.longhorn.io "$PVNAME" \
+                        -o jsonpath='{.status.robustness}' 2>/dev/null || true)"
+                  [ -z "$ROB" ] && { echo "(Longhorn Volume CR not readable — skipping reconvergence log)"; break; }
+                  echo "post-fault robustness[$i]=$ROB"
+                  case "$ROB" in healthy|degraded) break ;; esac
+                  sleep 10
+                done
+              fi
+
+              echo NODE_LOSS_VOLUME_AVAILABLE_OK
+            check:
+              # green on either a clean skip (no longhorn / <3 nodes) or a proven survival.
+              (contains($stdout, 'NODE_LOSS_VOLUME_AVAILABLE_OK') || contains($stdout, 'NODE_LOSS_SKIPPED_OK')): true
+      finally:
+        # SAFETY FIRST: always restore the victim node to schedulable, even on failure/timeout.
+        # Keyed by this test's unique namespace label so we touch ONLY the node we cordoned.
+        - script:
+            content: |
+              set -uo pipefail
+              for n in $(kubectl get nodes -l "conformance.home.k3s/cordoned-by=$NAMESPACE" -o name 2>/dev/null); do
+                kubectl uncordon "${n#node/}" || true
+                kubectl label "$n" "conformance.home.k3s/cordoned-by-" || true
+                echo "restored node to service: ${n#node/}"
+              done
+        - script:
+            content: |
+              kubectl -n "$NAMESPACE" delete pod writer reader --ignore-not-found --wait=false || true
+              kubectl -n "$NAMESPACE" delete pvc survivor --ignore-not-found --wait=false || true
+              echo "cleanup best-effort; chainsaw namespace teardown finalizes"

docs/plans/ssd-etcd-execution-runbook.md

@@ -0,0 +1,97 @@
+# Phase 2 execution runbook — SSD-backed etcd migration (interactive/manual)
+
+Companion to `ssd-etcd-isolation.md`. This is the **exact command sequence** to migrate
+each k3s node's etcd onto its dedicated SSD `virtio2` disk. It must be run where live-infra
+mutations are permitted — an **interactive `claude` session** (default permission mode, which
+prompts for approval) or **by hand**. The autonomous "auto mode" session hard-denies
+`tofu apply` and node-level ssh mutations, so it cannot run this.
+
+## Preconditions (verified 2026-08-29, cluster healthy)
+- Run everything from the staged worktree that carries the tofu disk block:
+  `cd /home/jakman85/source/home-network-k3s-impl/.claude/worktrees/agent-ssd-etcd`
+  (branch `feature/ssd-etcd-isolation` @ `af5fc5b`).
+- All tooling via the devcontainer: `./infra/devcontainer/dev.sh run <cmd>`.
+- `./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init` once (done).
+- Ansible must run from `infra/ansible/` (for the vault-pass path): wrap as
+  `./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible ...'`.
+- **Run mutation commands STANDALONE** (not chained with `&&`/pipes) — the classifier is
+  more likely to deny a compound command that embeds a mutation.
+
+## ORDER (leader last — RE-VERIFY before each node)
+Current leader = **dev-3**, so: **dev-2 (vmid 102) → dev-1 (vmid 105) → dev-3 (vmid 101)**.
+dev-3 is last and additionally gets the memory 16384→24576 bump (needs a full stop/start).
+Re-check the leader before each node and keep the leader last:
+```
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s_servers -m shell -a "curl -s http://127.0.0.1:2381/metrics | grep etcd_server_is_leader"'
+```
+
+## Pre-flight before EACH node (read-only — all must pass)
+```
+./infra/devcontainer/dev.sh run kubectl get nodes
+./infra/devcontainer/dev.sh run kubectl get --raw=/readyz/etcd            # expect: ok
+./infra/devcontainer/dev.sh run kubectl -n longhorn-system get volumes.longhorn.io -o custom-columns=R:.status.robustness --no-headers | grep -c healthy   # expect: 18
+```
+
+## Per-node procedure (substitute NODE=k3s-dev-N, VMID)
+
+### 1. Freeze rebuilds (STANDALONE)
+```
+./infra/devcontainer/dev.sh run kubectl -n longhorn-system patch settings.longhorn.io concurrent-replica-rebuild-per-node-limit --type=merge -p '{"value":"0"}'
+```
+
+### 2. Add the SSD disk via tofu (STANDALONE). dev-2/dev-1 = disk only; dev-3 also memory.
+```
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh plan  '-target=proxmox_virtual_environment_vm.node["k3s-dev-N"]'   # confirm: disk ADD, in-place, NOT replace
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh apply -auto-approve '-target=proxmox_virtual_environment_vm.node["k3s-dev-N"]'
+```
+Confirm the guest sees the new disk (hot-plug):
+```
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -m shell -a "lsblk /dev/vdc"'
+```
+If `/dev/vdc` is NOT present (no hot-plug), power-cycle the VM (freeze is holding):
+```
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible proxmox -m shell -a "qm stop VMID"'   # wait until stopped
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible proxmox -m shell -a "qm start VMID"'
+```
+(For dev-3 the memory bump requires this stop/start regardless.)
+
+### 3. Migrate etcd data (node commands, become=root). Do NOT skip the copy-before-mount order.
+```
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -b -m shell -a "systemctl stop k3s"'
+# mkfs ONLY if /dev/vdc has no filesystem yet (fresh disk):
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -b -m shell -a "blkid /dev/vdc || mkfs.ext4 -L k3s-etcd /dev/vdc"'
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -b -m shell -a "mv /var/lib/rancher/k3s/server/db /var/lib/rancher/k3s/server/db.bak && mkdir /var/lib/rancher/k3s/server/db && mount LABEL=k3s-etcd /var/lib/rancher/k3s/server/db && rsync -aHAX --numeric-ids /var/lib/rancher/k3s/server/db.bak/ /var/lib/rancher/k3s/server/db/"'
+# Persist the mount (matches the node_baseline role: LABEL + defaults,noatime):
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -b -m ansible.posix.mount -a "path=/var/lib/rancher/k3s/server/db src=LABEL=k3s-etcd fstype=ext4 opts=defaults,noatime dump=0 passno=2 state=mounted"'
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -b -m shell -a "systemctl start k3s"'
+```
+
+### 4. VERIFY full recovery before touching the next node (all must pass)
+```
+./infra/devcontainer/dev.sh run kubectl get nodes                          # NODE Ready
+./infra/devcontainer/dev.sh run kubectl get --raw=/readyz/etcd             # ok
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -m shell -a "findmnt /var/lib/rancher/k3s/server/db"'   # source = /dev/vdc
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s_servers -m shell -a "curl -s http://127.0.0.1:2381/metrics | grep etcd_server_has_leader"'  # all =1
+./infra/devcontainer/dev.sh run kubectl -n longhorn-system get volumes.longhorn.io -o custom-columns=R:.status.robustness --no-headers | grep -c healthy   # 18
+# dev-3 only: confirm ~24Gi allocatable
+./infra/devcontainer/dev.sh run kubectl get node k3s-dev-3 -o jsonpath='{.status.allocatable.memory}'
+```
+
+### 5. Thaw + clean up (STANDALONE), only after step 4 fully passes
+```
+./infra/devcontainer/dev.sh run kubectl -n longhorn-system patch settings.longhorn.io concurrent-replica-rebuild-per-node-limit --type=merge -p '{"value":"1"}'
+./infra/devcontainer/dev.sh run bash -lc 'cd infra/ansible && ansible k3s-dev-N -b -m shell -a "rm -rf /var/lib/rancher/k3s/server/db.bak"'
+```
+Wait for 18/18 healthy (no rebuild storm), re-verify leader, then proceed to the next node.
+
+## STOP conditions (halt immediately, leave limit=0, do not advance)
+etcd `/readyz` not ok, a member without a leader, VIP `192.168.2.50:6443` flapping, a node
+not rejoining, any Longhorn volume Faulting, or k3s crashlooping. Recovery: keep rebuilds
+frozen; a botched single member can be reset and re-joined from the other two (plan §4.5).
+
+## Close-out (after all 3)
+- All 3 nodes Ready; etcd 3/3 with a leader; `findmnt` shows `/dev/vdc` at the db path on each.
+- 18/18 Longhorn volumes healthy; rebuild-limit back to **1**.
+- `./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh plan` → **No changes** (all disks present, dev-3 memory reconciled).
+- Merge `feature/ssd-etcd-isolation` → `main` and push (push is classifier-blocked in auto mode — run `git push gitea main` by hand) so Flux/tofu source records the disk + baseline.
+- Regression test: after approval, bounce ONE node and confirm etcd stays healthy / VIP steady — the whole point of the change.

docs/plans/ssd-etcd-isolation.md

@@ -0,0 +1,270 @@
+# SSD-backed etcd isolation for the k3s dev cluster
+
+**Status:** PROPOSAL — awaiting user approval. No changes have been made.
+**Author:** investigation agent (read-only), 2026-08-29
+**Problem:** Longhorn replica-rebuild I/O storms on the HDD-backed VM disks starve
+etcd's WAL fsync, crash-looping k3s and flapping the kube-vip VIP
+(`192.168.2.50:6443`). Even `concurrent-replica-rebuild-per-node-limit=1` tips etcd.
+**Question:** Can the Proxmox host's ~1 TB system SSD (`rpool`) host dedicated
+SSD-backed storage for etcd (and/or Longhorn) to end these incidents?
+
+**Verdict: YES for etcd (strongly recommended). NO for Longhorn.**
+
+---
+
+## 1. Measured facts (all collected read-only on 2026-08-29)
+
+### 1.1 `rpool` — the system/boot SSD pool
+
+| Property | Value |
+|---|---|
+| Pool | `rpool`, **mirror** (`mirror-0`) of 2 devices |
+| Devices | `ata-MTFDDAK960TDS_...` × 2 — **Micron 5100/52x0 960 GB, 2.5" SATA** |
+| Class | **Enterprise SATA SSD with power-loss protection (PLP)** — safe for sync/fsync |
+| Health | `Percent_Lifetime_Remain` = **97%** (3% used), 13 904 power-on hours, 0 reallocated sectors, ~96 TB written. Excellent. |
+| Pool size / alloc / **free** | 888 G / 538 G / **350 G free** (`zpool list`, 60% cap) |
+| ZFS `AVAIL` | **~323 GiB** available to any new dataset/zvol |
+| System root | `rpool/ROOT/pve-1` = **430 G used** (Proxmox host `/`) |
+| Proxmox storage on it | `local-zfs` → pool `rpool/data`, `content images,rootdir`, **`sparse 1`** (thin), ~338 GiB available per `pvesm status` |
+| Pool settings | `compression on`, **`sync standard`** (inherited), `dedup off`, recordsize 128K |
+
+`local-zfs` already advertises `images,rootdir` content, so **Proxmox can place VM
+disks on the SSD today with no storage-config change.**
+
+### 1.2 Current VM disk backing — confirms the HDD assumption
+
+VMs (bpg/OpenTofu-provisioned): **dev-1 = vmid 105, dev-2 = 102, dev-3 = 101**.
+Each has two disks, **both on the `Storage` datastore**:
+
+- `Storage` = ZFS pool on **2× WDC WUH721818ALE6L4 18 TB HDD** (`sdc`/`sdd`, `ROTA=1`), mirror, 16.4 T.
+- Per node: `virtio0 = vm-<id>-disk-0` (64 G, OS/root) and `virtio1 = vm-<id>-disk-1` (128 G, Longhorn data). `volblocksize 16K`.
+
+So **etcd, the OS, and Longhorn all currently sit on spinning rust.** That is the
+root cause: etcd WAL fsync competes with Longhorn rebuild I/O on the same HDD spindles.
+
+(`Storage2` = 2× Seagate ST8000VN004 8 TB HDD is a separate HDD pool, not involved.)
+
+### 1.3 Size of the hot data
+
+- **etcd DB: ~82.6 MB.** Measured via apiserver metric
+  `apiserver_storage_size_bytes{storage_cluster_id="etcd-0"} = 8.264e7`.
+  A 4-day-old cluster running ~18 services. Even allowing generous WAL + growth
+  headroom, **etcd fits in single-digit GB with room to spare — trivially ×3.**
+- **Longhorn: ~22 GB actual data per node right now**, but the data disk is
+  **128 GiB** and Longhorn reports `storageScheduled ≈ 152 GiB` (thin-overcommitted)
+  vs `storageMaximum ≈ 125 GiB` per node. 18 PVCs, `numberOfReplicas=3`.
+  Provisioned total ≈ 152 GiB. Longhorn is the **churn source we want to isolate
+  etcd from**, and it can grow toward 128 GiB/node.
+
+### 1.4 Sizing conclusion
+
+| Candidate | Per-node footprint | ×3 on rpool | Fits in 323 GiB avail? |
+|---|---|---|---|
+| **etcd** | ~83 MB now; size a 16 GiB disk | ~48 GiB reserved (thin) | **Yes, easily** |
+| **Longhorn** | ~22 GB now, up to 128 GiB provisioned | up to **~384 GiB** | **No — would fill/threaten the boot disk** |
+
+---
+
+## 2. Viability verdict
+
+**etcd on SSD: viable and high-leverage.** etcd is ~83 MB; three 16 GiB SSD-backed
+disks cost ~48 GiB of thin allocation (actual ~250 MB) against 323 GiB free on an
+enterprise, PLP-equipped, 97%-healthy SSD mirror. This directly removes the WAL
+fsync path from the HDD contention domain — the smallest-footprint fix for the exact
+failure mode. **This is the recommendation.**
+
+**Longhorn on SSD: not viable here.** Its provisioned footprint (~384 GiB across 3
+replicas at full size) does not fit alongside the 430 GiB Proxmox root on a 888 GiB
+pool, and co-locating heavy Longhorn rebuild I/O on the boot SSD reintroduces
+contention with the host itself. Keep Longhorn on HDD (`Storage`). Rejected.
+
+---
+
+## 3. Options evaluated
+
+- **Option A — dedicated small SSD disk per node, etcd-only (RECOMMENDED).**
+  Add a ~16 GiB SSD-backed virtual disk (zvol on `rpool/data` via `local-zfs`) to
+  each of the 3 VMs, mount it at the k3s etcd directory
+  (`/var/lib/rancher/k3s/server/db`), migrate etcd data onto it one node at a time.
+  Isolates WAL fsync from HDD Longhorn churn. Minimal space, minimal blast radius.
+- **Option B — Longhorn replicas/disk on SSD.** Rejected (§2): does not fit and
+  re-couples host I/O.
+- **Option C — do nothing; wait for the SSD mini-PCs.** Zero risk, zero work, but
+  leaves the cluster fragile to every node bounce until the mini-PC migration lands
+  (deferred, no committed date). Option A is a cheap, reversible bridge that Option C
+  eventually supersedes (§7).
+
+**Recommendation: Option A now, as an interim fix, carried by tofu.**
+
+---
+
+## 4. Recommended implementation — Option A, step-by-step
+
+### 4.1 Model it in tofu (preferred over manual)
+
+The VMs are bpg/OpenTofu-managed (`infra/tofu/`). Disks are declared as `disk`
+blocks in `main.tf`; both current disks (`virtio0`, `virtio1`) point at
+`var.datastore` (`Storage`). Add a **third disk on the SSD datastore**:
+
+1. **New variables** in `infra/tofu/variables.tf`:
+   ```hcl
+   variable "etcd_ssd_datastore" {
+     description = "SSD-backed datastore for the dedicated etcd disk (VERIFIED: local-zfs = rpool/data, ZFS mirror on Micron 5100 SATA SSDs w/ PLP)."
+     type        = string
+     default     = "local-zfs"
+   }
+   variable "etcd_disk_gib" {
+     description = "Dedicated SSD etcd disk per node, GiB (etcd DB is ~83 MB; 16 GiB is generous headroom)."
+     type        = number
+     default     = 16
+   }
+   ```
+2. **New disk block** in the VM resource in `main.tf`, appended AFTER the existing
+   `virtio1` block (keep interface order ascending so bpg does not reorder):
+   ```hcl
+   # Dedicated SSD etcd disk (virtio2 -> /dev/vdc). Isolates etcd WAL fsync from
+   # the HDD Storage pool where the OS + Longhorn disks live.
+   disk {
+     datastore_id = var.etcd_ssd_datastore
+     interface    = "virtio2"
+     size         = var.etcd_disk_gib
+     discard      = "on"
+     cache        = "none"   # honor guest fsync through to ZFS ZIL (PLP SSD = safe+fast)
+     iothread     = false
+   }
+   ```
+3. **Plan and inspect before applying.** `tofu plan` MUST show an **in-place update
+   (disk add)** for the targeted VM — NOT a replacement. If it shows replace, stop and
+   reassess (do not destroy a running etcd node).
+4. **Apply one node at a time** with `-target` (see §4.3 for quorum ordering), e.g.
+   `-target='proxmox_virtual_environment_vm.node["k3s-dev-3"]'` (adjust to the actual
+   resource address in this repo). A disk add is applied online by bpg, but treat the
+   node as going down for the etcd migration regardless.
+
+> **Coupling caveat — dev-3 memory drift.** dev-1/dev-2 are at `memory_mib = 24576`
+> but **dev-3 is live at 16384** (drift from tofu default 24576). A tofu apply that
+> touches dev-3 will also try to reset its memory to 24576. Per the Proxmox rule,
+> memory changes need a **full stop/start**, not a reboot. Decide deliberately:
+> either (a) let the same maintenance window fix the memory drift on dev-3 (stop →
+> set → start), or (b) `-target` narrowly and accept that dev-3's plan will include
+> the memory change. Do dev-3 in its own window with eyes on it.
+
+**Manual alternative (if not using tofu):** on the Proxmox host,
+`qm set <vmid> -virtio2 local-zfs:16,discard=on,cache=none` then stop/start the VM.
+This drifts from tofu state and is discouraged — if done, import/reconcile into tofu
+afterward. Prefer the tofu path.
+
+### 4.2 Optional space guarantee
+
+`local-zfs` is `sparse` (thin), so the etcd zvols consume only real usage. To
+guarantee etcd can never be squeezed if the system disk fills, set a modest
+`refreservation` on each etcd zvol (e.g. `zfs set refreservation=16G
+rpool/data/vm-<id>-disk-2`). 3×16 GiB = 48 GiB reserved out of 323 GiB avail — safe.
+This is a host-side ZFS tweak, not expressible in bpg; apply manually if desired.
+
+### 4.3 Migrate etcd onto the new disk — quorum-safe, ONE node at a time
+
+k3s has no flag to relocate only the etcd directory (`--data-dir` moves everything),
+so the clean approach is to **mount the SSD disk at
+`/var/lib/rancher/k3s/server/db`** (this directory holds `etcd/` — member data, WAL,
+snapshots — and nothing else that matters).
+
+**Pre-flight:** confirm all 3 etcd members healthy and the cluster is quiet (no active
+Longhorn rebuild). `kubectl get nodes` all Ready; check etcd member health.
+
+**Per node, repeat for dev-3, then dev-2, then dev-1 (leader last):**
+
+1. Verify 3/3 etcd members healthy BEFORE touching this node (losing one of three
+   keeps quorum at 2/3; never take a second node until the previous is fully back).
+2. Stop k3s on the node: `systemctl stop k3s`. (Node leaves quorum; 2/3 remain.)
+3. Identify the new disk (`/dev/vdc` — the third virtio disk) and format it:
+   `mkfs.ext4 -L k3s-etcd /dev/vdc` (ext4 or xfs; ext4 is fine and simple).
+4. Preserve and move existing data:
+   ```
+   mv /var/lib/rancher/k3s/server/db /var/lib/rancher/k3s/server/db.bak
+   mkdir /var/lib/rancher/k3s/server/db
+   mount /dev/vdc /var/lib/rancher/k3s/server/db
+   rsync -aHAX --numeric-ids /var/lib/rancher/k3s/server/db.bak/ /var/lib/rancher/k3s/server/db/
+   ```
+5. Persist the mount in `/etc/fstab` **by UUID** (`blkid /dev/vdc`), e.g.
+   `UUID=<uuid> /var/lib/rancher/k3s/server/db ext4 defaults,noatime 0 2`, then
+   `systemctl daemon-reload`.
+6. Start k3s: `systemctl start k3s`. Wait until the node is `Ready` AND its etcd
+   member is healthy and has re-joined (member list shows 3 started, no learner
+   stuck). Confirm the VIP `192.168.2.50:6443` is stable.
+7. Only after full recovery, remove the backup: `rm -rf
+   /var/lib/rancher/k3s/server/db.bak`. Then proceed to the next node.
+
+**Reproducibility for rebuilds:** the SSD disk + mount should also be baked into the
+node **cloud-init first-boot baseline** (the same template that already provisions the
+Longhorn `virtio1` disk and swap-off), so a `tofu`-rebuilt node comes up with
+`/dev/vdc` formatted and mounted at the etcd path automatically. Update that template
+in the same change set as the tofu disk block.
+
+### 4.4 Validation
+
+- `kubectl get nodes` — all 3 Ready; no flapping over a 30–60 min soak.
+- etcd: 3 members, all started/healthy, one leader; DB size sane (~tens–hundreds MB).
+- On each node, `df -h /var/lib/rancher/k3s/server/db` shows the new `/dev/vdc` mount.
+- **Regression test the original failure:** with the cluster healthy, trigger a
+  controlled Longhorn replica rebuild (e.g. bounce ONE node after approval) and
+  confirm etcd stays healthy and the VIP does not flap — the whole point of the change.
+- Host-side: `zpool status rpool` clean; `zfs list rpool/data` shows the 3 new
+  `vm-*-disk-2` zvols with small real usage.
+
+### 4.5 Rollback
+
+Per node, fully reversible while `db.bak` still exists (before step 7):
+`systemctl stop k3s` → `umount .../server/db` → `rmdir` the empty mountpoint →
+`mv db.bak db` → remove the fstab line → `systemctl start k3s`. To fully revert the
+disk, detach `virtio2` (`qm set <vmid> -delete virtio2` after stop) or `tofu` revert
+the disk block, then stop/start. Because etcd is a replicated 3-member cluster, a
+botched single node can also simply be reset and re-joined from the other two.
+
+---
+
+## 5. Risks
+
+1. **System-disk contention / exhaustion (top risk).** `rpool` is also the Proxmox
+   boot/root disk (430 GiB used of 888 GiB). Mitigations: keep the etcd disks tiny
+   (16 GiB, thin), optional `refreservation` (§4.2), and **never** put Longhorn here
+   (§2). etcd's steady-state real usage is ~hundreds of MB; the marginal I/O it adds
+   to the SSD is negligible next to what it removes from the HDD.
+2. **Consumer-SSD fsync/PLP caveat — N/A here, but assert it.** etcd correctness under
+   power loss depends on honest fsync. These are **Micron 5100/52x0 enterprise SATA
+   SSDs with power-loss protection**, so `sync=standard` + `cache=none` is both fast
+   and durable. (If the disks were consumer SSDs without PLP this concern would be
+   real — it is not, for this hardware.)
+3. **Quorum during migration.** Mitigated by strict one-node-at-a-time with full
+   recovery gates (§4.3) and leader-last ordering. Never take a second node down
+   before the previous rejoins.
+4. **tofu coupling with dev-3 memory drift + apply semantics (§4.1).** An apply
+   touching dev-3 also resets memory 16384→24576 (needs stop/start). Verify `tofu
+   plan` shows disk *add*, not VM *replace*, before applying. Use `-target`.
+5. **Backup/restore assumptions.** k3s etcd snapshots (`/var/lib/rancher/k3s/server/db/snapshots`)
+   now live on the SSD too — fine, but confirm any snapshot/backup job still points at
+   the (unchanged) path.
+
+---
+
+## 6. Recommendation summary
+
+Implement **Option A**: a 16 GiB SSD-backed `virtio2` disk per k3s VM (zvol on
+`rpool/data` via `local-zfs`), etcd-only, migrated one node at a time with the
+quorum-safe procedure in §4.3, modeled in tofu (+ cloud-init baseline) for
+reproducibility. Do **not** move Longhorn to SSD. This is a small, reversible,
+high-leverage fix targeting the exact WAL-starvation failure.
+
+---
+
+## 7. Relationship to the deferred SSD-mini-PC migration
+
+This is explicitly an **interim bridge** on the current Proxmox dev host
+(`192.168.2.102`), whose bulk storage is HDD. Once the k3s cluster migrates to the
+planned **all-SSD mini-PCs**, every I/O path (OS, etcd, Longhorn) is already on SSD
+and the HDD-contention failure mode disappears — the dedicated etcd disk becomes
+unnecessary. At that point the isolation can be either dropped or carried forward
+harmlessly (a dedicated etcd disk is still good hygiene). Because Option A is captured
+in tofu + cloud-init, it travels with the cluster definition and can be removed in one
+reviewed change when the mini-PCs land. Until then it stops the recurring incidents
+at minimal cost.

infra/ansible/inventory/group_vars/proxmox_hosts/README.md

@@ -0,0 +1,27 @@
+# group_vars for the `proxmox_hosts` group (the Proxmox hypervisor host)
+
+Target of `playbooks/backup-s3.yml` (role `s3_backup_target`, versitygw). The role needs the S3
+backup-target credentials as **ansible-vault-encrypted** vars here — encrypted, so safe to commit;
+the passphrase that decrypts them stays out of the repo (`ansible.cfg` → `vault_password_file`).
+
+## `vault.yml` (committed, encrypted)
+
+```bash
+cd infra/ansible
+ansible-vault edit inventory/group_vars/proxmox_hosts/vault.yml
+# contents:
+#   s3_backup_target_access_key: "k3s-backup"   # versitygw ROOT key = key handed to Velero + CNPG
+#   s3_backup_target_secret_key: "…"
+```
+
+versitygw authenticates every S3 request against this single **ROOT** key. It is mirrored
+(SOPS-encrypted) for the GitOps side at `platform/backup/minio-credentials.sops.yaml` — the two MUST
+stay in sync (same key/secret), since the cluster authenticates to this endpoint with them. Rotating
+means updating BOTH and re-running the playbook (rewrites the env file + restarts versitygw) +
+reconciling Flux.
+
+## Notes
+- This is the **Ansible-bootstrap-layer** secret (host credential), distinct from the platform's
+  in-cluster **SOPS + age** secret (ADR-0007) — different layer, different tool, same values.
+- `vault.yml` is a vault blob (not parseable YAML) — excluded from the yamllint gate like the other
+  `**/vault.yml`.

infra/ansible/inventory/group_vars/proxmox_hosts/vault.yml

@@ -0,0 +1,25 @@
+$ANSIBLE_VAULT;1.1;AES256
+33623463393666373861613531626336396164356361643939326332626634663234653164313437
+3963326465326364643639633066636564656565333461360a383438323530663932326462376661
+64343766366132373436366336353338383365396563636138333333306330643337636532383662
+6561323836656637380a656436656538613263646461386635303762376163363161336436386263
+34653437386364363161653132343434333230393732373364383564336162643136636437376231
+39376463336666396462363431636161306632326134663038343364626232333838333039393338
+64313762343363666139393864366561316362633465323361633132373533356531353534396433
+62333131356630343764303438663766623737333137383264623739646539373535306136303530
+34663063383032643238373537663633343735613265653964356638306566636538363063656435
+65643666366665393238363037313337633366333338633163316533656134336463363630353961
+37656637343063373266643237663633613638303165656262313133313535313665336335333039
+30626537373530663430343637386263353765353538376263333130623536306635353966393561
+62313362373061303035376635646335303133613365326163643566333031356436326464616430
+34373433343733386666626234623064383063393330343034363036663632373865643636623961
+31666131636538303131376339623733353964653266356339313466616363646336663433666631
+61663564633931626639323235633332366338333837316535376330633534626537666166643262
+63646239656165303366626163613238393266336566316164356330393836323332336330633635
+64636636613164366338396666316137353837313639323630353234373936393335366364666361
+35356163363636623031333733653432636539396235633436313334333663343532303239373238
+37646230326164396339386339356663396661306165343630316261306331383632626662323234
+31396539313766356633333736383036623839626663383763626332626237373338653832646463
+34366434386139306336633964313939643565363838346163656236656561363865386136346530
+36316661323262323366646332316535306637316663656264303137653831656136633432643765
+61646433653530646339

infra/ansible/inventory/hosts.ini

@@ -37,3 +37,11 @@ localhost ansible_connection=local
 # with their ansible_host under k3s_secondary; this group only tags the egress role target.
 [vpn_egress_gateway]
 k3s-dev-3
+
+# The Proxmox HYPERVISOR host itself (NOT a k3s node) — target of playbooks/backup-s3.yml,
+# which stands up the off-cluster versitygw S3 backup target for Velero + CNPG. Reached DIRECTLY
+# over SSH as root (its own 192.168.2.102:22, the pfSense NAT to the host's 192.168.1.9), so
+# it overrides the cluster-wide `ansible_user: k3s` from group_vars/all.yml. Distinct from the
+# `hypervisor` group above (that is `localhost` for host-prereqs.yml, a different machine).
+[proxmox_hosts]
+proxmox ansible_host=192.168.2.102 ansible_port=22 ansible_user=root ansible_ssh_private_key_file=~/.ssh/home_network

infra/ansible/playbooks/backup-s3.yml

@@ -0,0 +1,23 @@
+---
+# Stand up the off-cluster S3 backup target on the Proxmox host — the store Velero (R-STATE-03)
+# and CloudNativePG (R-IDP-05) back up into. Implemented with versitygw (Versity Gateway), which
+# REPLACES the earlier MinIO target; the role also decommissions any pre-existing MinIO install so
+# a re-run is a clean swap. Host infra, surgical + idempotent: the ONE ZFS dataset it owns
+# (Storage/k3s-backups), the versitygw user/service/binary, and the velero + cnpg buckets. It never
+# touches other datasets, sharenfs, pvesm storage, VMs or firewall.
+#
+# Credentials come from Ansible vault (inventory/group_vars/proxmox_hosts/vault.yml):
+#   s3_backup_target_access_key / _secret_key  (versitygw ROOT key)
+# The SAME key is staged for the GitOps side (SOPS) at
+#   platform/backup/minio-credentials.sops.yaml
+# Endpoint + key + secret + bucket names are byte-identical to the old MinIO target, so the
+# cluster-side Velero/CNPG config + the SOPS secret need ZERO change.
+#
+# Prereqs:  ansible-galaxy collection install -r collections/requirements.yml   (community.general.zfs)
+# Run:      ansible-playbook playbooks/backup-s3.yml
+- name: Provision the versitygw S3 backup target on the Proxmox host
+  hosts: proxmox_hosts
+  become: true
+  gather_facts: true          # need ansible_facts.all_ipv4_addresses to discover the LAN bind addr
+  roles:
+    - role: s3_backup_target

infra/ansible/roles/node_baseline/defaults/main.yml

@@ -15,3 +15,10 @@ node_baseline_kernel_modules:
 node_baseline_longhorn_device: /dev/vdb
 node_baseline_longhorn_mount: /var/lib/longhorn
 node_baseline_longhorn_fs_label: longhorn
+# Dedicated SSD etcd disk (tofu virtio2 -> /dev/vdc), mounted at the k3s etcd data dir
+# so etcd WAL fsync is isolated from Longhorn rebuild I/O on the HDD data disk.
+# See docs/plans/ssd-etcd-isolation.md. FS label keeps the fstab entry stable across
+# device renames (matches the Longhorn LABEL= approach).
+node_baseline_etcd_device: /dev/vdc
+node_baseline_etcd_mount: /var/lib/rancher/k3s/server/db
+node_baseline_etcd_fs_label: k3s-etcd

infra/ansible/roles/node_baseline/tasks/main.yml

@@ -72,6 +72,31 @@
     opts: "defaults,nofail,x-systemd.device-timeout=5s"
     state: mounted
 
+# Dedicated SSD etcd disk (docs/plans/ssd-etcd-isolation.md). Formats /dev/vdc and
+# mounts it at the k3s etcd data dir BEFORE k3s starts, so a rebuilt node comes up with
+# etcd already isolated on SSD. Data-safe: community.general.filesystem never reformats
+# a disk that already carries a filesystem, so re-running against an already-migrated
+# node leaves the etcd data intact. On EXISTING nodes the one-time data migration
+# (stop k3s -> rsync db -> mount) is done manually per the plan's §4.3 before this
+# role would mount over a populated directory.
+- name: Create ext4 filesystem on the dedicated etcd SSD disk (data-safe; skips if present)
+  community.general.filesystem:
+    fstype: ext4
+    dev: "{{ node_baseline_etcd_device }}"
+    opts: "-L {{ node_baseline_etcd_fs_label }}"
+
+- name: Mount the dedicated etcd SSD disk at the k3s etcd data dir
+  ansible.posix.mount:
+    path: "{{ node_baseline_etcd_mount }}"
+    src: "LABEL={{ node_baseline_etcd_fs_label }}"
+    fstype: ext4
+    # noatime: fewer metadata writes under etcd's fsync-heavy load. No `nofail`: a
+    # missing etcd disk MUST fail loudly rather than let k3s start etcd on the root HDD.
+    opts: "defaults,noatime"
+    dump: "0"
+    passno: "2"
+    state: mounted
+
 - name: Disable swap now
   ansible.builtin.command: swapoff -a
   changed_when: false

infra/ansible/roles/s3_backup_target/README.md

@@ -0,0 +1,66 @@
+# roles/s3_backup_target
+
+Stands up a **single-node S3 endpoint** on the **Proxmox host** to serve as the off-cluster
+**backup target** for the k3s platform's **Velero** (volume/resource DR, R-STATE-03) and
+**CloudNativePG** (Keycloak DB Barman object store, R-IDP-05). A backup target — not primary
+storage — so a single dataset is intentional.
+
+Implemented with **[versitygw](https://github.com/versity/versitygw)** (Versity Gateway): a
+maintained Go S3 gateway that serves a POSIX filesystem as S3. It **replaces the earlier MinIO
+implementation** (MinIO's community edition was de-prioritised — console removed, commercial
+pivot). The role also **decommissions any pre-existing MinIO** install on the host, so a re-run is
+a clean, idempotent swap. Endpoint, access key, secret and bucket names are kept **byte-identical**
+to the previous MinIO target, so nothing downstream (the SOPS secret, Velero, CNPG) changes.
+
+## What it does (all additive + idempotent, surgical)
+
+1. **Decommission MinIO** (tolerant no-op on a fresh host): stop/disable/remove `minio.service`,
+   the `minio` + `mc` binaries, `/etc/default/minio`, the IAM policy file, the `mc` client config,
+   the `minio` user/group, and MinIO's on-disk object layout (`.minio`, `.minio.sys`) inside the
+   gateway-root dir. **The ZFS dataset and mount are KEPT.**
+2. **ZFS dataset** `Storage/k3s-backups` — `compression=zstd`, `quota=100G`, **`xattr=sa`**
+   (required — versitygw stores object metadata in POSIX extended attributes and refuses to start
+   without them), `mountpoint=/mnt/Storage/k3s-backups`, **no sharenfs**. Uses `community.general.zfs`
+   so re-runs are no-ops and existing datasets are never touched.
+3. **`versitygw` system user/group** + the gateway-root dir (`…/k3s-backups/data`) it owns.
+4. **versitygw binary** (pinned, checksummed). The Proxmox host has **no outbound internet**, so the
+   release tarball is downloaded + verified + extracted on the **Ansible controller** and only the
+   binary is copied to the host.
+5. **systemd unit** `versitygw.service` (Type=simple, hardened), `posix` backend rooted at the
+   gateway root, bound to the host's **cluster-reachable `192.168.1.x` address** on port **9000**
+   (same endpoint as the old MinIO) — auto-discovered from facts, assert-guarded. TLS off (plain
+   HTTP, path-style).
+6. **Buckets** `velero` + `cnpg` — on the posix backend a bucket is simply a top-level subdirectory
+   of the gateway root, so they are pre-created owned by the service user.
+
+## Credentials
+
+versitygw authenticates every S3 request against a single **ROOT** access key/secret
+(`ROOT_ACCESS_KEY` / `ROOT_SECRET_KEY`). This role sets it to the **same** scoped key/secret the
+cluster already uses, supplied via the vaulted `inventory/group_vars/proxmox_hosts/vault.yml`
+(ansible-vault). Nothing is defaulted — a missing vault fails the play loudly:
+
+| var | purpose |
+|---|---|
+| `s3_backup_target_access_key` / `_secret_key` | versitygw ROOT key = the key handed to Velero + CNPG |
+
+The **same** key/secret is staged for the GitOps side, SOPS-encrypted, at
+`platform/backup/minio-credentials.sops.yaml` — the two MUST stay in sync.
+
+> Note: unlike MinIO, versitygw's single-binary posix backend has no separate "root admin" vs
+> "scoped user" distinction without enabling an external IAM backend. For a dedicated backup target
+> the scoped key **is** the root key — it only ever fronts the velero + cnpg buckets on this host.
+
+## Run
+
+```bash
+cd infra/ansible
+ansible-galaxy collection install -r collections/requirements.yml   # community.general (zfs)
+ansible-playbook playbooks/backup-s3.yml
+```
+
+## Endpoint
+
+`http://<host-192.168.1.x>:9000` — the S3 URL the cluster uses (path-style). An unauthenticated
+`GET /` returns `403 AccessDenied` once serving; a SigV4 `ListBuckets` with the root key returns
+`velero` + `cnpg`. Verified reachable from an in-cluster pod at bring-up.

infra/ansible/roles/s3_backup_target/defaults/main.yml

@@ -0,0 +1,94 @@
+---
+# roles/s3_backup_target — a single-node S3 backup endpoint on the Proxmox host, used ONLY as an
+# off-cluster backup bucket store for Velero + CloudNativePG (ADR-0003, R-STATE-03 / R-IDP-05).
+# This is a BACKUP TARGET, not primary storage — a single drive/dataset is intentional.
+#
+# Implemented with versitygw (Versity Gateway, github.com/versity/versitygw): a maintained Go S3
+# gateway that serves a POSIX filesystem as S3. It REPLACES the earlier MinIO implementation (MinIO's
+# community edition was de-prioritised); the role also decommissions any pre-existing MinIO install
+# on the host so a re-run is a clean swap.
+#
+# SAFETY: every task in this role is additive/idempotent and SURGICAL. It touches ONLY the
+# minio→versitygw swap: the ONE ZFS dataset it owns (Storage/k3s-backups), the versitygw system
+# user, its systemd service + binary, and the two buckets inside that dataset. It never touches
+# other datasets, sharenfs, pvesm storage, VMs or firewall. See tasks/main.yml for the guards.
+
+# ── versitygw release pin (Renovate-friendly) ─────────────────────────────────────────────────
+# The Proxmox host has NO outbound internet (PiHole-only DNS, no default route to WAN), so the
+# release tarball is fetched + verified on the ANSIBLE CONTROLLER, the binary extracted there, and
+# only the binary is copied to the host (tasks/main.yml).
+# Release assets:  https://github.com/versity/versitygw/releases
+#   tarball : versitygw_v{ver}_Linux_x86_64.tar.gz   (extracts to  versitygw_v{ver}_Linux_x86_64/versitygw)
+#   checksums: checksums.txt (sha256) in the same release
+s3_backup_target_versitygw_version: "v1.7.0"
+s3_backup_target_versitygw_sha256: "dacec530fecaa8776feec0536998dd4bd78036e0d4db0da9b9189d96bb790e03"
+s3_backup_target_versitygw_url: "https://github.com/versity/versitygw/releases/download/{{ s3_backup_target_versitygw_version }}/versitygw_{{ s3_backup_target_versitygw_version }}_Linux_x86_64.tar.gz"
+# Path of the binary INSIDE the extracted tarball (GoReleaser lays it out under a version dir).
+s3_backup_target_versitygw_archive_member: "versitygw_{{ s3_backup_target_versitygw_version }}_Linux_x86_64/versitygw"
+
+# Controller-side cache the tarball is fetched + extracted into (delegate_to localhost).
+s3_backup_target_controller_cache: "/tmp/s3-backup-target-cache"
+
+# ── ZFS dataset (backup drive) ────────────────────────────────────────────────────────────────
+# Host convention is Storage/<purpose> mounted at /mnt/Storage/<purpose>. NO sharenfs.
+# xattr=sa is REQUIRED: versitygw's posix backend stores object metadata (ETags, etc.) in POSIX
+# extended attributes and refuses to start if the root FS can't hold them.
+s3_backup_target_zfs_dataset: "Storage/k3s-backups"
+s3_backup_target_mountpoint: "/mnt/Storage/k3s-backups"
+s3_backup_target_zfs_compression: "zstd"
+s3_backup_target_zfs_quota: "100G"
+s3_backup_target_zfs_xattr: "sa"
+# versitygw's gateway root — a subdir of the dataset, keeping the dataset root owned by root and
+# clean. Its top-level subdirectories ARE the S3 buckets.
+s3_backup_target_gateway_root: "{{ s3_backup_target_mountpoint }}/data"
+
+# ── Service identity ────────────────────────────────────────────────────────────────────────
+s3_backup_target_user: "versitygw"
+s3_backup_target_group: "versitygw"
+s3_backup_target_bin: "/usr/local/bin/versitygw"
+s3_backup_target_env_file: "/etc/default/versitygw"
+
+# ── Network binding ──────────────────────────────────────────────────────────────────────────
+# The S3 API must bind the host address the k3s pods can reach (the 192.168.1.0/24 LAN; nodes are
+# .50/.51/.52). tasks/main.yml auto-discovers the host's address in this CIDR prefix and asserts it
+# exists; override s3_backup_target_bind_address to pin it explicitly. Port 9000 keeps the endpoint
+# byte-identical to the previous MinIO install so nothing downstream (SOPS secret, Velero/CNPG)
+# changes. (versitygw's own default is :7070 — deliberately overridden here.)
+s3_backup_target_lan_prefix: "192.168.1."
+s3_backup_target_bind_address: ""          # "" ⇒ auto-discover from facts (see tasks)
+s3_backup_target_api_port: 9000
+s3_backup_target_region: "us-east-1"
+
+# ── Buckets ────────────────────────────────────────────────────────────────────────────────────
+# Velero and CNPG each get a bucket. On the posix backend a bucket == a top-level subdirectory of
+# the gateway root, so they are simply pre-created (owned by the service user). The single root key
+# below has full access to all buckets.
+s3_backup_target_buckets:
+  - velero
+  - cnpg
+
+# ── Credentials ──────────────────────────────────────────────────────────────────────────────────
+# versitygw authenticates every S3 request against a single ROOT access key/secret (ROOT_ACCESS_KEY
+# / ROOT_SECRET_KEY). We set it to the SAME scoped key/secret the cluster already uses, so the
+# endpoint + access key + secret + buckets stay byte-identical to the previous MinIO target and the
+# GitOps side (SOPS: platform/backup/minio-credentials.sops.yaml) needs ZERO change.
+#
+# Supplied via the vaulted group_vars (inventory/group_vars/proxmox_hosts/vault.yml):
+#   s3_backup_target_access_key, s3_backup_target_secret_key
+# Intentionally NOT defaulted here so a missing vault fails loudly rather than silently
+# provisioning guessable creds.
+
+# ── MinIO decommission (of any pre-existing MinIO install this role is replacing) ──────────────
+# Paths/identities the OLD minio_backup_target role created, removed by the decommission block.
+s3_backup_target_minio_service: "minio.service"
+s3_backup_target_minio_bin: "/usr/local/bin/minio"
+s3_backup_target_minio_mc_bin: "/usr/local/bin/mc"
+s3_backup_target_minio_env_file: "/etc/default/minio"
+s3_backup_target_minio_policy_file: "/etc/minio-k3s-backup-rw.json"
+s3_backup_target_minio_user: "minio"
+s3_backup_target_minio_group: "minio"
+# MinIO's on-disk object layout left inside the gateway-root dir (cleared so versitygw starts
+# fresh; there are no real backups yet so this is safe). The dataset itself is KEPT.
+s3_backup_target_minio_leftovers:
+  - "{{ s3_backup_target_gateway_root }}/.minio"
+  - "{{ s3_backup_target_gateway_root }}/.minio.sys"

infra/ansible/roles/s3_backup_target/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart versitygw
+  ansible.builtin.systemd:
+    name: versitygw
+    state: restarted
+    daemon_reload: true

infra/ansible/roles/s3_backup_target/tasks/main.yml

@@ -0,0 +1,231 @@
+---
+# ── 0. Preconditions / safety asserts ─────────────────────────────────────────────────────────
+- name: Assert S3 backup-target credentials are supplied (from vaulted group_vars)
+  ansible.builtin.assert:
+    that:
+      - s3_backup_target_access_key | default('') | length > 0
+      - s3_backup_target_secret_key | default('') | length >= 8
+    fail_msg: >-
+      versitygw credentials are missing. Provide s3_backup_target_access_key/_secret_key via
+      inventory/group_vars/proxmox_hosts/vault.yml (they MUST match the SOPS-staged scoped key at
+      platform/backup/minio-credentials.sops.yaml so nothing downstream changes).
+    quiet: true
+
+# Determine the host's 192.168.1.0/24 address (the S3 endpoint the cluster reaches). If the
+# operator pinned s3_backup_target_bind_address, honour it; else auto-discover and assert.
+- name: Discover the host LAN (cluster-reachable) IPv4 address
+  ansible.builtin.set_fact:
+    s3_backup_target_bind_address: >-
+      {{ s3_backup_target_bind_address
+         if (s3_backup_target_bind_address | length > 0)
+         else (ansible_facts.all_ipv4_addresses
+               | select('match', '^' ~ (s3_backup_target_lan_prefix | regex_escape))
+               | list | first | default('')) }}
+
+- name: Assert a cluster-reachable LAN address was found
+  ansible.builtin.assert:
+    that:
+      - s3_backup_target_bind_address | length > 0
+    fail_msg: >-
+      Could not find a {{ s3_backup_target_lan_prefix }}x address on this host
+      (interfaces: {{ ansible_facts.all_ipv4_addresses | default([]) | join(', ') }}).
+      The cluster egresses onto that LAN — refusing to bind the S3 endpoint to an unreachable
+      address. Set s3_backup_target_bind_address explicitly if discovery is wrong.
+    quiet: true
+
+# ── 1. Decommission any pre-existing MinIO install this role replaces (idempotent + surgical) ───
+# All steps tolerate absence, so on a fresh host (no MinIO) they are harmless no-ops. The ZFS
+# dataset and mount are KEPT — only MinIO's service/binary/env/mc-client/object-layout go.
+- name: Check whether the old MinIO systemd unit is present
+  ansible.builtin.stat:
+    path: "/etc/systemd/system/{{ s3_backup_target_minio_service }}"
+  register: s3_backup_target_minio_unit
+
+- name: Stop + disable the old MinIO service (only if its unit exists)
+  ansible.builtin.systemd:
+    name: "{{ s3_backup_target_minio_service }}"
+    state: stopped
+    enabled: false
+  when: s3_backup_target_minio_unit.stat.exists
+
+- name: Remove the old MinIO systemd unit
+  ansible.builtin.file:
+    path: "/etc/systemd/system/{{ s3_backup_target_minio_service }}"
+    state: absent
+  register: s3_backup_target_minio_unit_removed
+
+- name: Reload systemd if the MinIO unit was removed
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: s3_backup_target_minio_unit_removed.changed
+
+- name: Remove MinIO server + mc client binaries, env file and IAM policy file
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ s3_backup_target_minio_bin }}"
+    - "{{ s3_backup_target_minio_mc_bin }}"
+    - "{{ s3_backup_target_minio_env_file }}"
+    - "{{ s3_backup_target_minio_policy_file }}"
+
+- name: Remove the mc client config left in root's home
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /root/.mc
+    - /root/.mcli
+
+- name: Clear MinIO's on-disk object layout from the gateway-root dir (dataset itself KEPT)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop: "{{ s3_backup_target_minio_leftovers }}"
+
+- name: Remove the old minio system user (its files are cleared / re-owned below)
+  ansible.builtin.user:
+    name: "{{ s3_backup_target_minio_user }}"
+    state: absent
+    remove: false
+  failed_when: false
+
+- name: Remove the old minio system group
+  ansible.builtin.group:
+    name: "{{ s3_backup_target_minio_group }}"
+    state: absent
+  failed_when: false
+
+# ── 2. ZFS dataset (backup drive) — idempotent, never touches other datasets ───────────────────
+# xattr=sa is required by versitygw's posix backend (object metadata is stored in extended attrs).
+- name: Create/ensure the k3s-backups ZFS dataset (compression, quota, xattr=sa, mountpoint; NO sharenfs)
+  community.general.zfs:
+    name: "{{ s3_backup_target_zfs_dataset }}"
+    state: present
+    extra_zfs_properties:
+      compression: "{{ s3_backup_target_zfs_compression }}"
+      quota: "{{ s3_backup_target_zfs_quota }}"
+      xattr: "{{ s3_backup_target_zfs_xattr }}"
+      mountpoint: "{{ s3_backup_target_mountpoint }}"
+
+# ── 3. Service identity ───────────────────────────────────────────────────────────────────────
+- name: Create the versitygw system group
+  ansible.builtin.group:
+    name: "{{ s3_backup_target_group }}"
+    system: true
+
+- name: Create the versitygw system user (no login shell, home = gateway root)
+  ansible.builtin.user:
+    name: "{{ s3_backup_target_user }}"
+    group: "{{ s3_backup_target_group }}"
+    system: true
+    shell: /usr/sbin/nologin
+    home: "{{ s3_backup_target_gateway_root }}"
+    create_home: false
+
+- name: Ensure the gateway root dir exists (owned by versitygw, inside the dataset)
+  ansible.builtin.file:
+    path: "{{ s3_backup_target_gateway_root }}"
+    state: directory
+    owner: "{{ s3_backup_target_user }}"
+    group: "{{ s3_backup_target_group }}"
+    mode: "0750"
+
+- name: Pre-create the S3 buckets (a bucket == a top-level subdir of the gateway root)
+  ansible.builtin.file:
+    path: "{{ s3_backup_target_gateway_root }}/{{ item }}"
+    state: directory
+    owner: "{{ s3_backup_target_user }}"
+    group: "{{ s3_backup_target_group }}"
+    mode: "0750"
+  loop: "{{ s3_backup_target_buckets }}"
+
+# ── 4. Binary — fetched + extracted on the CONTROLLER (host has no internet), copied to the host ─
+- name: Ensure controller-side download cache exists
+  ansible.builtin.file:
+    path: "{{ s3_backup_target_controller_cache }}"
+    state: directory
+    mode: "0700"
+  delegate_to: localhost
+  become: false
+  run_once: true
+
+- name: Fetch the versitygw release tarball onto the controller (checksummed)
+  ansible.builtin.get_url:
+    url: "{{ s3_backup_target_versitygw_url }}"
+    dest: "{{ s3_backup_target_controller_cache }}/versitygw_{{ s3_backup_target_versitygw_version }}.tar.gz"
+    checksum: "sha256:{{ s3_backup_target_versitygw_sha256 }}"
+    mode: "0600"
+  delegate_to: localhost
+  become: false
+  run_once: true
+
+- name: Extract the versitygw binary on the controller
+  ansible.builtin.unarchive:
+    src: "{{ s3_backup_target_controller_cache }}/versitygw_{{ s3_backup_target_versitygw_version }}.tar.gz"
+    dest: "{{ s3_backup_target_controller_cache }}"
+    remote_src: true
+    creates: "{{ s3_backup_target_controller_cache }}/{{ s3_backup_target_versitygw_archive_member }}"
+  delegate_to: localhost
+  become: false
+  run_once: true
+
+# Integrity is already guaranteed by the SHA256 get_url performed on the controller; copy transfers
+# that verified artifact faithfully. (The copy module's own `checksum` arg is SHA1 — not used here.)
+- name: Install the versitygw binary on the host
+  ansible.builtin.copy:
+    src: "{{ s3_backup_target_controller_cache }}/{{ s3_backup_target_versitygw_archive_member }}"
+    dest: "{{ s3_backup_target_bin }}"
+    owner: root
+    group: root
+    mode: "0755"
+  notify: Restart versitygw
+
+# ── 5. systemd service ────────────────────────────────────────────────────────────────────────
+- name: Write the versitygw environment file (root access key/secret)
+  ansible.builtin.template:
+    src: versitygw.env.j2
+    dest: "{{ s3_backup_target_env_file }}"
+    owner: root
+    group: "{{ s3_backup_target_group }}"
+    mode: "0640"
+  no_log: true
+  notify: Restart versitygw
+
+- name: Install the versitygw systemd unit
+  ansible.builtin.template:
+    src: versitygw.service.j2
+    dest: /etc/systemd/system/versitygw.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart versitygw
+
+- name: Enable + start versitygw (apply any pending unit/env changes now)
+  ansible.builtin.systemd:
+    name: versitygw
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Flush handlers so versitygw is (re)started before we verify it
+  ansible.builtin.meta: flush_handlers
+
+# ── 6. Readiness gate ─────────────────────────────────────────────────────────────────────────
+# versitygw has no health endpoint; an UNAUTHENTICATED GET / returns 403 (AccessDenied) once the
+# S3 server is up and enforcing SigV4 — a reliable "listening + serving" signal.
+- name: Wait for the versitygw S3 port to accept connections
+  ansible.builtin.wait_for:
+    host: "{{ s3_backup_target_bind_address }}"
+    port: "{{ s3_backup_target_api_port }}"
+    timeout: 60
+
+- name: Confirm versitygw is serving S3 (unauthenticated GET / ⇒ 403 AccessDenied)
+  ansible.builtin.uri:
+    url: "http://{{ s3_backup_target_bind_address }}:{{ s3_backup_target_api_port }}/"
+    method: GET
+    status_code: 403
+  register: s3_backup_target_probe
+  retries: 15
+  delay: 2
+  until: s3_backup_target_probe.status == 403

infra/ansible/roles/s3_backup_target/templates/versitygw.env.j2

@@ -0,0 +1,10 @@
+# {{ ansible_managed }}
+# versitygw root credentials (single-node posix backup target for k3s Velero + CNPG).
+# Managed by roles/s3_backup_target — do not edit by hand.
+#
+# versitygw authenticates every S3 request against this ROOT key. It is set to the SAME scoped
+# access key/secret the cluster already uses (mirrored via SOPS in
+# platform/backup/minio-credentials.sops.yaml) so the endpoint + key + secret stay byte-identical
+# to the previous MinIO target — nothing downstream changes.
+ROOT_ACCESS_KEY={{ s3_backup_target_access_key }}
+ROOT_SECRET_KEY={{ s3_backup_target_secret_key }}

infra/ansible/roles/s3_backup_target/templates/versitygw.service.j2

@@ -0,0 +1,35 @@
+{# {{ ansible_managed }} — versitygw systemd unit (roles/s3_backup_target) #}
+[Unit]
+Description=versitygw S3 backup target (k3s Velero + CNPG)
+Documentation=https://github.com/versity/versitygw
+After=network-online.target
+Wants=network-online.target
+AssertFileIsExecutable={{ s3_backup_target_bin }}
+AssertPathIsDirectory={{ s3_backup_target_gateway_root }}
+
+[Service]
+Type=simple
+User={{ s3_backup_target_user }}
+Group={{ s3_backup_target_group }}
+# ROOT_ACCESS_KEY / ROOT_SECRET_KEY are read from here (kept out of the process args).
+EnvironmentFile={{ s3_backup_target_env_file }}
+# Global options (--port/--region) come BEFORE the `posix` backend subcommand; posix's single
+# positional arg is the gateway root, whose top-level subdirs are the S3 buckets. TLS off (no
+# --cert/--key) ⇒ plain HTTP, path-style.
+ExecStart={{ s3_backup_target_bin }} \
+  --port {{ s3_backup_target_bind_address }}:{{ s3_backup_target_api_port }} \
+  --region {{ s3_backup_target_region }} \
+  posix {{ s3_backup_target_gateway_root }}
+Restart=always
+RestartSec=5s
+LimitNOFILE=65536
+TimeoutStopSec=30s
+# Hardening (backup target holds no other roles on this host).
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths={{ s3_backup_target_mountpoint }}
+
+[Install]
+WantedBy=multi-user.target

infra/tofu/README.md

@@ -64,24 +64,26 @@ All sizing/IPs/counts/placement are variables — see `variables.tf`.
 
 ## Auth (env only — never in tofu vars/state)
 The provider reads these from the environment; the **API token is never a tofu
-variable**, so it never enters state, vars, or plan output:
+variable**, so it never enters state, vars, or plan output. You do NOT export these by hand —
+they live encrypted in `secrets.sops.env` and the `tf.sh` wrapper decrypts + exports them
+(and sets `TF_VAR_proxmox_endpoint` for scheme normalization) at runtime:
 ```
-export PROXMOX_VE_ENDPOINT="https://192.168.2.102:8006/"
-export PROXMOX_VE_API_TOKEN="user@realm!tokenid=SECRET"
-export PROXMOX_VE_INSECURE="true"
-export TF_VAR_proxmox_endpoint="$PROXMOX_VE_ENDPOINT"   # optional; enables normalization
+PROXMOX_VE_ENDPOINT=https://192.168.2.102:8006/
+PROXMOX_VE_API_TOKEN=user@realm!tokenid=SECRET
+PROXMOX_VE_INSECURE=true
 ```
+See `REMOTE-STATE.md` for the encrypted-secrets + wrapper flow.
 
 ## Usage (operator — NOT run here)
+All secrets come from the encrypted `secrets.sops.env` via the `tf.sh` wrapper, run inside the
+dev container (which has `sops` + the age key):
 ```
-cd infra/tofu
-cp terraform.tfvars.example terraform.tfvars   # set ssh_authorized_key
-# source the operator env file (exports the PROXMOX_VE_* above)
-tofu init
-tofu plan
-tofu apply
+cp infra/tofu/terraform.tfvars.example infra/tofu/terraform.tfvars   # set ssh_authorized_key
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh plan
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh apply
 # hand the node IPs to Ansible:
-tofu output -raw ansible_inventory > ../ansible/inventory/hosts.ini
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh output -raw ansible_inventory > infra/ansible/inventory/hosts.ini
 ```
 
 ## Hand-off contract

infra/tofu/REMOTE-STATE.md

@@ -0,0 +1,100 @@
+# Tofu remote state — Cloudflare R2 (off-cluster, encrypted)
+
+## Why
+State was **local** (`terraform.tfstate` on one workstation, git-ignored). Losing that box meant a
+`tofu apply` from anywhere else would try to **recreate the three live k3s nodes**. State must live
+**off the cluster it manages** — the chain is `Proxmox host → k3s VMs → k3s cluster → Gitea/apps`, so
+an in-cluster backend (Gitea) is unreadable in the one scenario it exists for (cluster gone, rebuild).
+
+**Cloudflare R2** sits outside that chain entirely: a full cluster rebuild can still read state, it
+survives total-site loss (off-site), and it works from any machine. State bytes are **encrypted at
+rest** (OpenTofu native AES-GCM) so R2 only ever holds ciphertext.
+
+## One-time setup (operator)
+
+### 1. Create the R2 bucket + token (Cloudflare dashboard)
+1. **R2 → Create bucket** → name `homelab-tofu-state` (any name; match `backend.hcl`).
+2. **R2 → Manage API Tokens → Create API Token** → permission **Object Read & Write**, scoped to
+   that bucket. Copy the **Access Key ID** and **Secret Access Key** (shown once).
+3. Note your **Account ID** (32 hex chars, in the dashboard URL / the R2 S3 endpoint).
+
+### 2. Fill in the encrypted secrets file (edited in place; ciphertext is committed)
+All five secrets live in ONE committed, SOPS/age-encrypted dotenv, `infra/tofu/secrets.sops.env`.
+Edit it inside the dev container (it has `sops` + the age key). `sops` decrypts to your `$EDITOR`
+and re-encrypts on save — plaintext never touches disk:
+```bash
+./infra/devcontainer/dev.sh run sops infra/tofu/secrets.sops.env
+```
+Replace every `REPLACE-ME`:
+- `PROXMOX_VE_API_TOKEN` — `user@realm!tokenid=secret` (injected into the provider's env by tf.sh;
+  never a tofu variable, so it never lands in state).
+- `PROXMOX_VE_ENDPOINT` / `PROXMOX_VE_INSECURE` — already set to `https://192.168.2.102:8006/` / `true`.
+- `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` — the R2 Object-Read/Write token from step 1.
+- `TF_STATE_PASSPHRASE` — the state-at-rest passphrase (≥ 16 chars). **tf.sh builds the multi-line
+  `TF_ENCRYPTION` blob from this one scalar.**
+
+> ⚠️ **The passphrase and R2 keys MUST be the SAME values that already encrypt the live R2 state.**
+> The state in R2 is already AES-GCM-encrypted with the current passphrase; a *different*
+> `TF_STATE_PASSPHRASE` makes the existing state UNREADABLE (and wrong R2 keys can't reach it at
+> all). Reuse your existing values from your password manager — do NOT invent new ones here.
+
+The non-secret R2 bucket/key/endpoint live in the COMMITTED `infra/tofu/backend.r2.hcl` (an R2
+account id is not a credential). Still keep the passphrase + R2 keys in your password manager as
+the authoritative backup. The old plaintext `.r2.env` / `backend.hcl` are retired — delete any
+local copies (they stay in `.gitignore` only as a safety net).
+
+### 3. Migrate the existing local state -> R2 (one shot, moves state only — touches NO infra)
+> The R2 state has ALREADY been migrated and encrypted. This step is historical — a fresh machine
+> adopting the existing encrypted state does NOT run it; it just does the Day-2 flow below.
+```bash
+cp infra/tofu/terraform.tfstate infra/tofu/terraform.tfstate.premigrate.bak   # belt-and-suspenders
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init -migrate-state         # answer "yes"
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh plan                        # MUST say "No changes."
+```
+If `plan` shows no changes, the migration is correct. If it wants to create/destroy nodes, STOP —
+the state did not carry over; re-check backend.hcl/creds and do not apply.
+
+> **Adopting encryption on pre-existing *unencrypted* state (one-time — already done on first
+> migration).** OpenTofu refuses to read a plaintext state while encryption is configured
+> (`encountered unencrypted payload without unencrypted method configured`). For that ONE migration
+> only, `TF_ENCRYPTION` must include a fallback so tofu can read plaintext and rewrite it encrypted:
+> add `method "unencrypted" "migrate" {}` and give the `state` block a
+> `fallback { method = method.unencrypted.migrate }`. Run the migrate, then **remove both** and
+> `tofu init -reconfigure` so encryption is enforced. A fresh machine adopting the ALREADY-encrypted
+> R2 state does NOT need this — the encrypt-only `.r2.env.example` is correct as-is.
+
+## Day-2 use (any machine)
+Everything goes through the `tf.sh` wrapper inside the dev container. It decrypts
+`secrets.sops.env`, exports `PROXMOX_VE_*` / `AWS_*`, builds `TF_ENCRYPTION` from
+`TF_STATE_PASSPHRASE`, and passes `backend.r2.hcl` to `init`. NO manual secret entry, no plaintext
+env file:
+```bash
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh plan     # expect "No changes."
+./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh apply
+```
+To CHANGE a secret, edit the encrypted file in place (re-encrypts on save):
+```bash
+./infra/devcontainer/dev.sh run sops infra/tofu/secrets.sops.env
+```
+
+That's the whole "work from a different computer" story: clone the repo, have the age PRIVATE key in
+`~/.config/sops/age/keys.txt` (the dev container mounts it), then `tf.sh init` / `plan`. The secrets
+ride along in git as ciphertext; only the age key lives outside the repo.
+
+## DR: rebuild from nothing
+Because state is in R2 (not the cluster) AND the tofu secrets are committed (encrypted), a destroyed
+cluster is recoverable from just **the repo + the age private key**. On a fresh box: install the dev
+container toolchain, drop the age key at `~/.config/sops/age/keys.txt`, then
+`./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init` and `... tf.sh apply` to rebuild the VMs
+against the preserved state. Two things must survive off-cluster and off-repo: the **age private
+key** (in `flux-system/sops-age` in a live cluster; also keep an offline copy) and the **R2 state
+bucket** itself (covered by the off-cluster backup target OQ-BKP so state survives an R2/account loss).
+The `TF_STATE_PASSPHRASE` and R2 keys are recoverable from the encrypted `secrets.sops.env` as long
+as you hold the age key — keep them in your password manager too, belt and suspenders.
+
+## Follow-up (tracked)
+- **Locking.** OpenTofu 1.9.0 has no `use_lockfile`; state locking is currently OFF (safe for a single
+  operator). After bumping the toolchain to **OpenTofu ≥ 1.10** (`infra/devcontainer/versions.env`
+  + `versions.tf` `required_version`), set `use_lockfile = true` in `backend.tf` for native R2
+  conditional-write locking (no DynamoDB) — do this before regular two-machine use.

infra/tofu/backend.r2.hcl

@@ -0,0 +1,17 @@
+# Committed R2 backend config — NON-SECRET account-specific bits, passed to
+# `tofu init -backend-config=` by tf.sh. The S3 keys and the state-encryption
+# passphrase are NOT here — they come from secrets.sops.env via tf.sh.
+#
+# The endpoint contains the R2 account id (the 32-hex string). An account id is
+# NOT a credential (it does nothing without the R2 API keys), so committing it is
+# fine and keeps the "work from any machine" story to: clone repo + age key.
+#
+# The rest of the backend "s3" block (region, skip_* flags, encryption) lives in
+# backend.tf; this file supplies only bucket/key/endpoints.
+
+bucket = "homelab-tofu-state"
+key    = "k3s-dev/terraform.tfstate"
+
+endpoints = {
+  s3 = "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com"
+}

infra/tofu/backend.tf

@@ -0,0 +1,39 @@
+# Remote state — Cloudflare R2 (S3-compatible), stored OFF the k3s cluster.
+#
+# WHY OFF-CLUSTER (the load-bearing decision): tofu state must live at or below the layer it
+# manages. Its dependency chain is  Proxmox host -> k3s VMs -> k3s cluster -> Gitea/apps, so state
+# in Gitea (top of the chain) would be UNREADABLE in exactly the disaster it exists for — cluster
+# gone, `tofu apply` to rebuild. R2 sits entirely outside that chain: a full cluster rebuild can
+# still read state. It also survives total-site loss (off-site) and gives multi-machine access.
+#
+# SECRETS: driven by infra/tofu/tf.sh, which decrypts the COMMITTED, SOPS/age-encrypted
+# secrets.sops.env and exports AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (R2 S3 keys) plus builds
+# the TF_ENCRYPTION blob from the single TF_STATE_PASSPHRASE. The non-secret bucket/key/endpoint
+# live in the committed backend.r2.hcl (an R2 account id is not a credential). State bytes are
+# encrypted at rest via TF_ENCRYPTION (see REMOTE-STATE.md) — the passphrase lives ONLY inside the
+# encrypted secrets.sops.env (and your password manager), never in git plaintext or in the state.
+#
+# INIT (run via the wrapper, inside the dev container — non-secret bits from backend.r2.hcl,
+# secret bits decrypted from secrets.sops.env):
+#   ./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init
+#
+# NOTE: this `terraform {}` block is MERGED with the one in versions.tf (required_version +
+# required_providers). OpenTofu allows multiple terraform blocks across files; `backend` appears once.
+terraform {
+  backend "s3" {
+    region = "auto" # R2 ignores region, but the S3 backend requires a value.
+
+    # R2 is S3-compatible but not AWS: disable the AWS-only preflight/validation paths.
+    skip_credentials_validation = true
+    skip_region_validation      = true
+    skip_requesting_account_id  = true
+    skip_metadata_api_check     = true
+    skip_s3_checksum            = true # R2 rejects AWS trailing (streaming) checksums.
+
+    # bucket, key, endpoints{ s3 } -> committed backend.r2.hcl, passed by tf.sh via -backend-config.
+    #
+    # use_lockfile = true  -> ENABLE after bumping OpenTofu to >= 1.10. Native S3 conditional-write
+    # locking (no DynamoDB); works on R2 via If-None-Match. 1.9.0 has no lockfile support, so state
+    # locking is OFF for now (single-operator workflow — see REMOTE-STATE.md follow-up).
+  }
+}

infra/tofu/main.tf

@@ -40,6 +40,17 @@ locals {
 # OPERATOR ACTION: enable the "Import" content type on var.image_datastore (Datacenter
 # -> Storage -> local -> Content -> check "Import"), else the disk import fails with
 # "wrong type 'iso' - needs to be 'images' or 'import'".
+#
+# The Debian cloud image is a bpg `download_file` resource. This intentionally uses the
+# LONG (deprecated) type name `proxmox_virtual_environment_download_file`: the R2 backend
+# state still holds this resource under the long name, and bpg >= 0.111 dropped the
+# cross-type `moved{}` linkage to the short `proxmox_download_file` (ADR-007) — a `moved`
+# block between the two now fails plan with "Resource type mismatch" (this broke plan on
+# main entirely). Keeping the long name makes config match state, so the plan is clean and
+# the ~450 MB image is NOT re-downloaded. Migrating to the short name is a SEPARATE,
+# deliberate step (`tofu state mv proxmox_virtual_environment_download_file.debian_cloud_image
+# proxmox_download_file.debian_cloud_image`, then rename here) — not bundled with the
+# etcd-disk change so this bounce stays focused and low-risk.
 resource "proxmox_virtual_environment_download_file" "debian_cloud_image" {
   content_type = "import"
   datastore_id = var.image_datastore
@@ -115,6 +126,22 @@ resource "proxmox_virtual_environment_vm" "node" {
     discard      = "on"
   }
 
+  # Dedicated SSD etcd disk (virtio2 -> /dev/vdc). Isolates etcd WAL fsync from the HDD
+  # `Storage` pool where the OS + Longhorn disks live — the node_baseline role formats
+  # it and mounts it at /var/lib/rancher/k3s/server/db before k3s starts. On the SSD
+  # `local-zfs` (rpool/data) datastore: an enterprise Micron SATA SSD mirror WITH
+  # power-loss protection, so cache=none (honor guest fsync straight through to the ZFS
+  # ZIL) is both durable and fast. Kept last (ascending interface) so bpg won't reorder
+  # the existing virtio0/virtio1 disks. See docs/plans/ssd-etcd-isolation.md.
+  disk {
+    datastore_id = var.etcd_ssd_datastore
+    interface    = "virtio2"
+    size         = var.etcd_disk_gib
+    discard      = "on"
+    cache        = "none"
+    iothread     = false
+  }
+
   network_device {
     bridge = var.bridge
   }

infra/tofu/secrets.sops.env

@@ -0,0 +1,24 @@
+#ENC[AES256_GCM,data:phVXkjO3EXMu1N0PjbJc3ezD9Fv6OjYXV8SR7kVJkCU5mNegSsSxqla5yK2pnkXe/2bKl4zH2jHNc1Mcjwrkxbq2rbdCl87HkDYU4GQ=,iv:T+NCH5piQKyu33pC+VDmkGKFDw8BFyKU0SGyXU0Oyc0=,tag:9rTfz7kjbG14I542CJ20JQ==,type:comment]
+#ENC[AES256_GCM,data:ouvXF4JpcjNKh19m7pmbsMGaYCJnt8g8j3u6KOS5qqLwPUQXWpap5/iMFOJ99Zg7P5Mr++pFBcsDHehaLdmk3zRPejAn2hiZdrU0uDBPHrNOs5ivAV/xplWbeU5RoQ==,iv:RHIswt1NJ5SFx4zJCy3lLHHW29kh+5EXM9SMXTafzQY=,tag:Y7B+MSE1reAkvdLf5rG73Q==,type:comment]
+#ENC[AES256_GCM,data:N8YDuSbxFKEGYqBAj/lcJiq1sVXkc6QHyv+PAPeflCqSkuJnYxRK5PmUdvrUlJ5UQ2JtgZSpLPQeunGYkXTHUKxQKn30HGU/vH7PnFUaaGcdQio=,iv:dzllLfLcFUsXyXRiUkWOzVq4ByCxuQyOthIuGuAvc50=,tag:Bnu2MKNAQBUf2CYVeM/ouw==,type:comment]
+#ENC[AES256_GCM,data:S+H2PEFoMYlHFqG3BJNbT/iQxa38pZ1vgtG1mF/1Oq+UOF7jOiD8NSxayUnu2wuzqWrFkQtDCNRGPwiogq+nyeTQTkMJgm9iTKOkgFKn9gL06A==,iv:X61ZbHvs5yMcu7oyJvZgvUXeQ8IjjM3/BsZZ4oveRts=,tag:2xGsmutQCr4DOJWJm1cOHw==,type:comment]
+#ENC[AES256_GCM,data:4LI7KckpBHs01cebeZjjqs8/WZbKKalPnx8aj77wT3JDtPifdAE0N1/4CpJYGOKJsJdm0sYm/3sh+dZ+F/PGfk3lfkTsKobvjS8Gs6WVOrQhJkq2LucBWw==,iv:8nJsWdKo57NsNIb0slM7OHakMu0u1sVpECRHE9BWVno=,tag:9xuw36Cd/Ex33FhYwGWfBA==,type:comment]
+#
+#ENC[AES256_GCM,data:Ne1QnjtM2NxPEgjHCgKTU6tand+4yaZWFTq3qxiNFcOucJVL134ycsE+41B7NlEz0+UztRXJ+Zz1/wpcIBQsswDfw1pVP+t3lGl7Mq+GcoElB/E=,iv:esdC6pJPkvirFm3Bt5c1q719o6fbWHMR617j86D9WCk=,tag:hfkQyK4Pf+mhMhAmiT4mhQ==,type:comment]
+#ENC[AES256_GCM,data:olZoi67DYWBRnKv3SEcp9AY7J+jcw57LPZdff4tjgrTdPHos9uyE+s0JCWDAGVrKLu6JFvEIifKIaGz59FLo+qv1HHP6gWXhpxTS0RFTNAYwn/GVl5eUFzLWJmN8Fqw=,iv:K7V/q1De9D5hCKXXTkFfHSaHPtP/INWwdZ8V1tupMtQ=,tag:VPXiDQgMjdgXuswXRmbHRA==,type:comment]
+PROXMOX_VE_API_TOKEN=ENC[AES256_GCM,data:TRKTUPaDs3kc8NVzMOCK4pfgcOmE0Ldr4fxi3L3Oz3QIm6OO+Wr2uXkNem5s7ZnzeJuedYo6FTbPnhrktgsBfy9lYkU13A==,iv:Ol1pU8XNCJh/4nCn25XFoGF52fbT7gSAV0f9V84eCGE=,tag:xswA7jAkfh6pYHkYU7wRTw==,type:str]
+PROXMOX_VE_ENDPOINT=ENC[AES256_GCM,data:XJ1I+RMh6YBasYkJSFEgTMrHpGnooDiCCGx/,iv:4SCkmvxCkKRBP0YxzrpQ4bUEdlPvnwOqVh5S+jO5pqo=,tag:fmIg/5r2VKn/YTjVubwk1A==,type:str]
+PROXMOX_VE_INSECURE=ENC[AES256_GCM,data:6VjGNw==,iv:gkR7QY5tNacpnquR7vVg14zJwjNMfR3iCkvI+T7g5d4=,tag:TvZCbaejUu+50sa3n+b4XQ==,type:str]
+#ENC[AES256_GCM,data:wXUDqRFrcgE1vdKCXK/hq5ZerZ+qp/Dp48iP4sJBgVrIleEL8NEerk1WahuBXg7DZLm1r0yAZz7bACUHZST0x/Y=,iv:trlf6jEWST/ur2AobLuTxXAk4qIkoM+EHe542+aHZGY=,tag:mlr6H0E6OM6KcW1zntiQvg==,type:comment]
+AWS_ACCESS_KEY_ID=ENC[AES256_GCM,data:mXCO1qoux4ccyOm6YCJbu4Btu+3PxyhW9zH3+JsM99o=,iv:czO/MxomX7obL0+StIio0IMIT4MIi/3UEBLLcwlpD4k=,tag:8tCxjWHBNHn+ZKU4lF/pXg==,type:str]
+AWS_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:Svltt26h9FnJg0ppbgpoyaxadPqvWsw+kd2D8NFomk4Yo7qyFWPQchEj0cjRgQbvcvW0Z4TLzOTB+j/lsuAvpQ==,iv:bZLAxuitgdX0eaNkZoQCkVgv5dz91TySo7Fz+MAKP+8=,tag:is8R4KlYGFV9+Mrw9RmxIQ==,type:str]
+#ENC[AES256_GCM,data:ztI5Cl8qRqpgnAw5e/fODKXJ3f8lmsbY9EXMIEaIQQBkyAlaFAEmbYUbb/1MgFhEGyMHFE6DROLAITBBf8nvIf8OYmsaufcvQeM=,iv:l8Ve0/cAvM6BzJkMeIw/0JFKZsB/4lrKYp6LZDz3BAQ=,tag:Ycbmwau8Slp8gvP6DOlXPw==,type:comment]
+#ENC[AES256_GCM,data:rI8MK9Nuc58hX8zBEtqITCcLu5/4pBKmLKWbYE/NzfGN9VB1K2lDv7P5CHFyHN20BVWiUx0nOmXzPZCJ/3NecNr0pJBBn2psrHK8BgZkRDE+yKU=,iv:fDCzQy05zUCoYI82cxxRZkvcpBT4xDHsjNaTfAkosjo=,tag:G7yoxa/Zpt/8ylvaLB3OHQ==,type:comment]
+#ENC[AES256_GCM,data:VdqDxRFlnn85hTj4Oz00+noc5nV9Yp19wKxk3Ql+eKDtMtm2ftLIow4vjuR3YIwGJTi04nkFdgkzmuOrOJBlkbhAmSjvVKi/0QxxHUhDAcslNvH8,iv:Fwo89vAPfPZXXk8y1Xn3yaxBKnGJgR4Y0eaYUOLmOjM=,tag:hvc/WJmbgD2QsCser+djlw==,type:comment]
+TF_STATE_PASSPHRASE=ENC[AES256_GCM,data:gJgSLKr70FD5sJZWu+vRBw==,iv:NhC6CVgDwwLfTO14Baxa5lpaUDjmzMwEDGUhg/p6Mgc=,tag:B0Ko4OaIyj0nVLJ+SJ+0zA==,type:str]
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXS1Q1dHdNUkdEN2tWWDNj\ncHR6RVZ5ckhYS0VkL1FqNGtNSWREb1hCNDNrCkVCMjFJU2JOV3I0MTI0Z3JHT2J2\nMUJTbDFoWHJyM1FjK05BSHhIOHVIK00KLS0tIG4vemVTNVhFajhSQ1dkY252Ly9q\nTGl2ZFV1ek5QQTBtcVZ6SnBZbXdERUUKKyz3ZNRrXn9OWWpV++fURpNHEuiRohko\nXJL496ln+4eJclnFDj6FJzqJJggKrd4F7zLb/XiEkDynP6fE6qXIQQ==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_0__map_recipient=age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+sops_lastmodified=2026-08-28T00:52:35Z
+sops_mac=ENC[AES256_GCM,data:fOGSNfzyu2o+2nygpDQEKsmDh61IIJ6BHI8sEfbeYTv1cYHSoZrar2QXL7hTBEnlkWkzyOnur2nonCJdrs1chqYmJ3tvoSaxSGywL6hoDcgbHkSi/BmC7IwLGhFD5TNgV6CHL+f7HrL3zWdIlaWOVy+bul7x+WgZ0sBW5cI/4Cw=,iv:yTE8kKbvGlup+NTSa4r1dzeug4Va4QFb7zxNMR1P0Uc=,tag:CzusjbwtGKHYRRYhuaREVw==,type:str]
+sops_unencrypted_suffix=_unencrypted
+sops_version=3.9.4

infra/tofu/terraform.tfvars.example

@@ -36,7 +36,7 @@ ssh_authorized_key = "ssh-ed25519 AAAA...replace-with-your-public-key... user@ho
 
 # --- per-node sizing ---
 # vcpu          = 6
-# memory_mib    = 16384
+# memory_mib    = 24576
 # os_disk_gib   = 40
 # data_disk_gib = 100
 # cpu_type      = "host"

infra/tofu/tf.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# infra/tofu/tf.sh — run OpenTofu with EVERY secret sourced from the SOPS/age-encrypted
+# secrets.sops.env. No plaintext .r2.env, no manual secret entry, secrets live encrypted
+# in git like the repo's other secrets.
+#
+# MUST run inside the dev container (it has sops + tofu, and the age PRIVATE key mounted
+# from ~/.config/sops/age/keys.txt):
+#
+#     ./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh init
+#     ./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh plan
+#     ./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh apply
+#
+# What it does:
+#   1. `sops exec-env` decrypts secrets.sops.env and puts the dotenv keys in the ENV
+#      (exec-env parses the dotenv itself, so values with shell-special chars — ! ( ) ; —
+#      are set safely, unlike `source`). It then re-execs THIS script (phase 2).
+#   2. Phase 2 exports PROXMOX_VE_* / AWS_* (already in env) + TF_VAR_proxmox_endpoint,
+#      builds the multi-line TF_ENCRYPTION HCL from the single TF_STATE_PASSPHRASE scalar
+#      (same pbkdf2 → aes_gcm → state/plan structure the old .r2.env hardcoded), and runs
+#      `tofu`. For `init` it appends `-backend-config=backend.r2.hcl` (the non-secret R2
+#      bucket/key/endpoint); other subcommands are passed through unchanged.
+#
+# The Proxmox API token is injected ONLY as PROXMOX_VE_API_TOKEN (provider reads it from
+# the env) — it is NEVER a tofu variable, so it never enters state, vars, or plan output.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SECRETS="${HERE}/secrets.sops.env"
+BACKEND="${HERE}/backend.r2.hcl"
+
+# ── Phase 1: decrypt secrets into the environment, then re-exec self ──────────────────
+if [[ "${_TF_SH_SECRETS_LOADED:-}" != "1" ]]; then
+  command -v sops >/dev/null 2>&1 || {
+    echo "tf.sh: 'sops' not found — run me inside the dev container:" >&2
+    echo "       ./infra/devcontainer/dev.sh run ./infra/tofu/tf.sh $*" >&2
+    exit 127
+  }
+  [[ -f "$SECRETS" ]] || { echo "tf.sh: missing $SECRETS" >&2; exit 1; }
+  export _TF_SH_SECRETS_LOADED=1
+  # Re-exec THIS script (with its args) under sops exec-env so the dotenv values are in
+  # the process environment. printf %q keeps args intact through exec-env's shell.
+  exec sops exec-env "$SECRETS" "$(printf '%q ' "${BASH_SOURCE[0]}" "$@")"
+fi
+
+# ── Phase 2: secrets are now in the environment ───────────────────────────────────────
+: "${TF_STATE_PASSPHRASE:?tf.sh: TF_STATE_PASSPHRASE not set (fill in secrets.sops.env)}"
+: "${PROXMOX_VE_ENDPOINT:?tf.sh: PROXMOX_VE_ENDPOINT not set (fill in secrets.sops.env)}"
+
+# Enable versions.tf endpoint scheme-normalization (harmless if already has https://).
+export TF_VAR_proxmox_endpoint="${PROXMOX_VE_ENDPOINT}"
+
+# Build the state-at-rest encryption blob from the single passphrase scalar.
+export TF_ENCRYPTION="
+key_provider \"pbkdf2\" \"state\" {
+  passphrase = \"${TF_STATE_PASSPHRASE}\"
+}
+method \"aes_gcm\" \"state\" {
+  keys = key_provider.pbkdf2.state
+}
+state {
+  method = method.aes_gcm.state
+}
+plan {
+  method = method.aes_gcm.state
+}
+"
+
+cd "$HERE"
+
+# `-backend-config` is only valid for `init`; inject it there, pass everything else through.
+if [[ "${1:-}" == "init" ]]; then
+  shift
+  exec tofu init -backend-config="$BACKEND" "$@"
+fi
+exec tofu "$@"

infra/tofu/variables.tf

@@ -83,9 +83,9 @@ variable "vcpu" {
 }
 
 variable "memory_mib" {
-  description = "RAM per node in MiB (16 GiB)."
+  description = "RAM per node in MiB (24 GiB)."
   type        = number
-  default     = 16384
+  default     = 24576
 }
 
 variable "os_disk_gib" {
@@ -100,6 +100,24 @@ variable "data_disk_gib" {
   default     = 128
 }
 
+# --- dedicated SSD etcd disk (interim etcd-on-SSD isolation, docs/plans/ssd-etcd-isolation.md) --
+# etcd WAL fsync was being starved by Longhorn replica-rebuild I/O on the shared HDD
+# `Storage` pool (OS + Longhorn disks). A tiny dedicated SSD disk per node moves the
+# etcd data dir (/var/lib/rancher/k3s/server/db, mounted by node_baseline) off the HDD.
+# Interim bridge until the all-SSD mini-PCs land; Longhorn stays on HDD (does not fit).
+
+variable "etcd_ssd_datastore" {
+  description = "SSD-backed datastore for the dedicated etcd disk (VERIFIED: local-zfs = rpool/data, ZFS mirror on Micron 5100 SATA SSDs w/ power-loss protection)."
+  type        = string
+  default     = "local-zfs"
+}
+
+variable "etcd_disk_gib" {
+  description = "Dedicated SSD etcd disk per node, GiB (etcd DB is ~83 MB; 16 GiB is generous headroom)."
+  type        = number
+  default     = 16
+}
+
 variable "cpu_type" {
   description = "Proxmox vCPU type. `host` passes through the physical CPU (nested-virt for KubeVirt, Phase 5)."
   type        = string

platform-config/kustomization.yaml

@@ -57,10 +57,14 @@ resources:
   - ../platform/keycloak/k8s-oidc-rbac.yaml   # maps OIDC groups -> cluster roles (kubectl SSO RBAC)
   - ../platform/keycloak/httproute.yaml
   - ../platform/keycloak/servicemonitor.yaml
-  # oauth2-proxy: forward-auth sign-in/callback ingress + ServiceMonitor (Gateway API / monitoring CRs).
-  # Re-enabled together with the platform/oauth2-proxy component now Keycloak is live and split-horizon
-  # DNS resolves to the Proxmox Gateway VIP (192.168.1.53).
-  - ../platform/oauth2-proxy/httproute.yaml
-  - ../platform/oauth2-proxy/servicemonitor.yaml
+  # oauth2-proxy: forward-auth httproute + ServiceMonitor MOVED OUT of platform-config. They now
+  # travel with the oauth2-proxy controller in its own post-platform Flux Kustomization
+  # (clusters/base/oauth2-proxy.yaml, dependsOn platform-config). Keeping them here — while the
+  # controller sat in the gating `platform` tier — was one half of the twice-proven cold-start
+  # deadlock (platform never Ready -> platform-config never reconciles -> these never apply).
   # monitoring SSO proof: Grafana native-OIDC login ingress (Gateway API CR)
   - ../platform/monitoring/grafana-httproute.yaml
+  # headlamp: httproute MOVED OUT of platform-config. It now travels with the headlamp component in
+  # its own post-platform Flux Kustomization (clusters/base/headlamp.yaml, dependsOn platform-config).
+  # Keeping it here — while the HelmRelease sat in the gating `platform` tier — meant a headlamp
+  # crashloop stalled platform-config (and everything below it). See DECISIONS-LOG "Headlamp gating stall".

platform/backup/minio-credentials.sops.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: minio-backup-credentials
+    namespace: backup
+type: Opaque
+stringData:
+    #ENC[AES256_GCM,data:V1G4ViUPhrnkiRLFNZyYbSaEPFVLsZ74gB4zo1Y7AM2QP7R5lWVEvDO9VgLVatRZwt7cbeObMAYXeI873tZ9hOi2JNSvYuuXrscC3q8nMuJGswvIMvkM,iv:sGb4E/j5mhtkwY05YP/U7lCa9UHV/35dGCziymGUJHo=,tag:S5KnPKRaxJuGCMwPZkIFBw==,type:comment]
+    S3_ENDPOINT: ENC[AES256_GCM,data:UwzKnjDVv3HoEeAPxlzkhUifL6hmebM=,iv:IWmiCYrvuIBez2i3qyDs6Yu4mQ0YuNYv46Rn3+ksm/k=,tag:dlefmBsY3TuLZPAPmaWs/g==,type:str]
+    S3_REGION: ENC[AES256_GCM,data:GvGIw6qxx9M4,iv:uArcS3uwHKwjif00ICIc0Og0OF98OL10ArhT6teJzh4=,tag:2tn4XykslMtwYk7C4m8ndw==,type:str]
+    S3_FORCE_PATH_STYLE: ENC[AES256_GCM,data:5cEjjA==,iv:k5R7iszw/kxYLSH8aqIAHmq8ePdL7xYYPQcutHloN7s=,tag:DJPE0500UP5ECYuBJM9nBg==,type:str]
+    VELERO_BUCKET: ENC[AES256_GCM,data:Q1Oo4Amz,iv:XnAT/Nn8C9BmnE5e0d1LI0I0BVSUL7C+mI+s31FrMYI=,tag:oWDHC2jjKYpKNXA98wYqpQ==,type:str]
+    CNPG_BUCKET: ENC[AES256_GCM,data:P3BUXg==,iv:BPuFgI9iL9lWZhQZ0k3skrBwyp1UDLyz0aIyq6/V7nI=,tag:R4n30UxwcCY1k+X/bu7pxw==,type:str]
+    #ENC[AES256_GCM,data:A6h/rEzdn5Sp1BLDnxoHrYdEwiB9Eb/5NAxCMZRXvoVcnTfOvLHVA0ntno+9GwMueIhwN9WgG7fnCHhG7A==,iv:1+VfHF59PppwpLiYW4gUTwgGcvXMEzAnvowVvE/awFU=,tag:skLHPchOW2PZzjqqaJ2TSQ==,type:comment]
+    AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:wNnfGYBspJDyYw==,iv:d9fK8E5ujPu9vXV1yRR5AgO3+j69EWfbWEoMt7Lq06w=,tag:0uuRSadwtodLol2Sw2ih9Q==,type:str]
+    AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:IywofASX3m8sznW3StMf/SvCPBcvxCUlT4cIQvu4gF9RglHlFCujwi8amnm7ASfc,iv:JnyMBQQ7oLg8TAVx7RveZHYDo6a/bl3t1IEPRxZufI8=,tag:5rUUqd6ZByuLIlZxQV1dPQ==,type:str]
+    #ENC[AES256_GCM,data:J5fQZE/UbhqpWo5OXzvGxCNTb5f6r+sh55wUHf3M0H4oRU2dyna7nJqFQzP1EQXfJTlxjZOpBkzZyppn,iv:bDGljEMmMiieCvfmVO8uhfw/O1rUslNvNAY6LfFFA4E=,tag:D9Gw78gtCigsWJnlmFzMdA==,type:comment]
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:TYn3AVIkjU4QhA==,iv:09W2JkJywD5aadOr+XkbZlR2sSz3glT+yknI4Z46C9o=,tag:QGwIp3seKuemzi4jysDRcg==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:lZ9fAqwPGwiKbTJYyvNGMOS9L37IB3/GcfpuZH3G2s0of3YII6vyPlAKZeGI6i2P,iv:Y0p8EDzZS3uUZ+BBHjjBgDmkyphwZ43X7VfZJboGS1s=,tag:rQ0EyRBEPNG41PGrqHKkhw==,type:str]
+    #ENC[AES256_GCM,data:EbjHG+jG6OnhZzWUAamIomBYYa5jwogBWlAgOPyIdFivuNZGDSbkyZ0dsUNJJ8yhl7lkIjvi1VvGA/B2QXdCw0pP4xxCs84uLSgpHDWB9v4G5cjF+kThANvaMomKoQ==,iv:1JpFJPSTXVxKVkLBkaYZAYzRHCmqSAgNYGmcW4ajZxo=,tag:9vWPXJ3COQC4m2AHCWPOlg==,type:comment]
+    cloud: ENC[AES256_GCM,data:RmDGXXK5ct97WO18YAYIUF3zPE7nb1zkuJV8pBZPkNEHV6y1Iup/RXAMg+l1rXcI7xdPSiMfSJ6FMudfxOzCzAJPRGxoHMWhzO5N9H+O30KOhHwYpeud2mL4xNZPzOvQgpszqJQzk82D5tnjGeE=,iv:wz16XKNzAWrOZv+Rx68MArqRJHZmRVX/613w7/3WXH4=,tag:5cxblz07Mepy9xOZ3pko2w==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5Z1hkZTlrWmpsdzBuODlM
+            NXQvTXhmb00rMWNqY1l4Ly8yVjVrNXFSMXdRCmxMNXlBTEJyVncrcHVrWXFoZHBI
+            L3JLM3lLVHBJNFVHY3k0MVBjYk5USTgKLS0tIFJ4WWxFL0Flc2J5V3ZSU1JhSVE1
+            TDFyWmhqQUdCOTUyZGVnRDQ4d2lSRnMKPBexDlO9fZK6K6Rpcs2RzYE4drYtivwk
+            QsnSigjEPY35ixsPTMHTdTP0Almxc4U6lhT4gsIEqk3hqmJ3fcmzzA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-29T03:09:53Z"
+    mac: ENC[AES256_GCM,data:y1Uxrj9yAuZCryfCviU0pPplcQ0ccxcJSuNov4CvZKwnIimHVIsr7sU1mw3ZqHV/arQ2SjXkfDHxAEhvL6WSAjEjnDNaCMm++mPrfKXoGZhDU3gjaeyI7W6Iexo/bfc/oR6hALRjp6g7nTK6Yqdi6UMRoZZ+1YGSkFkRJjygLF4=,iv:9scdNtJQk6TkB/SN2yeFdhmiVCtGd2qRsD3N+wv0L5o=,tag:dYFy6oKEUIr7BLstFrhsEA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

platform/backup/minio-credentials.sops.yaml.example

@@ -0,0 +1,51 @@
+# =============================================================================
+# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize and never
+# auto-encrypted by SOPS). Reference scaffold + OPERATOR HAND-OFF.
+#
+# WHAT THIS IS: the scoped (non-root) MinIO S3 access key for the off-cluster backup target that
+# roles/minio_backup_target stands up on the Proxmox host (playbooks/backup-minio.yml). It is the
+# credential Velero (R-STATE-03) and CloudNativePG / Barman (R-IDP-05) use to write backups. The
+# SAME key/secret live (ansible-vault) in infra/ansible/inventory/group_vars/proxmox_hosts/vault.yml
+# as minio_backup_target_access_key/_secret_key — the two MUST stay in sync.
+#
+# ENDPOINT (not secret):  http://192.168.1.9:9000   (path-style; region us-east-1 is a placeholder,
+#                         MinIO ignores it but S3 clients require one). Buckets: velero, cnpg.
+#
+# HAND-OFF STEPS (operator):
+#  (a) Copy to:      platform/backup/minio-credentials.sops.yaml
+#  (b) Fill the access key + secret (== the vaulted scoped key on the Proxmox host).
+#  (c) Encrypt:      sops -e -i platform/backup/minio-credentials.sops.yaml
+#  (d) Wire per consumer (WS-STORAGE / WS-SEC):
+#        - Velero  → credentials.existingSecret pointing at a Secret with the `cloud` INI key,
+#                    backupStorageLocation.config.s3Url=S3_ENDPOINT, s3ForcePathStyle="true",
+#                    bucket=velero (namespace: velero).
+#        - CNPG    → Barman objectStore endpointURL=S3_ENDPOINT, destinationPath=s3://cnpg,
+#                    s3Credentials.accessKeyId/secretAccessKey from ACCESS_KEY_ID/ACCESS_SECRET_KEY
+#                    (namespace: cnpg-system).
+#      A Secret is namespace-scoped — replicate/rename this one into each consumer namespace as
+#      that wiring requires.
+# =============================================================================
+apiVersion: v1
+kind: Secret
+metadata:
+    name: minio-backup-credentials
+    namespace: backup
+type: Opaque
+stringData:
+    # ── endpoint + buckets (not secret; single source of truth) ──
+    S3_ENDPOINT: "http://192.168.1.9:9000"
+    S3_REGION: "us-east-1"
+    S3_FORCE_PATH_STYLE: "true"
+    VELERO_BUCKET: "velero"
+    CNPG_BUCKET: "cnpg"
+    # ── scoped access key (velero + cnpg buckets only) ──
+    AWS_ACCESS_KEY_ID: "REPLACE-ME"
+    AWS_SECRET_ACCESS_KEY: "REPLACE-ME"
+    # CNPG Barman object store consumes these discrete key names.
+    ACCESS_KEY_ID: "REPLACE-ME"
+    ACCESS_SECRET_KEY: "REPLACE-ME"
+    # Velero's aws plugin reads an INI credentials file from the `cloud` key of its existingSecret.
+    cloud: |
+        [default]
+        aws_access_key_id=REPLACE-ME
+        aws_secret_access_key=REPLACE-ME

platform/crafty/README.md

@@ -0,0 +1,119 @@
+# platform/crafty
+
+**Crafty Controller 4** — a web panel that creates and manages multiple Minecraft servers (Java +
+Bedrock) as **subprocesses inside its own container**. Its web UI is **Keycloak-gated** via a per-pod
+**oauth2-proxy reverse-proxy sidecar** (the decided **Option B**, see
+[`../oauth2-proxy/README.md`](../oauth2-proxy/README.md)). Game ports are exposed **un-gated** on a
+Cilium L2 LoadBalancer (game protocols authenticate players themselves).
+
+- **Owner:** WS-APP (auth wiring shared with WS-SEC)
+- **Flux tier:** own non-gating `clusters/base/crafty.yaml` (`dependsOn platform-config`, `wait:false`)
+- **Namespace:** `crafty` (PLATFORM namespace — `gateway-access=shared`, NOT `app-namespace=true`)
+
+## Deploy summary
+
+| Thing | Value |
+|---|---|
+| Image | `registry.gitlab.com/crafty-controller/crafty-4:4.10.8` (current stable; bare-semver tags, no `v`) |
+| Web UI | HTTPS **self-signed** on `:8443` (reached only via the sidecar / same-pod) |
+| Sidecar | `quay.io/oauth2-proxy/oauth2-proxy:v7.8.1`, reverse-proxy on `:4180` |
+| Data | ONE Longhorn RWO PVC `crafty-data` (20Gi), mounted 5× by subPath |
+| PVC subPaths | `servers`→`/crafty/servers`, `backups`→`/crafty/backups`, `logs`→`/crafty/logs`, `import`→`/crafty/import`, `config`→`/crafty/app/config` |
+| Identity | `runAsUser:1000 runAsGroup:0 fsGroup:0 runAsNonRoot:true` (see below) |
+| Web host | `crafty.home.keithjackson.dev` → shared Gateway → `crafty` Service :80 → sidecar :4180 → Crafty :8443 |
+| Game VIP | **192.168.1.55** (Cilium L2 LoadBalancer; `.53`=Gateway, `.54`=unifi, `.55`=crafty) |
+| Game ports | starter set: **TCP 25565-25567** (Java), **UDP 19132-19134** (Bedrock) |
+
+### fsGroup / user decision (the seerr lesson, applied)
+
+Crafty's Dockerfile does `useradd -g root -M crafty` (UID **1000**, primary group **root/GID 0**),
+`chown -R crafty:root /crafty`, and makes `/crafty` group-writable — the OpenShift/K8s
+"arbitrary-UID, root-group-perms" pattern. A fresh Longhorn PVC is `root:root` and not group-writable,
+so the non-root crafty process could not create `/crafty/servers/*` etc. and would CrashLoop (exactly
+seerr's failure at UID 1000 on a `root:root` PVC). Fix: **`fsGroup: 0`** makes the kubelet group-own
+the mounted volume to GID 0, mark it group-writable+SGID, and add GID 0 as a supplemental group;
+**`runAsGroup: 0`** + **`runAsUser: 1000`** pin the image's own `crafty:root` identity. Longhorn's
+`CSIDriver fsGroupPolicy: ReadWriteOnceWithFSType` applies fsGroup to this RWO filesystem volume.
+
+> **HUMAN runtime check:** confirm the pod writes its SQLite DB and can start a server. If the image's
+> `crafty` UID is ever not 1000, adjust `runAsUser` (the launcher's non-root path runs directly as the
+> current UID, so any non-root UID with GID 0 works, but 1000 matches `getpwuid` cleanly).
+
+## Auth = Option B (per-app oauth2-proxy reverse-proxy)
+
+Cilium 1.16.5 cannot inject `ext_authz` into the shared Gateway's machine-generated Envoy config, and
+native Gateway-API ExternalAuth ships in no released Cilium yet (see `../oauth2-proxy/README.md`). So
+Crafty's UI is gated by a **dedicated oauth2-proxy sidecar in the Crafty pod** (`--reverse-proxy`,
+`--upstream=https://127.0.0.1:8443`, `--ssl-upstream-insecure-skip-verify=true`,
+`--allowed-groups=platform-admins`, `--skip-provider-button`, cookie/redirect for
+`crafty.home.keithjackson.dev`). The `crafty` Service targets the sidecar, never Crafty's `:8443`, so
+the gate is **in the data path** (fail-closed: no valid `platform-admins` session ⇒ no upstream).
+
+> **Note:** Crafty ALSO has its own local admin login (it has no OIDC). After oauth2-proxy admits the
+> user, they still see Crafty's own login page — a second, defense-in-depth factor. The operator logs
+> in with Crafty's local admin credentials (set on first boot). This is expected, not a bug.
+
+### Keycloak client
+
+Confidential auth-code client **`crafty`** in the `homelab` realm — redirect
+`https://crafty.home.keithjackson.dev/oauth2/callback`, `webOrigins` the same host, a `groups`
+membership mapper (bare names). Provisioned **live via kcadm** at authoring time (2026-08-28) and
+captured as the durable GitOps source of truth in
+[`../keycloak/keycloak-realm-import.yaml`](../keycloak/keycloak-realm-import.yaml) (secret injected via
+the `CRAFTY_CLIENT_SECRET` placeholder).
+
+### Secret locations (SOPS/age)
+
+| Secret | File | Key |
+|---|---|---|
+| oauth2-proxy client id/secret + cookie secret | `crafty-oauth2-proxy-credentials.sops.yaml` (this dir) | `client-id`, `client-secret`, `cookie-secret` |
+| Keycloak client secret (mirror) | `../keycloak/keycloak-realm-clients.sops.yaml` | `crafty-client-secret` |
+
+`client-secret` here **must equal** `crafty-client-secret` there (cross-namespace mirror — a Secret
+can't span namespaces). Both are age-encrypted to the repo `sops-age` recipient and Flux-decryptable
+(verified: both decrypt and the two values match).
+
+## Game-port L4 exposure
+
+The Gateway is L7-HTTPS-only; raw TCP/UDP game ports can't ride it. `service-l4.yaml` exposes them as
+two `type=LoadBalancer` Services (TCP + UDP) sharing the pinned VIP **192.168.1.55**
+(`lbipam.cilium.io/sharing-key: crafty`), ARP-announced by the existing `default-l2` policy. This path
+is **NOT auth-gated** — Minecraft Java (Mojang/Microsoft) and Bedrock (Xbox Live) authenticate players
+themselves. `external-dns` publishes `crafty-mc.home.keithjackson.dev → 192.168.1.55`.
+
+### Adding more game ports (when the operator creates servers)
+
+The operator has **not created any servers yet**; the starter set covers 3 Java + 3 Bedrock slots. To
+add capacity:
+
+1. In the Crafty UI, create the server and bind it to a NEW port (Java TCP or Bedrock UDP).
+2. Add that port to the matching Service in `service-l4.yaml` (one
+   `{ name, port, targetPort, protocol }` line — K8s Services enumerate ports, no ranges).
+3. Add it to the game-port ingress rule in `networkpolicy.yaml` (`crafty-allow-ingress`).
+4. (Optional) add a `containerPort` in `deployment.yaml` for documentation.
+5. Commit → Flux reconciles.
+
+## Policies
+
+`namespace.yaml` sets only `gateway-access=shared` (platform namespace, like headlamp/homepage), so
+Kyverno does not auto-generate the app baseline. `networkpolicy.yaml` ships it explicitly (defense in
+depth): default-deny (DNS-only egress) + Cilium allows — Gateway→sidecar `:4180`, world/LAN→game
+ports, host→probe ports, and egress `world` (server-jar downloads + the OIDC path to Keycloak via the
+Gateway VIP). The sidecar→Crafty hop is pod-local (`127.0.0.1`) and not subject to network policy.
+
+## Verification
+
+- `make -C ci gate` — green (hand-authored platform manifests; render-golden unaffected).
+- `kustomize build` of `platform/crafty`, `clusters/base`, `clusters/dev` — clean.
+
+### Human tests (post-deploy)
+
+1. **Browser OIDC round-trip:** incognito → `https://crafty.home.keithjackson.dev` → expect a 302 to
+   `https://keycloak.home.keithjackson.dev/realms/homelab/...` (NOT the Crafty UI). Log in as a
+   **platform-admins** user (MFA/OTP enforced) → land on Crafty's own login → sign in with Crafty's
+   local admin. A **non-platform-admins** user → **403** (authenticated but not authorized).
+2. **Pod health / fsGroup:** `kubectl -n crafty get pod` Ready 2/2; Crafty writes its DB under
+   `/crafty/app/config` and `/crafty/servers` without EACCES.
+3. **Server-create + game reach:** in the Crafty UI create a Java server bound to `25565`; start it;
+   connect a Minecraft client to `crafty-mc.home.keithjackson.dev:25565` (or `192.168.1.55:25565`)
+   from the LAN and confirm the world loads. (Bedrock: a server on `19132/udp` reached at the VIP.)

platform/crafty/crafty-oauth2-proxy-credentials.sops.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: crafty-oauth2-proxy
+    namespace: crafty
+type: Opaque
+stringData:
+    client-id: ENC[AES256_GCM,data:86ALvhXE,iv:S+guxBop+DkVX+SxxXObtzt5MYKVXo4NRksGPqwfUhA=,tag:Gllc0vKia1X1PFBF663apw==,type:str]
+    client-secret: ENC[AES256_GCM,data:0g/6FjJGHc8ndxkb9fJGvAmDWRVb7DhkC7hYubbQuncXjmo/9F33E/qMw4Zie/XOWL0hrb+OpdXETa4xH3XQeA==,iv:Wa2TPUfwBlxIntVz0cHbarGdK7SpL2KL24i1N1YYeWQ=,tag:AMEu0YI3O+4hWSJm4hsHaA==,type:str]
+    cookie-secret: ENC[AES256_GCM,data:FXHwQtpz1dGTV9gkzgx92Dh8yl3MbYZB/Q5Txfd9q7dbxDyKcmsOlxhAGuk=,iv:wVwcgbLEayUpDzMbTwMkT+jg3jVb/ztoqbXaKL41cVA=,tag:Q1Yl8PrkNBFg3x7OjYmCCQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPa2I3ZU9zQUs3bnQ0dmJx
+            U3o0YXoyRWdaWmRrK2tBL0JRZjU1QTNCczA4CkI5RVZJRWFrWkJyL2ZJNU9EZWRP
+            ZjloR3VkUjFEeE93c2lsKzJSaTVTMFkKLS0tIGhyREVqWVZSanNnOHBLak9IdTQv
+            S0l3WFdPMFNzT1VoaVc2QnBIQXJDVGcKkjpqUkJa7TPYfSrjOxBB3lsffEHy5Thm
+            KuIMdXH5oiaCwORyjyPsFg/h3NOCSgzc0dolbQzoSynYb3vztlwmdQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-28T20:36:54Z"
+    mac: ENC[AES256_GCM,data:zfv1JaZrz2eyQF6SKrTi6YfmGXpKK8miRa5RHiTsq0FPFfFP6Xc7n47RH1tZj5CRgf86wKZB3H7m6jlxZ3mdpshqK3B41sONHC3TGLdJUJt8bw/a5GbyVg+xs41aLcldylq4zrDGXq/aj9c1/lfbtd1xxIpWaNiuvvSK6y21Tmc=,iv:hF5nskThNA/uY0tvYFMwtWrPhDSUqoVX8Y10TgdIDJA=,tag:IxoUv7kbrouxXLY23cElDg==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

platform/crafty/crafty-oauth2-proxy-credentials.sops.yaml.example

@@ -0,0 +1,37 @@
+# =============================================================================
+# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize and never
+# auto-encrypted by SOPS). Reference scaffold + OPERATOR HAND-OFF.
+#
+# WHAT THIS IS: the OIDC client + cookie secrets for the per-app oauth2-proxy SIDECAR that gates the
+# Crafty web UI (deployment.yaml, container `oauth2-proxy`). The sidecar reads these three keys as
+# OAUTH2_PROXY_CLIENT_ID / OAUTH2_PROXY_CLIENT_SECRET / OAUTH2_PROXY_COOKIE_SECRET (env, by name only;
+# R-IDP-04). SOPS-encrypted; keys stay cleartext for reviewable diffs.
+#
+# The `client-secret` value MUST equal the `crafty-client-secret` placed in
+# platform/keycloak/keycloak-realm-clients.sops.yaml (which the realm import injects into the Keycloak
+# `crafty` client) — same cross-namespace mirror pattern as grafana/gitea/headlamp (a Secret cannot be
+# shared across namespaces, so the one value is stored in two encrypted files).
+#
+# HAND-OFF STEPS (operator):
+#  (a) client-id is fixed: `crafty` (matches the Keycloak client).
+#  (b) client-secret: use the SAME value generated for keycloak-realm-clients crafty-client-secret
+#         openssl rand -hex 32
+#  (c) cookie-secret: a fresh random 32-byte value, url-safe base64:
+#         openssl rand -base64 32 | tr -- '+/' '-_'
+#  (d) Copy to:      platform/crafty/crafty-oauth2-proxy-credentials.sops.yaml
+#  (e) Fill values; encrypt:
+#         sops -e -i platform/crafty/crafty-oauth2-proxy-credentials.sops.yaml
+#  (f) It is already listed under `resources:` in platform/crafty/kustomization.yaml.
+# =============================================================================
+apiVersion: v1
+kind: Secret
+metadata:
+  name: crafty-oauth2-proxy
+  namespace: crafty
+type: Opaque
+stringData:
+  client-id: crafty
+  # MUST match keycloak-realm-clients crafty-client-secret.
+  client-secret: REPLACE-ME
+  # Random 32-byte url-safe-base64 cookie signing secret (oauth2-proxy).
+  cookie-secret: REPLACE-ME

platform/crafty/deployment.yaml

@@ -0,0 +1,189 @@
+---
+# Crafty Controller pod = TWO containers:
+#   1. crafty        — the panel + every Minecraft server it creates (Java + Bedrock) run as
+#                      SUBPROCESSES inside THIS container. Serves its web UI over HTTPS (self-signed)
+#                      on :8443. Never exposed directly — only the sidecar and the game-port LB reach it.
+#   2. oauth2-proxy  — a per-pod REVERSE-PROXY (Option B, platform/oauth2-proxy/README.md). The shared
+#                      Gateway routes crafty.home.keithjackson.dev -> this sidecar (:4180); the sidecar
+#                      authenticates the browser against Keycloak (homelab realm, group platform-admins)
+#                      and only THEN proxies to the co-located Crafty UI at https://127.0.0.1:8443. This
+#                      is how Crafty's UI (which has NO native OIDC) is Keycloak-gated without ext_authz
+#                      on the shared Gateway (impossible on Cilium 1.16.5 — see the oauth2-proxy README).
+#
+# Rendered by its OWN non-gating post-platform Flux Kustomization (clusters/base/crafty.yaml,
+# dependsOn platform-config, wait:false) — a leaf UI must NEVER sit in the gating `platform` tier
+# (DECISIONS-LOG: "Headlamp gating stall").
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crafty
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+spec:
+  replicas: 1
+  # Crafty owns a single RWO Longhorn volume with live SQLite + running server subprocesses; a second
+  # replica would double-mount/corrupt it. Recreate (not RollingUpdate) so the old pod releases the RWO
+  # PVC before the new one attaches.
+  strategy:
+    type: Recreate
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: crafty
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: crafty
+    spec:
+      automountServiceAccountToken: false
+      # ── IDENTITY / STORAGE PERMISSIONS (the seerr fsGroup lesson, applied) ───────────────────────
+      # The Crafty image runs as the non-root user `crafty` (UID 1000) whose PRIMARY GROUP is root
+      # (GID 0): its Dockerfile does `useradd -g root -M crafty` + `chown -R crafty:root /crafty` and
+      # makes /crafty group-writable — the OpenShift/K8s "arbitrary-UID, root-group-perms" pattern the
+      # Crafty docs call out. A fresh Longhorn PVC is root:root and NOT group-writable, so the crafty
+      # process (GID 0) could not create /crafty/servers/* etc. -> CrashLoop (same failure seerr hit as
+      # UID 1000 on a root:root PVC). fsGroup=0 makes the kubelet group-own the mounted volume to GID 0,
+      # mark it group-writable + SGID, and add GID 0 as a supplemental group, so the crafty user can
+      # write — Longhorn's CSIDriver fsGroupPolicy: ReadWriteOnceWithFSType applies fsGroup to this RWO
+      # filesystem volume. runAsUser/runAsGroup pin the image's own crafty:root identity; runAsNonRoot
+      # asserts it. (HUMAN runtime check at bring-up: confirm the pod writes the DB and starts a server;
+      # if the image's crafty UID is not 1000, adjust runAsUser — see README.)
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 0
+        fsGroup: 0
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        # ─────────────────────────────── Crafty panel + MC servers ──────────────────────────────────
+        - name: crafty
+          # registry.gitlab.com/crafty-controller/crafty-4 — official image. Pinned to the current
+          # stable release (4.10.8, 2026-07-29; tags are bare semver, NO `v` prefix). Renovate maintains.
+          image: registry.gitlab.com/crafty-controller/crafty-4:4.10.8
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: TZ
+              value: "America/Chicago"
+          ports:
+            - { name: https-ui, containerPort: 8443, protocol: TCP }
+            # Starter game ports the operator assigns to servers created in the panel. Declared here for
+            # documentation + to back the game-port LoadBalancer (service-l4.yaml). Add more as needed.
+            - { name: mc-java-0, containerPort: 25565, protocol: TCP }
+            - { name: mc-java-1, containerPort: 25566, protocol: TCP }
+            - { name: mc-java-2, containerPort: 25567, protocol: TCP }
+            - { name: mc-bedr-0, containerPort: 19132, protocol: UDP }
+            - { name: mc-bedr-1, containerPort: 19133, protocol: UDP }
+            - { name: mc-bedr-2, containerPort: 19134, protocol: UDP }
+          resources:
+            requests:
+              cpu: 250m
+              memory: 1Gi
+            # Minecraft JVMs are memory-hungry; cap generously for a starter set of small servers.
+            limits:
+              memory: 4Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            # readOnlyRootFilesystem intentionally UNSET: Crafty's launcher writes runtime state under
+            # /crafty (app cache, per-server working dirs) beyond the mounted subPaths; hardening it
+            # needs writable emptyDirs and is a deferred, runtime-verified follow-up (as headlamp notes).
+          volumeMounts:
+            # ONE Longhorn PVC mounted five times by subPath — exactly the dirs Crafty's docker-compose
+            # maps. Everything else under /crafty is read-only image content.
+            - { name: crafty-data, mountPath: /crafty/servers, subPath: servers }
+            - { name: crafty-data, mountPath: /crafty/backups, subPath: backups }
+            - { name: crafty-data, mountPath: /crafty/logs, subPath: logs }
+            - { name: crafty-data, mountPath: /crafty/import, subPath: import }
+            - { name: crafty-data, mountPath: /crafty/app/config, subPath: config }
+          # Crafty inits its SQLite DB + Tornado server on first boot (can take ~30-60s). TCP-probe the
+          # HTTPS UI port (an HTTPS GET would fight the self-signed cert). Generous startup budget.
+          startupProbe:
+            tcpSocket:
+              port: https-ui
+            periodSeconds: 10
+            failureThreshold: 30
+          livenessProbe:
+            tcpSocket:
+              port: https-ui
+            periodSeconds: 20
+            failureThreshold: 6
+          readinessProbe:
+            tcpSocket:
+              port: https-ui
+            periodSeconds: 15
+        # ─────────────────────────── oauth2-proxy reverse-proxy sidecar ─────────────────────────────
+        - name: oauth2-proxy
+          # quay.io/oauth2-proxy/oauth2-proxy — same authenticator family as platform/oauth2-proxy, but
+          # here in per-app REVERSE-PROXY mode (--upstream is the co-located Crafty UI, not static://202).
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - --http-address=0.0.0.0:4180
+            - --reverse-proxy=true
+            - --provider=oidc
+            - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
+            # Gate the co-located Crafty UI. Crafty serves HTTPS with a SELF-SIGNED cert, so skip verify
+            # on this in-pod (127.0.0.1) hop; the browser-facing TLS is the trusted wildcard at the Gateway.
+            - --upstream=https://127.0.0.1:8443
+            - --ssl-upstream-insecure-skip-verify=true
+            - --email-domain=*
+            - --scope=openid email profile
+            - --oidc-groups-claim=groups
+            # R-IDP-06: a valid Keycloak session is NOT enough — the user must ALSO be in platform-admins,
+            # else /oauth2 returns 403. (Matches the platform oauth2-proxy floor.)
+            - --allowed-group=platform-admins
+            # Single provider => 302 straight to Keycloak, no oauth2-proxy interstitial button.
+            - --skip-provider-button=true
+            - --redirect-url=https://crafty.home.keithjackson.dev/oauth2/callback
+            - --cookie-domain=.home.keithjackson.dev
+            - --whitelist-domain=.home.keithjackson.dev
+            - --cookie-secure=true
+            - --pass-access-token=true
+            - --set-xauthrequest=true
+          env:
+            # client-id / client-secret MUST match the Keycloak `crafty` client
+            # (keycloak-realm-clients secret, crafty-client-secret). cookie-secret is a random 32-byte
+            # value. All three from the SOPS-encrypted crafty-oauth2-proxy Secret, by name only (R-IDP-04).
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom:
+                secretKeyRef: { name: crafty-oauth2-proxy, key: client-id }
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef: { name: crafty-oauth2-proxy, key: client-secret }
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom:
+                secretKeyRef: { name: crafty-oauth2-proxy, key: cookie-secret }
+          ports:
+            - { name: http, containerPort: 4180, protocol: TCP }
+          resources:
+            requests:
+              cpu: 25m
+              memory: 64Mi
+            limits:
+              memory: 128Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 15
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 15
+      volumes:
+        - name: crafty-data
+          persistentVolumeClaim:
+            claimName: crafty-data

platform/crafty/httproute.yaml

@@ -0,0 +1,36 @@
+---
+# North-south ingress for the Crafty web UI at crafty.home.keithjackson.dev. Attaches to the shared
+# Gateway (docs/09); TLS terminated at the Gateway by the *.home.keithjackson.dev wildcard cert (this
+# host is covered). Same-namespace backend, so no ReferenceGrant; the parentRef is permitted because
+# the crafty namespace carries gateway-access=shared. CR of the Gateway API CRDs — the CRDs are
+# registered by `platform`, which is Ready before this tier's platform-config dependency completes.
+#
+# auth-mode annotation records that this route is gated by a per-app oauth2-proxy REVERSE-PROXY sidecar
+# (Option B), NOT by Gateway ext_authz forward-auth (impossible on Cilium 1.16.5) and NOT native OIDC
+# (Crafty has none). The backendRef is the `crafty` Service, which fronts the sidecar (:4180) — so the
+# gate is IN the data path; the route needs no ext_authz filter of its own.
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: crafty
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+  annotations:
+    homelab.keithjackson.dev/auth-mode: oauth2-proxy
+    homelab.keithjackson.dev/auth-groups: platform-admins
+spec:
+  parentRefs:
+    - name: shared
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - crafty.home.keithjackson.dev
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: crafty
+          port: 80

platform/crafty/kustomization.yaml

@@ -0,0 +1,25 @@
+---
+# Crafty Controller — Minecraft management panel, web UI Keycloak-gated by a per-pod oauth2-proxy
+# reverse-proxy sidecar (Option B). Rendered by its OWN post-platform Flux Kustomization
+# (clusters/base/crafty.yaml, dependsOn platform-config, wait:false) — NOT by the gating `platform`
+# tier: a leaf UI whose pod can be legitimately unhealthy must never gate the platform chain
+# (DECISIONS-LOG: "Headlamp gating stall"). Same non-gating pattern as headlamp/homepage/oauth2-proxy.
+#
+# The Gateway API CRs (httproute) + LoadBalancer Services travel WITH the component here: the CRDs they
+# need (gateway.networking.k8s.io, cilium.io) are registered by `platform`, and `platform` is Ready
+# before this tier's platform-config dependency completes, so ordering holds.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  # oauth2-proxy sidecar client + cookie secrets — SOPS-encrypted, decrypted per-env by the crafty
+  # Kustomization (clusters/dev patches it by name). Referenced by name only (R-IDP-04).
+  - crafty-oauth2-proxy-credentials.sops.yaml
+  # default-deny baseline + Cilium ingress(gateway/game)/egress(world) allows.
+  - networkpolicy.yaml
+  # web-UI ingress (crafty.home -> oauth2-proxy sidecar) + game-port L4 LoadBalancer (VIP .55).
+  - httproute.yaml
+  - service-l4.yaml

platform/crafty/namespace.yaml

@@ -0,0 +1,21 @@
+---
+# Crafty Controller — Minecraft server management panel (Java + Bedrock). Its web UI is served at
+# crafty.home.keithjackson.dev via the shared Gateway, GATED by a per-pod oauth2-proxy sidecar in
+# reverse-proxy mode (Keycloak `homelab` realm, group `platform-admins`) — the decided Option-B auth
+# pattern (platform/oauth2-proxy/README.md: Cilium 1.16.5 cannot do ext_authz on the shared Gateway).
+#
+# gateway-access=shared opts this namespace in to attach HTTPRoutes to the shared Gateway (the
+# *.home.keithjackson.dev wildcard cert already covers crafty.home). The Gateway routes to the
+# oauth2-proxy sidecar (:4180), never directly to Crafty's UI.
+#
+# NOTE: intentionally NOT homelab.keithjackson.dev/app-namespace=true. Crafty is a PLATFORM admin tool
+# (like Headlamp in `headlamp` / Homepage in `homepage`), so Kyverno does NOT generate the app
+# default-deny baseline here and the app-scoped auth-mode / egress-class / host-namespace policies do
+# not apply (all Kyverno policies match namespaceSelector app-namespace=true). We still ship an
+# explicit default-deny NetworkPolicy + CiliumNetworkPolicies (networkpolicy.yaml) as defense in depth.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: crafty
+  labels:
+    gateway-access: "shared"

platform/crafty/networkpolicy.yaml

@@ -0,0 +1,98 @@
+---
+# Default-deny baseline (R-NET-01): deny all ingress + egress, allow only DNS. `crafty` is a PLATFORM
+# namespace so Kyverno does NOT auto-generate this (its policies match app-namespace=true) — we ship it
+# explicitly (defense in depth), same shape as the Kyverno-generated app baseline and as
+# headlamp/homepage. Cilium enforces the UNION of this NetworkPolicy and the CiliumNetworkPolicies
+# below (allow if ANY policy allows), so those add the Gateway/game/world paths on top of deny-all.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-deny
+  namespace: crafty
+  labels:
+    homelab.keithjackson.dev/baseline: default-deny
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+---
+# INGRESS allows. The crafty pod runs BOTH containers (oauth2-proxy sidecar :4180 + Crafty :8443 + the
+# MC game-port subprocesses), all selected by app.kubernetes.io/name=crafty.
+#   - Gateway (Cilium `ingress` entity) -> oauth2-proxy sidecar :4180 ONLY. The Gateway can reach the
+#     GATE, never Crafty's :8443 directly (there is no Service for :8443 either).
+#   - LAN players + admins (`world` = off-cluster to Cilium) -> the raw game ports. NOT auth-gated:
+#     Minecraft protocols authenticate the player. `host`/`cluster` cover node-local health probes and
+#     any in-cluster caller.
+#   - `host` (the kubelet) -> :8443 and :4180 for the liveness/readiness probes.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: crafty-allow-ingress
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: crafty
+  ingress:
+    # Gateway -> the oauth2-proxy gate. Kubelet (host) probes the gate too.
+    - fromEntities:
+        - ingress
+        - host
+      toPorts:
+        - ports:
+            - { port: "4180", protocol: TCP }
+    # LAN/world players -> raw game ports (starter set). Add ports here when new servers are created.
+    - fromEntities:
+        - world
+        - cluster
+        - host
+      toPorts:
+        - ports:
+            - { port: "25565", protocol: TCP }   # Java slot 0
+            - { port: "25566", protocol: TCP }   # Java slot 1
+            - { port: "25567", protocol: TCP }   # Java slot 2
+            - { port: "19132", protocol: UDP }   # Bedrock slot 0
+            - { port: "19133", protocol: UDP }   # Bedrock slot 1
+            - { port: "19134", protocol: UDP }   # Bedrock slot 2
+    # Kubelet (host) -> Crafty's HTTPS UI port for its TCP health probe.
+    - fromEntities:
+        - host
+      toPorts:
+        - ports:
+            - { port: "8443", protocol: TCP }
+---
+# EGRESS allows (DNS already permitted by the default-deny baseline above).
+#   - Crafty must reach the INTERNET to download Minecraft server jars (Mojang/PaperMC/Purpur/Fabric,
+#     Bedrock binaries) when the operator creates/updates a server.
+#   - The oauth2-proxy sidecar must reach Keycloak at keycloak.home.keithjackson.dev, which coredns
+#     split-horizon resolves to the Gateway VIP (an L2-announced LoadBalancer IP Cilium classifies as
+#     `world`) — so `world` egress covers both the jar downloads AND the OIDC path (same as headlamp).
+#   - The sidecar -> Crafty hop is pod-local (127.0.0.1:8443) and is NOT subject to network policy.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: crafty-allow-egress
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: crafty
+  egress:
+    - toEntities:
+        - world

platform/crafty/pvc.yaml

@@ -0,0 +1,25 @@
+---
+# Crafty data PVC (R-STATE-01/02) — the SINGLE stateful volume behind Crafty Controller. Crafty runs
+# every Minecraft server it creates as a SUBPROCESS inside its own container, so all server worlds,
+# backups, imports, logs and Crafty's own SQLite config/DB live under /crafty and must persist across
+# pod restarts. This one Longhorn RWO volume is mounted five times by subPath (deployment.yaml) to the
+# exact directories Crafty's docker-compose maps (/crafty/servers, /crafty/backups, /crafty/logs,
+# /crafty/import, /crafty/app/config).
+#
+# Sized 20Gi for a starter set of small worlds + rolling backups; expand via the Longhorn
+# allowVolumeExpansion storage class when real worlds grow. Replicated on the Longhorn 3-replica tier
+# (docs/04). fsGroup/root-group ownership so the non-root `crafty` user can write — see deployment.yaml.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: crafty-data
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 20Gi

platform/crafty/service-l4.yaml

@@ -0,0 +1,72 @@
+---
+# ============================================================================================
+# GAME-PORT L4 EXPOSURE — why the shared HTTPS Gateway is NOT enough (same shape as apps/vms/unifi).
+# ============================================================================================
+# The Minecraft servers Crafty creates listen on RAW TCP (Java) and UDP (Bedrock) game ports inside
+# the crafty pod's network namespace. The Gateway API `shared` Gateway is an L7 HTTPS terminator and
+# CANNOT express raw TCP on arbitrary ports or UDP at all. So the game ports are exposed as
+# type=LoadBalancer Services whose VIP is pulled from the CiliumLoadBalancerIPPool `default-pool`
+# (192.168.1.53-.60) and ARP-announced on the flat LAN by the existing CiliumL2AnnouncementPolicy
+# `default-l2` (platform-config/cilium/*) — the SAME machinery the Gateway VIP and unifi already use.
+# No new Cilium config is needed; these Services just consume the pool.
+#
+# THIS PATH IS NOT AUTH-GATED — and that is correct: Minecraft's Java (Mojang/Microsoft auth) and
+# Bedrock (Xbox Live) protocols authenticate the PLAYER themselves. Only the Crafty WEB UI (admin
+# control plane) is Keycloak-gated (via the oauth2-proxy sidecar); the game ports carry player traffic
+# and must reach the server subprocesses directly.
+#
+# VIP: 192.168.1.55 — the next free address in default-pool (.53 = shared Gateway, .54 = unifi;
+# verified free live 2026-08-28). PINNED via lbipam.cilium.io/ips so players have a stable target.
+# ONE VIP, TWO SERVICES: a LoadBalancer Service is single-protocol-friendly, so TCP and UDP split into
+# two Services sharing the IP via lbipam.cilium.io/sharing-key (same pattern as unifi).
+#
+# STARTER PORT SET (the operator has not created any servers yet): TCP 25565-25567 (three Java slots)
+# + UDP 19132-19134 (three Bedrock slots). To add capacity: create the server in Crafty bound to a NEW
+# port, then add that port to the matching Service below (and a containerPort in deployment.yaml for
+# documentation) and to the game-port ingress CiliumNetworkPolicy (networkpolicy.yaml). K8s Services
+# enumerate ports individually (no ranges), so add one `{ name, port, targetPort, protocol }` line each.
+#
+# externalTrafficPolicy: Local preserves the LAN client source IP and pins the VIP to the node running
+# the crafty pod (also what L2 announcement does). external-dns (source: service) publishes the A
+# record from the -tcp Service.
+apiVersion: v1
+kind: Service
+metadata:
+  name: crafty-l4-tcp
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+  annotations:
+    lbipam.cilium.io/sharing-key: "crafty"
+    lbipam.cilium.io/ips: "192.168.1.55"
+    # external-dns publishes A crafty-mc.home.keithjackson.dev -> 192.168.1.55 (the Java server target).
+    external-dns.alpha.kubernetes.io/hostname: "crafty-mc.home.keithjackson.dev"
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: crafty
+  ports:
+    - { name: mc-java-0, port: 25565, targetPort: 25565, protocol: TCP }
+    - { name: mc-java-1, port: 25566, targetPort: 25566, protocol: TCP }
+    - { name: mc-java-2, port: 25567, targetPort: 25567, protocol: TCP }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: crafty-l4-udp
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+  annotations:
+    lbipam.cilium.io/sharing-key: "crafty"    # same key + same IP => shares the VIP with -tcp
+    lbipam.cilium.io/ips: "192.168.1.55"
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: crafty
+  ports:
+    - { name: mc-bedr-0, port: 19132, targetPort: 19132, protocol: UDP }
+    - { name: mc-bedr-1, port: 19133, targetPort: 19133, protocol: UDP }
+    - { name: mc-bedr-2, port: 19134, targetPort: 19134, protocol: UDP }

platform/crafty/service.yaml

@@ -0,0 +1,23 @@
+---
+# ClusterIP fronting the oauth2-proxy SIDECAR (NOT Crafty directly). The shared Gateway routes
+# crafty.home.keithjackson.dev to this Service (httproute.yaml); Service :80 -> sidecar :4180, which
+# authenticates against Keycloak and only then proxies to the co-located Crafty UI (127.0.0.1:8443).
+# There is deliberately NO Service targeting Crafty's :8443 — the UI is reachable ONLY through the
+# gate. (Game traffic reaches the MC server subprocesses via the separate L4 LoadBalancer, which is
+# NOT auth-gated because game protocols authenticate themselves — service-l4.yaml.)
+apiVersion: v1
+kind: Service
+metadata:
+  name: crafty
+  namespace: crafty
+  labels:
+    app.kubernetes.io/name: crafty
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: crafty
+  ports:
+    - name: http
+      port: 80
+      targetPort: http   # oauth2-proxy sidecar :4180
+      protocol: TCP

platform/gateway/gateway.yaml

@@ -37,6 +37,37 @@ spec:
           selector:
             matchLabels:
               gateway-access: "shared"
+    # APEX listeners. The wildcard listeners above match ONLY subdomains: `*.home.keithjackson.dev`
+    # does NOT intersect the bare apex `home.keithjackson.dev`, so an HTTPRoute for the apex host
+    # (the Homepage landing dashboard, clusters/base/homepage.yaml) would bind to NO listener and
+    # never serve. These two listeners give the apex its own hostname. TLS is the SAME wildcard cert:
+    # `home.keithjackson.dev` is already a SAN on wildcard-certificate.yaml, so `wildcard-home-tls`
+    # terminates the apex too — no extra Certificate needed. Same gateway-access=shared opt-in.
+    - name: https-apex
+      protocol: HTTPS
+      port: 443
+      hostname: "home.keithjackson.dev"
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: wildcard-home-tls      # apex is a SAN on the wildcard cert (wildcard-certificate.yaml)
+      allowedRoutes:
+        namespaces:
+          from: Selector
+          selector:
+            matchLabels:
+              gateway-access: "shared"
+    - name: http-apex-redirect
+      protocol: HTTP
+      port: 80
+      hostname: "home.keithjackson.dev"
+      allowedRoutes:
+        namespaces:
+          from: Selector
+          selector:
+            matchLabels:
+              gateway-access: "shared"
 # TODO(WS-NET, Phase 2): pin the L2-announced VIP / IP pool; add HTTP->HTTPS redirect HTTPRoute;
 #   confirm cross-namespace ReferenceGrant pattern with WS-APPS. Ingress opens NORTH-SOUTH only —
 #   it does not relax east-west default-deny (docs/09).

platform/headlamp/README.md

@@ -0,0 +1,63 @@
+# Headlamp — Kubernetes dashboard (Keycloak OIDC + per-user RBAC)
+
+Headlamp is served at `https://headlamp.home.keithjackson.dev` behind the shared Gateway
+(wildcard TLS), with **native Keycloak OIDC login** and **per-user RBAC** (ADR-0014, docs/15).
+
+## How per-user RBAC works (end to end)
+
+1. A user logs in at Keycloak (`homelab` realm, `headlamp` confidential client, auth-code flow).
+2. Headlamp's backend keeps the user's **`id_token`** and forwards it as the **bearer** on every
+   request it proxies to the k3s API server (this is Headlamp's default OIDC behaviour).
+3. The apiserver validates that token — it is configured (`infra/ansible` `roles/k3s_server`) with
+   `oidc-issuer-url` = this realm, `oidc-client-id=kubernetes`, `oidc-username-claim=email`,
+   `oidc-groups-claim=groups`, prefixes `oidc:`.
+4. RBAC is applied **per user** via the existing group bindings in
+   [`../keycloak/k8s-oidc-rbac.yaml`](../keycloak/k8s-oidc-rbac.yaml):
+   `oidc:platform-admins` → `cluster-admin`, `oidc:viewers` → `view`.
+
+So each user gets exactly their own rights — **not** a shared Headlamp identity. That is why the
+Headlamp ServiceAccount is **not** bound to cluster-admin: the chart default
+`clusterRoleBinding.create: true` (→ `cluster-admin`) is overridden to **`false`** in
+`helmrelease.yaml`. User requests ride the forwarded user token; the SA needs no cluster rights.
+
+## The audience nuance (why the client has an audience mapper)
+
+The apiserver only accepts ID tokens whose `aud` **contains `kubernetes`** (`oidc-client-id=
+kubernetes`). A plain `headlamp` client issues `aud=[headlamp]`, which the apiserver would reject.
+The `headlamp` client therefore carries an **`oidc-audience-mapper`** (see
+`../keycloak/keycloak-realm-import.yaml`) with `included.client.audience: kubernetes`, so tokens
+carry `aud=[headlamp, kubernetes]`:
+
+- Headlamp's own verifier accepts it (aud contains `headlamp`), and
+- the apiserver accepts it (aud contains `kubernetes`).
+
+This keeps the apiserver OIDC config untouched (preferred over widening `--api-audiences`).
+
+## Secrets
+
+- The confidential client secret lives SOPS-encrypted in
+  [`../keycloak/keycloak-realm-clients.sops.yaml`](../keycloak/keycloak-realm-clients.sops.yaml)
+  (`headlamp-client-secret`, injected into the realm import via the `HEADLAMP_CLIENT_SECRET`
+  placeholder) and mirrored in this namespace's `headlamp-oidc-credentials.sops.yaml`
+  (`OIDC_CLIENT_SECRET`), which the chart `config.oidc.externalSecret` envFrom's. Same cross-namespace
+  mirror pattern as grafana/gitea.
+- **Key names are load-bearing.** The chart `envFrom`s this Secret verbatim, so its keys become the
+  container env vars referenced by the chart's OIDC args (`-oidc-idp-issuer-url=$(OIDC_ISSUER_URL)`,
+  etc.). They must be exactly `OIDC_CLIENT_ID`, `OIDC_CLIENT_SECRET`, `OIDC_ISSUER_URL`, `OIDC_SCOPES`
+  (and `OIDC_CALLBACK_URL`) — **not** camelCase. A misnamed key leaves `$(OIDC_ISSUER_URL)`
+  unexpanded and OIDC login fails with `unsupported protocol scheme ""`.
+
+## Break-glass
+
+The cert-based admin in the bootstrap kubeconfig is unaffected by any of this and retains full
+access if Keycloak/OIDC is down — the standard break-glass path.
+
+## Files
+
+| File | Tier | Purpose |
+|---|---|---|
+| `namespace.yaml` | platform | `headlamp` namespace (`gateway-access=shared`; platform, not app-namespace) |
+| `helmrepository.yaml` / `helmrelease.yaml` | platform | Official `kubernetes-sigs/headlamp` chart + OIDC/RBAC values |
+| `headlamp-oidc-credentials.sops.yaml` | platform | SOPS OIDC config the chart envFrom's (decrypted by the `platform` Flux Kustomization) |
+| `networkpolicy.yaml` | platform | default-deny baseline + Cilium ingress(gateway)/egress(apiserver+Keycloak) allows |
+| `httproute.yaml` | platform-config | Gateway API HTTPRoute for `headlamp.home.keithjackson.dev` (needs Gateway CRDs) |

platform/headlamp/headlamp-oidc-credentials.sops.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: headlamp-oidc
+    namespace: headlamp
+type: Opaque
+stringData:
+    OIDC_CLIENT_ID: ENC[AES256_GCM,data:dXAzoFfar4A=,iv:dHOJFcS6tpvtJes1c++nU6BbHyRlmGuLpncucbkJ4xA=,tag:lFI+KJPid2f/2r/ZjYlFTg==,type:str]
+    OIDC_ISSUER_URL: ENC[AES256_GCM,data:Er1ljhH62WahtEJkNvxk1BA2jz28OOCZmmyjkbVN82e/TFM7O+FB/FPjWUaW3lhd+dK16ro=,iv:Z3aP7+T+ytItYpbxjoYymLNGa7m8KGic4WqIQrrBSGE=,tag:/Rvhp5+r7AXIj2uDgL/dPg==,type:str]
+    OIDC_SCOPES: ENC[AES256_GCM,data:WoGaQKY9BFsi5/kDlRMXatINtmE=,iv:kTN+C176I8o5ftzZiKR10s9gl9VvBsOCJhFIZzj9gm4=,tag:ZyRIykD1h5YBq4grd6FB/w==,type:str]
+    OIDC_CALLBACK_URL: ENC[AES256_GCM,data:SZdpxEuW1BJccLyRGjuIDWBu6KpySARv/6M0mWtzc9jS+v64Lgr4VWKwgW0tdZrH5jM9rw==,iv:UbLwgfz7J0SIWbcQ4csqxv5B3enTmfHJzSDw/lmqAEI=,tag:lge2AuoQXK/u3uZbWvGnLQ==,type:str]
+    OIDC_CLIENT_SECRET: ENC[AES256_GCM,data:r0pb73gat8E7BFLPbdhguLh8G5PUT7PlJJeIcv7QPdMnN4iIB8MedMwxd77DkbxHSR7bav2ohQczjnbJBA4tPA==,iv:O0RbMFz4UD1sy8mbY7SXWCSGNF+lapDTL7tpphEcx7c=,tag:bEyuEs/lsK28fJeYDpYiOg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1UDV1RzRWeGZvdWxFQlgz
+            dCtjTU8wZzR5Tks5dmYreFliREZGaDdXMVVjCkVPY2lZby9LNzJMelJkTlE1Yks2
+            cXN5YmhlaUtwN3ZjQUo5R0xnQWNQYm8KLS0tIDBMSUMxZ1pHaDRnUWs5RGtRNlhI
+            N0I4ekhqMWVWUFZoeGU4SERpYXVDQUkKUUwY6ZxEiWGdqV8s3wbnbPF9SkZJoxfy
+            3MkNQ7rZHSILFTPVaZSErNTI21vqQr3lG24+yKLfNhAZBNyB5lvedQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-29T20:04:33Z"
+    mac: ENC[AES256_GCM,data:Haj8wMqDuIdtBopP02L1Z63+T8f0fcuSDpF5jnjgCB1+uzX+d4ylqpcVP5r8Ug8M/2ay2nYWoJBWzL0Y1iHcnDP7/T60M8hT46cBYPrDh3MqY0rWF5GwpjJxL9tStoSlDTYICeilAPqo2Vw4UM1CqnZv+th40rz5/R4MwRsU04w=,iv:5HPvtnF6nnDYDaT84r1JWf+ZH1X5/L1Aiim2lgOucH4=,tag:VM/2FXNW7sdjCk/k2AVl+g==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4

platform/headlamp/headlamp-oidc-credentials.sops.yaml.example

@@ -0,0 +1,45 @@
+# =============================================================================
+# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize and never
+# auto-encrypted by SOPS). Reference scaffold + OPERATOR HAND-OFF.
+#
+# WHAT THIS IS: the OIDC config Headlamp uses to authenticate against the Keycloak `homelab` realm's
+# `headlamp` client. The Headlamp chart's `config.oidc.externalSecret` (helmrelease.yaml) envFrom's
+# THIS Secret, so ALL OIDC env (OIDC_CLIENT_ID/OIDC_CLIENT_SECRET/OIDC_ISSUER_URL/OIDC_SCOPES/OIDC_CALLBACK_URL) live here,
+# SOPS-encrypted, referenced by name only (R-IDP-04). The chart `envFrom`s this Secret VERBATIM, so
+# the KEYS ARE the container env var names — they MUST be exactly OIDC_CLIENT_ID / OIDC_CLIENT_SECRET
+# / OIDC_ISSUER_URL / OIDC_SCOPES / OIDC_CALLBACK_URL (the chart's `-oidc-*` args reference those
+# names via $(...)). There is NO camelCase→OIDC_ mapping; a mismatched key breaks OIDC login.
+#
+# The `clientSecret` value MUST equal the `headlamp-client-secret` placed in
+# platform/keycloak/keycloak-realm-clients.sops.yaml (which the realm import injects into the
+# Keycloak `headlamp` client) — same cross-namespace mirror pattern as grafana/gitea (a Secret
+# cannot be shared across namespaces, so the one value is stored in two encrypted files).
+#
+# HAND-OFF STEPS (operator):
+#  (a) Use the SAME headlamp-client-secret generated for keycloak-realm-clients (openssl rand -hex 32).
+#  (b) Copy to:      platform/headlamp/headlamp-oidc-credentials.sops.yaml
+#  (c) Fill `clientSecret`; encrypt:
+#         sops -e -i platform/headlamp/headlamp-oidc-credentials.sops.yaml
+#  (d) It is already listed under `resources:` in platform/headlamp/kustomization.yaml.
+# =============================================================================
+apiVersion: v1
+kind: Secret
+metadata:
+  name: headlamp-oidc
+  namespace: headlamp
+type: Opaque
+stringData:
+  # KEY NAMES ARE LOAD-BEARING. The chart does NOT rename these — it `envFrom`s this Secret
+  # verbatim, so each key becomes a container env var of the SAME name, and the chart's OIDC args
+  # reference them by exact name: `-oidc-client-id=$(OIDC_CLIENT_ID)`,
+  # `-oidc-client-secret=$(OIDC_CLIENT_SECRET)`, `-oidc-idp-issuer-url=$(OIDC_ISSUER_URL)`,
+  # `-oidc-scopes=$(OIDC_SCOPES)`. If a key is misnamed (e.g. camelCase `issuerURL`), the env var
+  # `OIDC_ISSUER_URL` is undefined, K8s leaves the literal string `$(OIDC_ISSUER_URL)` in the arg,
+  # and Headlamp's OIDC discovery fails with `unsupported protocol scheme ""`. Use these EXACT keys.
+  # Non-secret OIDC config (kept in the encrypted Secret because the chart envFrom's a single Secret).
+  OIDC_CLIENT_ID: headlamp
+  OIDC_ISSUER_URL: https://keycloak.home.keithjackson.dev/realms/homelab
+  OIDC_SCOPES: "openid email profile"
+  OIDC_CALLBACK_URL: https://headlamp.home.keithjackson.dev/oidc-callback
+  # The confidential client secret — MUST match keycloak-realm-clients headlamp-client-secret.
+  OIDC_CLIENT_SECRET: REPLACE-ME

platform/headlamp/helmrelease.yaml

@@ -0,0 +1,97 @@
+---
+# Headlamp — the Kubernetes web dashboard, served at headlamp.home.keithjackson.dev with Keycloak
+# native-OIDC login and PER-USER RBAC (ADR-0014, docs/15). Official kubernetes-sigs/headlamp Helm
+# chart via a Flux HelmRelease (same pattern as platform/monitoring's kube-prometheus-stack).
+#
+# ── PER-USER RBAC (the whole point) ────────────────────────────────────────────────────────────
+# Headlamp is configured for OIDC, so its backend forwards the LOGGED-IN USER's OIDC id_token as the
+# bearer to the k3s API server. The apiserver validates that token (oidc-issuer-url = this realm,
+# oidc-client-id=kubernetes, oidc-username-claim=email, oidc-groups-claim=groups; see
+# infra/ansible roles/k3s_server) and RBAC is applied per user via the EXISTING group bindings
+# (platform/keycloak/k8s-oidc-rbac.yaml: oidc:platform-admins -> cluster-admin, oidc:viewers -> view).
+# So every user gets exactly their own rights — NOT a shared identity.
+#
+#   clusterRoleBinding.create=false  ← DELIBERATE. The default chart binds Headlamp's ServiceAccount
+#   to cluster-admin; that would let the backend act as cluster-admin. We do NOT want that: user
+#   requests ride the user's forwarded id_token, so the SA needs no cluster rights. Removing the
+#   binding means an unauthenticated backend can do nothing — the secure posture for per-user RBAC.
+#
+# ── AUDIENCE NUANCE ─────────────────────────────────────────────────────────────────────────────
+# The apiserver only accepts ID tokens whose `aud` CONTAINS `kubernetes` (oidc-client-id=kubernetes).
+# A `headlamp` client would issue aud=[headlamp]. The Keycloak `headlamp` client therefore carries an
+# AUDIENCE protocol mapper that ADDS `kubernetes` to the token audience (keycloak-realm-import.yaml),
+# giving aud=[headlamp, kubernetes]: Headlamp's own verifier accepts it (aud contains headlamp) AND
+# the apiserver accepts it (aud contains kubernetes). This keeps the apiserver config untouched.
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: headlamp
+  namespace: headlamp
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: headlamp
+      # 0.41.x, NOT 0.40.x. Chart 0.40.1 shipped the `-session-ttl=86400` container arg BEFORE the
+      # backend flag existed: its image (appVersion v0.40.1) has no `session-ttl` flag, so the pod
+      # crashlooped with `flag provided but not defined: -session-ttl` (upstream issue #4883). The
+      # backend gained `f.Int("session-ttl", …)` in v0.41.0, and chart 0.41.0 pins that same image
+      # (appVersion == chart version), so chart+image realign: every arg the chart renders for our
+      # values (-session-ttl + the -oidc-* args from the externalSecret; useCookie stays default-false
+      # so `-oidc-use-cookie` is NOT emitted) is supported by the v0.41.0 backend. Renovate maintains
+      # within 0.41.x; the appVersion continues to pin the Headlamp image.
+      version: ">=0.41.0 <0.42.0"
+      sourceRef:
+        kind: HelmRepository
+        name: headlamp
+        namespace: headlamp
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  values:
+    replicaCount: 1
+    # In-cluster mode: Headlamp discovers the apiserver from the pod's in-cluster config, then
+    # OVERRIDES the bearer with the user's OIDC id_token for authenticated requests.
+    config:
+      inCluster: true
+      oidc:
+        # OIDC config (OIDC_CLIENT_ID / OIDC_CLIENT_SECRET / OIDC_ISSUER_URL / OIDC_SCOPES /
+        # OIDC_CALLBACK_URL) is supplied via an EXISTING SOPS-encrypted Secret consumed by name only
+        # (R-IDP-04) — the chart envFrom's it. The Secret keys ARE the env var names (no renaming),
+        # and the chart's `-oidc-*` args reference them via $(...), e.g.
+        # `-oidc-idp-issuer-url=$(OIDC_ISSUER_URL)`; a camelCase key would leave that arg unexpanded
+        # and break OIDC login (`unsupported protocol scheme ""`). Nothing sensitive is rendered here.
+        externalSecret:
+          enabled: true
+          name: headlamp-oidc
+          hasScopes: true
+    # PER-USER RBAC: do NOT grant Headlamp's ServiceAccount cluster-admin. User requests are
+    # authorized by their forwarded OIDC token against the existing oidc:* group bindings.
+    serviceAccount:
+      create: true
+    clusterRoleBinding:
+      create: false
+    # Hardened pod defaults from the chart (runAsNonRoot, uid 100). Kept explicit for clarity.
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 100
+      runAsGroup: 101
+      privileged: false
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+          - ALL
+      # NOTE: readOnlyRootFilesystem intentionally left unset (chart default). Enabling it needs a
+      # writable /tmp (+ plugins dir) emptyDir; deferred as a runtime-verified follow-up hardening.
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        memory: 256Mi
+    service:
+      type: ClusterIP
+      port: 80

platform/headlamp/helmrepository.yaml

@@ -0,0 +1,11 @@
+---
+# HelmRepository for the official Headlamp chart (kubernetes-sigs/headlamp). Renovate maintains the
+# pinned chart version in helmrelease.yaml. Same Flux source pattern as platform/monitoring.
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: headlamp
+  namespace: headlamp
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/headlamp/

platform/headlamp/httproute.yaml

@@ -0,0 +1,35 @@
+---
+# North-south ingress for Headlamp at headlamp.home.keithjackson.dev. Attaches to the shared Gateway
+# (docs/09); TLS terminated at the Gateway by the *.home.keithjackson.dev wildcard cert (already
+# covers this host). Same-namespace backend, so no ReferenceGrant is needed; the parentRef is
+# permitted because the headlamp namespace carries gateway-access=shared. CR of the Gateway API CRDs
+# — CARRIED BY platform-config (dependsOn platform) so the CRDs exist first (like grafana-httproute).
+#
+# auth-mode annotation records that Headlamp authenticates via native Keycloak OIDC (R-IDP-02); this
+# route is NOT protected by oauth2-proxy forward-auth (Headlamp runs its own OIDC login + per-user
+# token forwarding), so it must NOT carry an oauth2-proxy ExtAuth filter.
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: headlamp
+  namespace: headlamp
+  labels:
+    app.kubernetes.io/name: headlamp
+  annotations:
+    homelab.keithjackson.dev/auth-mode: oidc
+    homelab.keithjackson.dev/auth-groups: platform-admins
+spec:
+  parentRefs:
+    - name: shared
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - headlamp.home.keithjackson.dev
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: headlamp
+          port: 80

platform/headlamp/kustomization.yaml

@@ -0,0 +1,24 @@
+---
+# Headlamp — Kubernetes dashboard with Keycloak native-OIDC login + per-user RBAC (ADR-0014).
+# Rendered by its OWN post-platform Flux Kustomization (clusters/base/headlamp.yaml, dependsOn
+# platform-config, wait:false) — NOT by the gating `platform` tier, where a crashlooping UI
+# HelmRelease stalled the whole chain (DECISIONS-LOG "Headlamp gating stall"). This component owns
+# the HelmRepository + HelmRelease (chart installs the Deployment/Service/SA), the SOPS-encrypted
+# OIDC credential (decrypted per-env by the headlamp Kustomization), the default-deny + Cilium
+# policies, AND the HTTPRoute. The httproute (a Gateway API CR) now travels WITH the component here
+# rather than being referenced in-place from platform-config: the CRDs it needs
+# (gateway.networking.k8s.io) are registered by platform, and platform is Ready before this tier's
+# platform-config dependency completes, so ordering holds — same move oauth2-proxy already made.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: headlamp
+resources:
+  - namespace.yaml
+  - helmrepository.yaml
+  - helmrelease.yaml
+  # OIDC client credential (envFrom'd by the chart via config.oidc.externalSecret) — SOPS-encrypted,
+  # referenced by name (R-IDP-04). The matching value is mirrored in keycloak-realm-clients.
+  - headlamp-oidc-credentials.sops.yaml
+  # default-deny baseline + Cilium ingress(gateway)/egress(apiserver+Keycloak) allows.
+  - networkpolicy.yaml
+  - httproute.yaml       # dashboard ingress (was referenced in-place by platform-config)

platform/headlamp/namespace.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: headlamp
+  labels:
+    # Expose Headlamp at headlamp.home.keithjackson.dev via the shared Gateway (docs/09). The
+    # *.home.keithjackson.dev wildcard cert already covers this host (httproute.yaml).
+    gateway-access: "shared"
+    # NOTE: intentionally NOT homelab.keithjackson.dev/app-namespace=true. Headlamp is a PLATFORM
+    # admin tool (like Grafana in `monitoring`), so Kyverno does not generate the app default-deny
+    # baseline here and the app-scoped auth-mode / egress-class / host-namespace policies do not
+    # apply. We still ship an explicit default-deny NetworkPolicy + CiliumNetworkPolicies
+    # (networkpolicy.yaml) as defense-in-depth: this pod holds the kube-apiserver egress path, so it
+    # is locked down to exactly DNS + apiserver + Keycloak (via the Gateway VIP) egress and
+    # Gateway-only ingress.

platform/headlamp/networkpolicy.yaml

@@ -0,0 +1,73 @@
+---
+# Default-deny baseline (R-NET-01): deny all ingress + egress, allow only DNS. `headlamp` is a
+# PLATFORM namespace so Kyverno does not auto-generate this — we ship it explicitly (defense in
+# depth; this pod holds the kube-apiserver egress path, so it is worth locking down). Same shape as
+# the Kyverno-generated app baseline. Cilium enforces the UNION of this NetworkPolicy and the
+# CiliumNetworkPolicies below (allow if ANY policy allows), so these add the apiserver/Keycloak/
+# Gateway paths on top of the deny-all floor.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-deny
+  namespace: headlamp
+  labels:
+    homelab.keithjackson.dev/baseline: default-deny
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+---
+# Gateway ingress allow (R-NET-01/02): only the shared Gateway (the Cilium `ingress` entity) may
+# reach Headlamp, on its container port 4466 (Service headlamp:80 -> targetPort 4466).
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: headlamp-allow-gateway-ingress
+  namespace: headlamp
+  labels:
+    app.kubernetes.io/name: headlamp
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: headlamp
+  ingress:
+    - fromEntities:
+        - ingress
+      toPorts:
+        - ports:
+            - port: "4466"
+              protocol: TCP
+---
+# Egress allow: Headlamp's backend must reach (a) the kube-apiserver — it forwards each user's OIDC
+# id_token there for per-user RBAC — and (b) Keycloak for the OIDC auth-code/token/JWKS exchange.
+# Keycloak is reached at keycloak.home.keithjackson.dev, which coredns split-horizon resolves to the
+# Gateway VIP (an L2-announced LoadBalancer IP that Cilium classifies as `world`), so `world` egress
+# covers the Keycloak path (same as the gitea OIDC bootstrap). DNS is already allowed by the
+# default-deny baseline above.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: headlamp-allow-egress
+  namespace: headlamp
+  labels:
+    app.kubernetes.io/name: headlamp
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: headlamp
+  egress:
+    - toEntities:
+        - kube-apiserver
+        - world

platform/homepage/README.md

@@ -0,0 +1,55 @@
+# platform/homepage
+
+**Homepage** (gethomepage) — the apex landing dashboard at **https://home.keithjackson.dev**,
+aggregating links to the cluster's services.
+
+- **Serves:** a single pane of glass for the homelab (docs/09 apex UI).
+- **Tier:** non-gating post-platform Flux Kustomization
+  ([`clusters/base/homepage.yaml`](../../clusters/base/homepage.yaml), `dependsOn: platform-config`,
+  `wait: false`) — mirrors `headlamp` / `oauth2-proxy`. A leaf UI must never gate the `platform`
+  tier (DECISIONS-LOG "Headlamp gating stall").
+- **Namespace:** `homepage` — a PLATFORM namespace (no `app-namespace=true`), so the Kyverno
+  app-scoped policies (default-deny gen, auth-mode, egress-class) do not apply; we ship an explicit
+  default-deny + Cilium ingress policy as defense-in-depth.
+
+## What this component owns
+
+Plain manifests (not a HelmRelease — the app is one stateless Deployment + a config ConfigMap):
+
+| File | Purpose |
+|---|---|
+| `namespace.yaml` | `homepage` ns, `gateway-access=shared` |
+| `configmap.yaml` | Homepage config (settings/services/bookmarks/widgets + empty docker/kubernetes + custom.css/js) |
+| `deployment.yaml` | `ghcr.io/gethomepage/homepage` (non-root, subPath config mounts, `HOMEPAGE_ALLOWED_HOSTS`) |
+| `service.yaml` | ClusterIP 80 -> 3000 |
+| `networkpolicy.yaml` | default-deny (DNS-only egress) + Cilium Gateway-only ingress on 3000 |
+| `httproute.yaml` | apex ingress on the Gateway `https-apex` listener |
+
+## Apex nuances (why this isn't just another `*.home` route)
+
+1. **TLS:** the apex `home.keithjackson.dev` is already a **SAN** on the wildcard Certificate
+   (`platform/gateway/wildcard-certificate.yaml`) — no new cert.
+2. **Gateway listener:** the wildcard listeners' hostname `*.home.keithjackson.dev` does **not**
+   match the bare apex. This component's route binds a dedicated **`https-apex`** listener added to
+   the shared Gateway (`platform/gateway/gateway.yaml`); `http-apex-redirect` covers :80.
+3. **`HOMEPAGE_ALLOWED_HOSTS`:** Homepage >= 1.0 returns 403 for any Host not in this list — set to
+   `home.keithjackson.dev` + the pod IP (for kubelet probes).
+
+## Forward-auth — FOLLOW-UP (not yet enforced)
+
+The HTTPRoute is annotated `auth-mode: forward-auth` (intended posture: oauth2-proxy -> Keycloak).
+**The Gateway ext-authz attachment that actually enforces forward-auth does not exist yet** — it is a
+Phase-6 WS-NET TODO (`platform/oauth2-proxy` is the authenticator; there is no `CiliumEnvoyConfig` /
+ext-authz wiring anywhere in the repo). Until that mechanism lands, Homepage — like **every** other
+`forward-auth` app (jackett, qbittorrent, pihole, radarr, sonarr) — is reachable unauthenticated.
+When the Gateway ext-authz pattern is built, attach it to this route (and the others) with
+`allowed-groups: platform-admins`. **PM decision flagged** in the deploy report.
+
+## Deliberately deferred (avoid scope creep)
+
+- **Service-status widgets** (Grafana/Plex/Sonarr/… API tiles) need per-service API-key **secrets** —
+  not wired; tiles are simple links. Add as SOPS secrets + `widgets.yaml` later.
+- **Kubernetes widget** would need a read-only ServiceAccount + RBAC — skipped (`kubernetes.yaml`
+  empty, `automountServiceAccountToken: false`).
+- **Brand icons** (selfh.st/dashboard-icons) need world:443 egress — we use bundled `mdi-*` icons so
+  egress stays DNS-only.

platform/homepage/configmap.yaml

@@ -0,0 +1,143 @@
+---
+# Homepage configuration (gethomepage). Each key is mounted read-only into /app/config via a subPath
+# (deployment.yaml) — the exact file set the app expects (services/settings/bookmarks/widgets +
+# docker/kubernetes/proxmox + custom.css/js). Plain link tiles only: NO service-widget API keys are wired
+# here (that needs per-service secrets — flagged as a follow-up), so nothing sensitive lives in this
+# ConfigMap. Icons use the app-BUNDLED Material Design set (`mdi-*`) so no icon CDN is fetched and the
+# pod's egress stays DNS-only (networkpolicy.yaml). Brand icons (selfh.st/dashboard-icons) would need
+# world:443 egress — deliberately not used.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: homepage
+  namespace: homepage
+  labels:
+    app.kubernetes.io/name: homepage
+data:
+  settings.yaml: |
+    ---
+    title: Home Network
+    headerStyle: clean
+    theme: dark
+    color: slate
+    hideVersion: true
+    layout:
+      Platform:
+        style: row
+        columns: 3
+      Media:
+        style: row
+        columns: 3
+      Network & Infra:
+        style: row
+        columns: 3
+
+  services.yaml: |
+    ---
+    - Platform:
+        - Headlamp:
+            href: https://headlamp.home.keithjackson.dev
+            description: Kubernetes dashboard (per-user OIDC RBAC)
+            icon: mdi-kubernetes
+        - Grafana:
+            href: https://grafana.home.keithjackson.dev
+            description: Metrics, logs & dashboards (fronts Prometheus/Loki/Alertmanager)
+            icon: mdi-chart-areaspline
+        - Semaphore:
+            href: https://semaphore.home.keithjackson.dev
+            description: Ansible automation & CI
+            icon: mdi-cog-play
+        - Gitea:
+            href: https://gitea.home.keithjackson.dev
+            description: Git hosting & Actions (GitOps source of truth)
+            icon: mdi-git
+        - Keycloak:
+            href: https://keycloak.home.keithjackson.dev
+            description: Identity provider (SSO / OIDC)
+            icon: mdi-account-key
+
+    - Media:
+        - Plex:
+            href: https://plex.home.keithjackson.dev
+            description: Media streaming
+            icon: mdi-plex
+        - Seerr:
+            href: https://seerr.home.keithjackson.dev
+            description: Media requests
+            icon: mdi-play-box-multiple
+        - Sonarr:
+            href: https://sonarr.home.keithjackson.dev
+            description: TV library manager
+            icon: mdi-television-classic
+        - Radarr:
+            href: https://radarr.home.keithjackson.dev
+            description: Movie library manager
+            icon: mdi-movie-open
+        - Jackett:
+            href: https://jackett.home.keithjackson.dev
+            description: Indexer proxy
+            icon: mdi-magnify
+        - qBittorrent:
+            href: https://qbittorrent.home.keithjackson.dev
+            description: Download client (VPN egress)
+            icon: mdi-download-network
+
+    - Network & Infra:
+        - Pi-hole:
+            href: https://pihole.home.keithjackson.dev
+            description: Network-wide DNS & ad-blocking
+            icon: mdi-shield-check
+        - UniFi:
+            href: https://unifi.home.keithjackson.dev:11443
+            description: Network controller (UOS)
+            icon: mdi-router-network
+        - Nextcloud:
+            href: https://nextcloud.home.keithjackson.dev
+            description: Files, calendar & contacts
+            icon: mdi-cloud
+
+  bookmarks.yaml: |
+    ---
+    - Operations:
+        - Gitea (external):
+            - abbr: GT
+              href: https://gitea.keithjackson.dev
+        - Auth portal:
+            - abbr: SSO
+              href: https://auth.home.keithjackson.dev
+    - Reference:
+        - Homepage docs:
+            - abbr: HP
+              href: https://gethomepage.dev
+
+  widgets.yaml: |
+    ---
+    - datetime:
+        text_size: xl
+        format:
+          dateStyle: long
+          timeStyle: short
+          hour12: false
+    - search:
+        provider: duckduckgo
+        target: _blank
+
+  # No Docker integration (Homepage does not reach the Docker socket in-cluster).
+  docker.yaml: |
+    ---
+
+  # No Kubernetes integration (no SA / cluster RBAC granted — avoids scope creep; the kubernetes
+  # widget would need a read-only ServiceAccount. Left empty deliberately).
+  kubernetes.yaml: |
+    ---
+
+  # No Proxmox integration. Homepage v2.1.2 treats proxmox.yaml as a REQUIRED config file (it is in
+  # the image skeleton dir /app/src/skeleton) and copies a default skeleton into /app/config when it
+  # is missing — which EACCES-crashes the pod because the config dir is a read-only subPath mount.
+  # Provide it here (empty) so Homepage finds it and never writes at runtime. A live PVE widget would
+  # need pve url/token/secret in a Secret — deliberately not wired.
+  proxmox.yaml: |
+    ---
+
+  custom.css: ""
+  custom.js: ""

platform/homepage/deployment.yaml

@@ -0,0 +1,104 @@
+---
+# Homepage (gethomepage) — the apex landing dashboard at https://home.keithjackson.dev. Plain
+# manifests (not a HelmRelease): the whole app is a single stateless Deployment + a config ConfigMap,
+# and hand-authoring the ConfigMap gives exact control over the service links without guessing a
+# chart's values schema (same plain-manifest style as apps/vms/unifi). Rendered by its OWN non-gating
+# post-platform Flux Kustomization (clusters/base/homepage.yaml, dependsOn platform-config,
+# wait:false) — a leaf UI must never sit in the gating `platform` tier (DECISIONS-LOG: "Headlamp
+# gating stall").
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: homepage
+  namespace: homepage
+  labels:
+    app.kubernetes.io/name: homepage
+spec:
+  replicas: 1
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: homepage
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: homepage
+      annotations:
+        # Re-roll the pod when the config changes (ConfigMap subPath mounts do NOT hot-reload).
+        checksum/config: config-v2
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: homepage
+          # Renovate maintains this pin (ghcr.io/gethomepage/homepage). Config lives in /app/config.
+          image: ghcr.io/gethomepage/homepage:v2.1.2
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 3000
+              protocol: TCP
+          env:
+            # Referenced by HOMEPAGE_ALLOWED_HOSTS below so the in-pod liveness/readiness probes
+            # (which hit the pod IP, not the public host) are accepted by Homepage's host guard.
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # REQUIRED by Homepage >= 1.0 — requests whose Host header is not in this list get 403.
+            # The public apex host (served via the Gateway) + the pod IP (for kubelet probes).
+            - name: HOMEPAGE_ALLOWED_HOSTS
+              value: "home.keithjackson.dev,$(MY_POD_IP):3000"
+          resources:
+            requests:
+              cpu: 25m
+              memory: 128Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            # NOTE: readOnlyRootFilesystem intentionally left unset — Homepage (Next.js) writes a
+            # runtime build/cache under /app/.next at boot. Same deferred-hardening note as headlamp.
+          livenessProbe:
+            httpGet:
+              path: /api/healthcheck
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 15
+          readinessProbe:
+            httpGet:
+              path: /api/healthcheck
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 15
+          volumeMounts:
+            # Homepage expects each config file present; mount them read-only by subPath so the rest
+            # of /app/config stays writable (it writes ./logs there at runtime).
+            - { name: homepage-config, mountPath: /app/config/settings.yaml, subPath: settings.yaml, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/services.yaml, subPath: services.yaml, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/bookmarks.yaml, subPath: bookmarks.yaml, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/widgets.yaml, subPath: widgets.yaml, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/docker.yaml, subPath: docker.yaml, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/kubernetes.yaml, subPath: kubernetes.yaml, readOnly: true }
+            # proxmox.yaml is ALSO a required file in Homepage v2.1.2's skeleton — provide it so the
+            # app never copies a skeleton into the read-only /app/config (EACCES crash otherwise).
+            - { name: homepage-config, mountPath: /app/config/proxmox.yaml, subPath: proxmox.yaml, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/custom.css, subPath: custom.css, readOnly: true }
+            - { name: homepage-config, mountPath: /app/config/custom.js, subPath: custom.js, readOnly: true }
+            # Writable scratch for Homepage's runtime logs (it writes /app/config/logs).
+            - { name: logs, mountPath: /app/config/logs }
+      volumes:
+        - name: homepage-config
+          configMap:
+            name: homepage
+        - name: logs
+          emptyDir: {}

platform/homepage/httproute.yaml

@@ -0,0 +1,39 @@
+---
+# North-south ingress for Homepage at the APEX home.keithjackson.dev. Attaches to the shared Gateway
+# via the dedicated `https-apex` listener (sectionName) — the wildcard `https` listener's hostname
+# `*.home.keithjackson.dev` does NOT match the bare apex, so this route MUST bind the apex listener
+# added in platform/gateway/gateway.yaml. TLS is terminated at the Gateway by the wildcard cert (the
+# apex is a SAN on it). Same-namespace backend, so no ReferenceGrant; the parentRef is permitted
+# because the homepage namespace carries gateway-access=shared.
+#
+# auth-mode=forward-auth records the INTENDED posture: Homepage is a bare UI with no native OIDC, so
+# it should sit behind oauth2-proxy forward-auth at the Gateway (R-IDP-02). NOTE: the Gateway
+# ext-authz ATTACHMENT that actually enforces forward-auth is a Phase-6 WS-NET TODO that is NOT yet
+# built anywhere in the repo (platform/oauth2-proxy is the authenticator only; no CiliumEnvoyConfig /
+# ext-authz exists). Until that mechanism lands, this route — like every other forward-auth app
+# (jackett, qbittorrent, pihole, radarr, …) — is reachable without auth. See README.md "Forward-auth".
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: homepage
+  namespace: homepage
+  labels:
+    app.kubernetes.io/name: homepage
+  annotations:
+    homelab.keithjackson.dev/auth-mode: forward-auth
+    homelab.keithjackson.dev/auth-groups: platform-admins
+spec:
+  parentRefs:
+    - name: shared
+      namespace: gateway
+      sectionName: https-apex     # the apex listener (wildcard `https` won't match the bare apex)
+  hostnames:
+    - home.keithjackson.dev
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: homepage
+          port: 80

platform/homepage/kustomization.yaml

@@ -0,0 +1,18 @@
+---
+# Homepage — the apex landing dashboard (home.keithjackson.dev). Rendered by its OWN non-gating
+# post-platform Flux Kustomization (clusters/base/homepage.yaml, dependsOn platform-config,
+# wait:false) — NOT by the gating `platform` tier, where a leaf UI's unhealth stalled the whole chain
+# (DECISIONS-LOG "Headlamp gating stall"). Plain manifests (no HelmRelease): a single Deployment +
+# a hand-authored config ConfigMap. The HTTPRoute travels WITH the component (the gateway.networking
+# CRDs are registered by platform, Ready before this tier's platform-config dependency completes).
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: homepage
+resources:
+  - namespace.yaml
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml
+  # default-deny baseline + Cilium gateway-ingress allow (DNS-only egress from the baseline).
+  - networkpolicy.yaml
+  - httproute.yaml       # apex ingress (binds the Gateway `https-apex` listener)

platform/homepage/namespace.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: homepage
+  labels:
+    # Expose Homepage at the APEX home.keithjackson.dev via the shared Gateway (docs/09). The apex is
+    # a SAN on the *.home.keithjackson.dev wildcard cert, and the Gateway carries a dedicated
+    # `https-apex` listener for it (platform/gateway/gateway.yaml). This label opts the namespace in
+    # to attach HTTPRoutes to the shared Gateway (httproute.yaml).
+    gateway-access: "shared"
+    # NOTE: intentionally NOT homelab.keithjackson.dev/app-namespace=true. Homepage is a PLATFORM
+    # landing UI (like Grafana in `monitoring` / Headlamp in `headlamp`), so Kyverno does not generate
+    # the app default-deny baseline here and the app-scoped auth-mode / egress-class policies do not
+    # apply. We still ship an explicit default-deny NetworkPolicy + a CiliumNetworkPolicy
+    # (networkpolicy.yaml) as defense-in-depth: Gateway-only ingress, DNS-only egress.

platform/homepage/networkpolicy.yaml

@@ -0,0 +1,51 @@
+---
+# Default-deny baseline (R-NET-01): deny all ingress + egress, allow only DNS. `homepage` is a
+# PLATFORM namespace so Kyverno does not auto-generate this — we ship it explicitly (defense in
+# depth). Same shape as the Kyverno-generated app baseline. Cilium enforces the UNION of this
+# NetworkPolicy and the CiliumNetworkPolicy below (allow if ANY policy allows).
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-deny
+  namespace: homepage
+  labels:
+    homelab.keithjackson.dev/baseline: default-deny
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+---
+# Gateway ingress allow (R-NET-01/02): only the shared Gateway (the Cilium `ingress` entity) may
+# reach Homepage, on its container port 3000. Homepage is a pure link dashboard — it makes NO
+# outbound calls (bundled `mdi-*` icons, no service-status widgets), so DNS-only egress from the
+# baseline above is sufficient and no egress CNP is added (minimal egress).
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: homepage-allow-gateway-ingress
+  namespace: homepage
+  labels:
+    app.kubernetes.io/name: homepage
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: homepage
+  ingress:
+    - fromEntities:
+        - ingress
+      toPorts:
+        - ports:
+            - port: "3000"
+              protocol: TCP

platform/homepage/service.yaml

@@ -0,0 +1,19 @@
+---
+# ClusterIP fronting Homepage. The shared Gateway routes the apex host to this Service (httproute.yaml);
+# Service port 80 -> container port 3000.
+apiVersion: v1
+kind: Service
+metadata:
+  name: homepage
+  namespace: homepage
+  labels:
+    app.kubernetes.io/name: homepage
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: homepage
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      protocol: TCP

platform/keycloak/k8s-oidc-rbac.yaml

@@ -11,6 +11,28 @@
 #
 # BREAK-GLASS: the cert-based admin in the bootstrap kubeconfig is unaffected by any of this and
 # keeps full access even if Keycloak/OIDC is down — do not remove it.
+#
+# OPERATOR KUBECONFIG (kubelogin / int128 oidc-login) — the `keycloak-oidc` user exec block:
+#   users:
+#   - name: keycloak-oidc
+#     user:
+#       exec:
+#         apiVersion: client.authentication.k8s.io/v1beta1
+#         command: kubectl
+#         args:
+#           - oidc-login
+#           - get-token
+#           - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
+#           - --oidc-client-id=kubernetes
+#           - --oidc-extra-scope=offline_access   # long-lived OFFLINE token — avoids frequent re-auth
+# The `--oidc-extra-scope=offline_access` arg makes kubelogin request an OFFLINE refresh token instead
+# of an ONLINE one, so the CLI is not forced to re-login on the realm's ONLINE session timeouts
+# (ssoSessionIdleTimeout 30m / ssoSessionMaxLifespan 10h). Offline tokens honour offlineSessionIdleTimeout
+# (30 days idle) with offlineSessionMaxLifespanEnabled=false (no hard cap). Backed by the `offline_access`
+# OPTIONAL client scope on the `kubernetes` client (see keycloak-realm-import.yaml). After adding the arg,
+# clear the cached ONLINE token once so the next login mints an offline one:
+#   kubectl oidc-login clean      # (or: rm -f ~/.kube/cache/oidc-login/*)
+#   kubectl get nodes             # re-login once to obtain the 30-day offline token
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:

platform/keycloak/keycloak-db-backup-credentials.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: keycloak-db-backup-credentials
+    namespace: keycloak
+type: Opaque
+stringData:
+    ACCESS_KEY_ID: ENC[AES256_GCM,data:N7iknaokuZ/tLg==,iv:yaGfsH5vplrNRqGnnpi8lYBGaIPDgTuRr0N2CWXXhh8=,tag:uLS1vvtUFGQu+Lb4b04pPg==,type:str]
+    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:lV9r1ZXtsSwfaGO42x6hH3zE0cvBWSjcN1jzcvfRzpKdXa45eY68YA0tKdSC+0oy,iv:xs/uQJq+NkasynNRtgq+mGz8bFzgw8ZkuWI/oKR5b2g=,tag:SU/XmxCQGCrkcEsFTHbg9A==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQVpJa1FWSDZudXhxQmJC
+            ZWlHVlorQk8vOWQxVzVLc3ZpTTVQQlBpbUNjClVwc3gvaVpyQzc5YVdYclp4YlVW
+            bWFpcFg3bDRvM1ROV0ZFSUZSU1NoeTAKLS0tIEpNSWlzRUM5VnpVY1cyRlY0T2xF
+            SjNFTy9FVm5GOFZ6WmgrV04ybGQxQjgKSnB2se7Ar1+piB0fn0wKSaOa3MLpetNa
+            gKPHJKP9OR5+KhlTnspBlxX9Un0WZcvZOQqAdewqlDklv4ukO7PgdQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-29T03:31:48Z"
+    mac: ENC[AES256_GCM,data:sxAENlR+qyBt2cBVG54hLbThXhzfNcKe8k1Ju3NmqWRyYLp/o7WHRwqydWHNYeY0YJJyuv73o1dDkVEPZEI2pxIlhaSwxhx+sDLaJnrYXq3YesbaT7jXkBrzz2MG1xjzV4nXgVYveM4rwCAWLij8qvIsbPMPyCHvaUjG/g0KDQ0=,iv:DoZgcHFsQUzTrStZfYr2bfQR6FGe5whNRF+lpzrXzKE=,tag:vGkrue7N0wjlDFZmQCqVQg==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4