diff --git a/conformance/tests/l0-static/cloudflared-ingress-rules/chainsaw-test.yaml b/conformance/tests/l0-static/cloudflared-ingress-rules/chainsaw-test.yaml index 82ea151..a32ab3e 100644 --- a/conformance/tests/l0-static/cloudflared-ingress-rules/chainsaw-test.yaml +++ b/conformance/tests/l0-static/cloudflared-ingress-rules/chainsaw-test.yaml @@ -39,7 +39,9 @@ spec: Parse platform/cloudflared/config.yaml with BASE_DOMAIN substituted, assert every rule's target against the Gateway Service derived from platform/gateway/gateway.yaml, assert the excluded hostnames 404 first, check the egress CiliumNetworkPolicy and the Flux wiring, and - run the evaluator on mutated configs to prove it is not vacuous. + run the evaluator on mutated configs to prove it is not vacuous; and assert the + credentials Secret volume is readable by the container's run-as uid/groups (fsGroup + + mode), self-tested by mutation. try: - script: content: bash ./check.sh diff --git a/conformance/tests/l0-static/cloudflared-ingress-rules/check.sh b/conformance/tests/l0-static/cloudflared-ingress-rules/check.sh index b1a4db6..ceba41d 100755 --- a/conformance/tests/l0-static/cloudflared-ingress-rules/check.sh +++ b/conformance/tests/l0-static/cloudflared-ingress-rules/check.sh @@ -26,9 +26,19 @@ # (5) the Flux tier is ISOLATED: its own Kustomization, wait:false, not listed in # platform/kustomization.yaml, nothing anywhere dependsOn it, and both overlays carry its # SOPS decryption patch — so a missing or invalid credential can stall nothing else; -# AND (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config +# (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config # (an API rule added, plex moved below the wildcard, the catch-all removed) and must -# flag each one. +# flag each one; +# AND (7) every Secret volume the container mounts — the tunnel credentials above all — is +# READABLE by the container's run-as identity, modelled the way the kubelet lays it out: +# files owned by uid 0, group-owned by fsGroup if set (else gid 0), with the item mode or +# defaultMode (API default 0644). The identity is the container securityContext over +# the pod's; the process's groups are runAsGroup + fsGroup + supplementalGroups. An +# unset runAsUser is a FAIL, not a pass: the image default cannot be read offline. The +# model is deliberately conservative — it does not credit the kubelet's habit of +# OR-ing group-read into fsGroup-owned volumes, so 0400+fsGroup fails here though it +# would work. Self-tested by mutation (fsGroup removed; mode 0400; runAsUser unset). +# This is the 2026-11-08 CrashLoop: runAsUser 65532, no fsGroup, 0440 root:root. # # NOT proven (static): that Cloudflare routes only the hostnames whose DNS points at the tunnel, # that the Gateway's certificate validates from cloudflared, or that Envoy enforces (4) — those are @@ -236,6 +246,69 @@ check(dep["spec"]["template"]["metadata"]["labels"].get("homelab.keithjackson.de check(dep["spec"]["template"]["spec"].get("automountServiceAccountToken") is False, "(4) the pods mount no ServiceAccount token") +# (7) the credentials Secret is readable by the run-as identity +def unreadable_secret_volumes(pod_spec): + """Return a description per Secret volume the containers mount but cannot read.""" + bad = [] + psc = pod_spec.get("securityContext") or {} + vols = {v["name"]: v for v in pod_spec.get("volumes") or [] if "secret" in v} + for c in (pod_spec.get("initContainers") or []) + (pod_spec.get("containers") or []): + csc = c.get("securityContext") or {} + uid = csc.get("runAsUser", psc.get("runAsUser")) + gid = csc.get("runAsGroup", psc.get("runAsGroup")) + fs_group = psc.get("fsGroup") + groups = {g for g in [gid, fs_group, *(psc.get("supplementalGroups") or [])] + if g is not None} + for vm in c.get("volumeMounts") or []: + v = vols.get(vm["name"]) + if v is None: + continue + sec = v["secret"] + if uid is None: + bad.append(f"container {c['name']} mounts Secret volume {vm['name']} but sets no " + f"runAsUser (pod or container) — readability cannot be proven offline") + continue + default = sec.get("defaultMode", 0o644) + modes = [it.get("mode", default) for it in sec.get("items") or []] or [default] + owner_gid = fs_group if fs_group is not None else 0 + for mode in modes: + readable = ((uid == 0 and mode & 0o400) or (owner_gid in groups and mode & 0o040) + or mode & 0o004) + if not readable: + bad.append(f"container {c['name']} (uid {uid}, groups {sorted(groups)}) cannot " + f"read Secret volume {vm['name']} ({sec.get('secretName')}): files " + f"are 0:{owner_gid} mode {mode:04o} — set pod fsGroup to a group " + f"the container runs with, keeping 0440") + return bad + + +pod_spec = dep["spec"]["template"]["spec"] +creds_vol = [v for v in pod_spec.get("volumes") or [] + if (v.get("secret") or {}).get("secretName") == "cloudflared-credentials"] +check(len(creds_vol) == 1, "(7) the Deployment mounts Secret cloudflared-credentials as a volume") +for v in unreadable_secret_volumes(pod_spec): + check(False, f"(7) {v}") +sec_mutants = {} +m = copy.deepcopy(pod_spec) +m["securityContext"].pop("fsGroup", None) +m["securityContext"].pop("supplementalGroups", None) +sec_mutants["fsGroup removed"] = m +m = copy.deepcopy(pod_spec) +for v in m["volumes"]: + if "secret" in v: + v["secret"]["defaultMode"] = 0o400 + for it in v["secret"].get("items") or []: + it["mode"] = 0o400 +sec_mutants["mode 0400"] = m +m = copy.deepcopy(pod_spec) +m["securityContext"].pop("runAsUser", None) +for c in m["containers"]: + (c.get("securityContext") or {}).pop("runAsUser", None) +sec_mutants["runAsUser unset"] = m +for name, mut in sec_mutants.items(): + check(unreadable_secret_volumes(mut), + f"(7) self-test: the readability model did NOT flag the mutant '{name}' — vacuous") + if CHECKS < 25: FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated") if FAILURES: diff --git a/decisions/2026-11-08-cloudflared-creds-permission.md b/decisions/2026-11-08-cloudflared-creds-permission.md new file mode 100644 index 0000000..e78350e --- /dev/null +++ b/decisions/2026-11-08-cloudflared-creds-permission.md @@ -0,0 +1,59 @@ +--- +date: 2026-11-08 +branch: fix/cloudflared-creds-permission +title: cloudflared CrashLoop — credentials Secret unreadable by uid 65532; add fsGroup, pin it in L0 +--- + +## What changed + +- **`platform/cloudflared/deployment.yaml`**: pod `securityContext.fsGroup: 65532`. Root cause of + the CrashLoop that followed #152/#153: the pod runs as `65532:65532` with no `fsGroup`, and the + Secret volume is `defaultMode: 0440`. The kubelet therefore laid `credentials.json` down as + `root:root 0440`, which uid 65532 (not owner, not in gid 0) cannot read. cloudflared logged + `couldn't read tunnel credentials ... permission denied`, then fell back to origin-cert lookup + and ended on the misleading `error parsing tunnel ID: Error locating origin cert`. With + `fsGroup` the files are group-owned by 65532, readable by the container and no other uid; the + mode stays 0440 (not loosened to 0444). Same fix as `platform/keycloak/client-secret-reconciler.yaml`. + Still non-root, `runAsNonRoot: true`, unchanged otherwise. +- **`conformance/tests/l0-static/cloudflared-ingress-rules/check.sh`**: new assertion (7). For each + Secret volume a container mounts, it models kubelet ownership (uid 0, gid = fsGroup or 0, item + mode / defaultMode / API default 0644) against the container's effective runAsUser and its groups + (runAsGroup + fsGroup + supplementalGroups) and fails if the file is unreadable. An unset + runAsUser fails (image default is not knowable offline). Conservative: it does not credit the + kubelet OR-ing group-read into fsGroup volumes. Mutation self-test: fsGroup removed, mode 0400, + runAsUser unset must each be flagged. Chainsaw step description updated to match. +- **`runbooks/cloudflare-tunnel.md`**: new first "likely cause" (permission denied on + credentials.json → the origin-cert error is a red herring) plus a "read the first error line" + note; causes renumbered. New section **Rotating the tunnel credential**: create a new tunnel, + swap the UUID and the encrypted credentials, merge, verify (new tunnel has two connectors, old + has none), then `cloudflared tunnel cleanup`/`delete `. +- NOT touched, on purpose: `tunnel:` in `config.yaml` and `cloudflared-credentials.sops.yaml`. The + operator is rotating the credential (it was exposed in a transcript) in a separate PR. + +## What I verified + +- Live (k3s-prod, read-only, 2026-11-08 ~21:38Z): Deployment pod securityContext is + `{"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532,"seccompProfile":...}`, no + `fsGroup`; creds volume `defaultMode: 288` (= 0440). All three pods CrashLoopBackOff (38 + restarts). Secret `cloudflared-credentials` has the one key `credentials.json` (key names only). +- Pinned image `cloudflare/cloudflared@sha256:4955524a...` config (`skopeo inspect --config`): + `User: 65532:65532`, `Entrypoint: [cloudflared]`, `Cmd: [version]`. +- The new assertion FAILS (rc 1) on the pre-fix deployment.yaml with + `container cloudflared (uid 65532, groups [65532]) cannot read Secret volume creds ... files are + 0:0 mode 0440`, and PASSES (51 assertions) with the fix. All three mutants are flagged. +- Kyverno admission policies under `platform/kyverno/admission/` that mention securityContext + fields are scoped to the conformance-trigger and pm-agent Jobs only; none constrains fsGroup here. +- Toolbox gate: see the PR description for the rc and skipped stages. + +## What I did NOT verify + +- **That the pods actually become Ready after merge.** Not deployed by me; Flux applies it. The + file mode inside the running container was not observed (pods crash before an exec is possible, + and exec would be an operator action anyway). The readability claim rests on the kubelet's + documented fsGroup behaviour and the identical, working keycloak reconciler. +- Whether the CURRENT credential is otherwise valid (UUID/TunnelID match, tunnel exists). Moot: + the operator is rotating it. +- The rotation runbook section has not been executed; `cloudflared tunnel cleanup` / `delete` + syntax is from the CLI's documented commands, not run here. +- The deployment comment says the image ENTRYPOINT is `cloudflared --no-autoupdate`; the pinned + image's ENTRYPOINT is plain `cloudflared`. Left unchanged (out of scope); flagged for follow-up. diff --git a/platform/cloudflared/deployment.yaml b/platform/cloudflared/deployment.yaml index c6e0fa4..388eed0 100644 --- a/platform/cloudflared/deployment.yaml +++ b/platform/cloudflared/deployment.yaml @@ -47,6 +47,14 @@ spec: # The image is distroless `nonroot` (65532). runAsUser: 65532 runAsGroup: 65532 + # fsGroup makes the kubelet GROUP-own the Secret volume's files to 65532. Without it they + # are root:root, and defaultMode 0440 (below) is unreadable to uid 65532 — every replica + # CrashLooped on "couldn't read tunnel credentials ... permission denied" (2026-11-08), + # which cloudflared then misreports as "Error locating origin cert". Same lesson as + # platform/keycloak/client-secret-reconciler.yaml. fsGroup + 0440 keeps the credential + # unreadable to any other uid rather than relaxing it to world-readable 0444. + # Pinned by the L0 check cloudflared-ingress-rules, assertion (7). + fsGroup: 65532 seccompProfile: type: RuntimeDefault topologySpreadConstraints: @@ -121,6 +129,8 @@ spec: secret: # cloudflared-credentials.sops.yaml — the tunnel credentials JSON, by name only. secretName: cloudflared-credentials + # 0440, owner root, group fsGroup 65532 (pod securityContext) — readable by the + # container user through its group, by nobody else. defaultMode: 0440 --- # One replica may be disrupted at a time — enough for kured's one-node-at-a-time drain to diff --git a/runbooks/cloudflare-tunnel.md b/runbooks/cloudflare-tunnel.md index eabbaf8..4e6bb3e 100644 --- a/runbooks/cloudflare-tunnel.md +++ b/runbooks/cloudflare-tunnel.md @@ -63,23 +63,36 @@ patch is missing; see causes below. ## Likely causes, most likely first -1. **The tunnel has not been enrolled yet** (Phase 1, steps 1-5 below not done). Logs show an - error about the tunnel `REPLACE-ME-TUNNEL-UUID` or a missing origin certificate. Do steps 1-5. -2. **UUID and credentials disagree** — `config.yaml` `tunnel:` is not the `TunnelID` in the +**Read the FIRST error line, not the last.** When cloudflared cannot use the credentials file it +falls back to looking for an origin certificate, and the error it ends on — +`Cannot determine default origin certificate path` / `error parsing tunnel ID: Error locating +origin cert` — is a **red herring** whatever the real cause was. The credentials file is the +whole of what this Deployment needs; no `cert.pem` ever belongs in the pod. + +1. **`permission denied` on `credentials.json` → the "origin cert" error is a red herring.** Logs: + `couldn't read tunnel credentials from /etc/cloudflared/creds/credentials.json: permission + denied`, then the origin-cert lines. The container (uid 65532) cannot read the Secret file: + the pod `securityContext.fsGroup: 65532` is missing, so the 0440 file is `root:root`. Fix it in + git (`platform/cloudflared/deployment.yaml`); the L0 check `cloudflared-ingress-rules`, + assertion (7), should already have refused that. Do NOT loosen `defaultMode` to 0444. (This is + what CrashLooped every replica on 2026-11-08.) +2. **The tunnel has not been enrolled yet** (Phase 1, steps 1-5 below not done). Logs show an + error about the tunnel `REPLACE-ME-TUNNEL-UUID` (and the origin-cert red herring). Do steps 1-5. +3. **UUID and credentials disagree** — `config.yaml` `tunnel:` is not the `TunnelID` in the credentials JSON. Logs: authentication / "tunnel not found". Check with step 4's verify command. -3. **Credentials still encrypted in the cluster** — the `cloudflared` decryption patch is missing +4. **Credentials still encrypted in the cluster** — the `cloudflared` decryption patch is missing from `clusters/prod/kustomization.yaml`. The L0 check guards this, so this means someone bypassed it. -4. **Egress blocked** — a policy change, or the edge moved off 7844. Look for drops: +5. **Egress blocked** — a policy change, or the edge moved off 7844. Look for drops: ``` infra/devcontainer/dev.sh run bash -c \ 'kubectl --context k3s-prod -n kube-system exec ds/cilium -c cilium-agent -- \ hubble observe --verdict DROPPED --from-namespace cloudflared --last 100 -o compact' ``` (`ds/cilium` picks one node's agent; repeat per node if empty.) -5. **The tunnel was deleted or its secret rotated in Cloudflare** — `cloudflared tunnel list` +6. **The tunnel was deleted or its secret rotated in Cloudflare** — `cloudflared tunnel list` (step 2's podman form) no longer shows it. Re-enrol with steps 2-5. -6. **`CloudflaredReplicasDegraded` only**: one replica cannot schedule (the spread is hard, one per +7. **`CloudflaredReplicasDegraded` only**: one replica cannot schedule (the spread is hard, one per node — a cordoned node during a kured reboot is normal and clears) or an image pull failed. ## When to escalate or stop @@ -270,6 +283,42 @@ disappears with the pod on the next rollout.) Finally **delete the `tunnel-canary` CNAME** in the dashboard and confirm it is gone: `dig +short tunnel-canary.home.keithjackson.dev @1.1.1.1` returns nothing. +## Rotating the tunnel credential + +Do this whenever the credentials JSON (`TunnelSecret`) or `cert.pem` may have been exposed — a +transcript, a log, a screen share. A tunnel's secret cannot be changed in place from this +workflow, so rotation is **a new tunnel**: the old credential stays valid until the old tunnel +is deleted, which is why the last step is not optional. + +1. **Create the new tunnel** (*Enrolling*, step 2's `cf` helper; `cf tunnel login` again if + `cert.pem` is not to hand). Use a NEW name — the old one is still taken: + ``` + cf tunnel list # note the OLD tunnel's id and name + cf tunnel create --credentials-file /root/.cloudflared/tunnel.json home-k3s-2 + ``` +2. **Swap the UUID** in your own worktree, on a new branch (CLAUDE.md §0): + ``` + OLD=$(sed -n 's/^tunnel: //p' platform/cloudflared/config.yaml) + sed -i "s/^tunnel: ${OLD}$/tunnel: /" platform/cloudflared/config.yaml + ``` +3. **Swap the encrypted credentials** with *Enrolling* step 4 — the same `sops set`, then the + same `ok`-only verify, which now checks the JSON's `TunnelID` against the NEW UUID. Never + `sops -d` to the terminal. +4. **Merge** — gate, PR, `/approve-override` by the operator (it touches `**/*.sops.yaml`), merge. + Flux rolls the Deployment because the content-hashed ConfigMap changed. +5. **Verify** with *Enrolling* step 6: both replicas Ready, `Registered tunnel connection` in + each replica's log, `cf tunnel info home-k3s-2` lists two connectors, and + `cf tunnel info ` lists **none**. +6. **Only then delete the old tunnel**, which revokes the exposed credential: + ``` + cf tunnel cleanup # drops any stale connector records first + cf tunnel delete + cf tunnel list # the old tunnel is gone + ``` + After the Phase 2 DNS flip, records point at `.cfargotunnel.com`: confirm external-dns + has repointed every record to the NEW UUID before deleting, or those hostnames go dark. + Then replace `tunnel.json`/`cert.pem` in the password manager and delete them from disk. + ## Rollback - **Phase 1 (no DNS on the tunnel):** revert the PR. Flux prunes the `cloudflared` namespace and