#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — cloudflared-ingress-rules # Serves: R-NET-01 / R-NET-02 (default-deny; allow rules declarative and scoped), and the four # operator decisions of 2026-11-08 recorded in decisions/2026-11-08-cloudflare-tunnel.md. # # WHY THIS EXISTS. platform/cloudflared/config.yaml is the whole of what the internet can reach # through the Cloudflare Tunnel. A one-line edit there — a rule pointing at the Kubernetes API, a # LAN-only hostname moved below the wildcard, a dropped catch-all — publishes something the operator # decided must never be public, and nothing in the cluster turns red: the tunnel stays connected and # the Kustomization stays Ready. So the decisions are asserted here, on every gate run. # # GIVEN the repository as authored, # THEN (1) every ingress rule's `service` is `http_status:404` or EXACTLY the shared Gateway's # in-cluster Service, derived from platform/gateway/gateway.yaml (so a Gateway rename # fails here rather than in production) — which by construction excludes the # Kubernetes API, and a named deny-list says so explicitly when it is violated; # (2) each LAN/VPN-only hostname (plex, plex-vip, jackett, pihole, pihole-dns, semaphore, # qbittorrent, radarr, sonarr, headlamp, unifi, crafty-mc) has a rule mapping it to http_status:404 that comes BEFORE any # Gateway-bound rule (cloudflared matches first-rule-wins); # (3) the apex and the wildcard are the Gateway rules, with matchSNItoHost, and nothing # disables origin TLS verification; the last rule is a hostname-less 404 catch-all; # (4) the egress CiliumNetworkPolicy admits none of kube-apiserver/host/remote-node/ # cluster/all and no CIDR rule, and its backend selector excludes the LAN-only # namespaces (the second, independent layer behind (2)); # (5) the Flux tier is ISOLATED: its own Kustomization, wait:false, not listed in # platform/kustomization.yaml, nothing anywhere dependsOn it, and both overlays carry its # SOPS decryption patch — so a missing or invalid credential can stall nothing else; # (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config # (an API rule added, plex moved below the wildcard, the catch-all removed) and must # flag each one; # AND (7) every Secret volume the container mounts — the tunnel credentials above all — is # READABLE by the container's run-as identity, modelled the way the kubelet lays it out: # files owned by uid 0, group-owned by fsGroup if set (else gid 0), with the item mode or # defaultMode (API default 0644). The identity is the container securityContext over # the pod's; the process's groups are runAsGroup + fsGroup + supplementalGroups. An # unset runAsUser is a FAIL, not a pass: the image default cannot be read offline. The # model is deliberately conservative — it does not credit the kubelet's habit of # OR-ing group-read into fsGroup-owned volumes, so 0400+fsGroup fails here though it # would work. Self-tested by mutation (fsGroup removed; mode 0400; runAsUser unset). # This is the 2026-11-08 CrashLoop: runAsUser 65532, no fsGroup, 0440 root:root. # # NOT proven (static): that Cloudflare routes only the hostnames whose DNS points at the tunnel, # that the Gateway's certificate validates from cloudflared, or that Envoy enforces (4) — those are # runtime properties, verified by the operator per runbooks/cloudflare-tunnel.md. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root exec python3 - "$ROOT" <<'PY' import copy import re import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("CHECK FAIL (cloudflared-ingress-rules): PyYAML is required", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) FAILURES, CHECKS = [], 0 def check(cond, desc): global CHECKS CHECKS += 1 if not cond: FAILURES.append(desc) def docs(path): return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)] COMP = ROOT / "platform" / "cloudflared" CONFIG = COMP / "config.yaml" if not CONFIG.is_file(): print(f"CHECK FAIL (cloudflared-ingress-rules): {CONFIG.relative_to(ROOT)} is absent — the " "component this check guards was moved or deleted; re-point or delete this check in the " "SAME change", file=sys.stderr) raise SystemExit(1) # The hostnames the operator decided never go through the tunnel (decisions 2026-11-08). # headlamp: operator decision 2026-11-08 (LAN/VPN-only). qbittorrent/radarr/sonarr: catalog # `exposure: internal`, like jackett/pihole/semaphore. EXCLUDED_HOSTS = ("plex", "plex-vip", "jackett", "pihole", "pihole-dns", "semaphore", "qbittorrent", "radarr", "sonarr", "headlamp", "cve", "unifi", "crafty-mc") # Their Gateway-backend namespaces (unifi / crafty-mc are L4 VIPs, not Gateway backends). EXCLUDED_NAMESPACES = ("plex", "jackett", "pihole", "semaphore", "qbittorrent", "radarr", "sonarr", "headlamp", "vuln-scanner") FORBIDDEN_ENTITIES = {"kube-apiserver", "host", "remote-node", "cluster", "all"} # Any of these in a `service` means a route toward the Kubernetes API (or a raw-TCP/SSH bastion). API_MARKERS = ("kubernetes", "kube-apiserver", "apiserver", ":6443", "10.43.0.1", "tcp://", "ssh://", "unix:", "bastion") # ── substitution: ${BASE_DOMAIN} from cluster-vars, the ONLY token allowed ────────────────────── cv = next(d for d in docs(ROOT / "clusters" / "base" / "cluster-vars.yaml") if d.get("kind") == "ConfigMap") base_domain = cv["data"]["BASE_DOMAIN"] raw = CONFIG.read_text() tokens = set(re.findall(r"\$\{([A-Za-z0-9_]+)\}", raw)) check(tokens == {"BASE_DOMAIN"}, f"(0) config.yaml uses exactly the ${{BASE_DOMAIN}} token (found {sorted(tokens)}); any other " f"token is substituted from cluster-vars unseen by this check") check(re.search(r"\$(?!\{BASE_DOMAIN\})", raw) is None, "(0) config.yaml contains a bare `$` — Flux envsubst would rewrite or empty it") cfg = yaml.safe_load(raw.replace("${BASE_DOMAIN}", base_domain)) # ── the Gateway Service, derived rather than hard-coded ───────────────────────────────────────── gw = next(d for d in docs(ROOT / "platform" / "gateway" / "gateway.yaml") if d.get("kind") == "Gateway") GATEWAY_SVC = (f"https://cilium-gateway-{gw['metadata']['name']}." f"{gw['metadata']['namespace']}.svc.cluster.local:443") def evaluate(cfg): """Return the list of rule violations for a parsed cloudflared config.""" bad = [] rules = cfg.get("ingress") or [] if len(rules) < len(EXCLUDED_HOSTS) + 3: bad.append(f"only {len(rules)} ingress rules — the scan is vacuous") return bad for i, r in enumerate(rules): svc = str(r.get("service", "")) if any(m in svc.lower() for m in API_MARKERS): bad.append(f"rule {i} ({r.get('hostname', '')}) targets {svc!r}, which " f"is a route toward the Kubernetes API / a raw bastion — the API stays " f"local-only (operator decision 2026-11-08)") if svc not in ("http_status:404", GATEWAY_SVC): bad.append(f"rule {i} ({r.get('hostname', '')}) targets {svc!r}; only " f"http_status:404 or {GATEWAY_SVC} are permitted") if (r.get("originRequest") or {}).get("noTLSVerify"): bad.append(f"rule {i} disables origin TLS verification (noTLSVerify)") if (cfg.get("originRequest") or {}).get("noTLSVerify"): bad.append("top-level originRequest disables origin TLS verification (noTLSVerify)") gw_idx = [i for i, r in enumerate(rules) if r.get("service") == GATEWAY_SVC] first_gw = min(gw_idx) if gw_idx else len(rules) for h in EXCLUDED_HOSTS: fqdn = f"{h}.{base_domain}" hits = [i for i, r in enumerate(rules) if r.get("hostname") == fqdn] if not hits: bad.append(f"no rule for LAN/VPN-only host {fqdn}; it would fall through to the " f"wildcard and reach the Gateway") continue i = hits[0] if rules[i].get("service") != "http_status:404": bad.append(f"{fqdn} maps to {rules[i].get('service')!r}, not http_status:404") if i > first_gw: bad.append(f"{fqdn}'s 404 rule (#{i}) comes AFTER the first Gateway rule (#{first_gw}); " f"cloudflared is first-match, so the Gateway rule wins") gw_hosts = {rules[i].get("hostname") for i in gw_idx} if gw_hosts != {base_domain, f"*.{base_domain}"}: bad.append(f"the Gateway-bound hostnames are {sorted(map(str, gw_hosts))}; expected " f"exactly the apex and the wildcard") for i in gw_idx: if (rules[i].get("originRequest") or {}).get("matchSNItoHost") is not True: bad.append(f"Gateway rule #{i} lacks originRequest.matchSNItoHost: true — the Gateway " f"would not see the SNI that selects its wildcard certificate") last = rules[-1] if "hostname" in last or last.get("service") != "http_status:404": bad.append("the last rule is not a hostname-less http_status:404 catch-all") return bad # (1)-(3) the real config for v in evaluate(cfg): check(False, f"(1-3) {v}") check(str(cfg.get("credentials-file", "")).startswith("/etc/cloudflared/creds/"), "(3) credentials-file points into the mounted Secret volume") # (6) mutation self-test — the evaluator must flag each broken variant mutants = {} m = copy.deepcopy(cfg) m["ingress"].insert(0, {"hostname": f"k8s.{base_domain}", "service": "https://kubernetes.default.svc:443"}) mutants["API rule added"] = m m = copy.deepcopy(cfg) plex = next(r for r in m["ingress"] if r.get("hostname") == f"plex.{base_domain}") m["ingress"].remove(plex) m["ingress"].insert(len(m["ingress"]) - 1, plex) mutants["plex moved below the wildcard"] = m m = copy.deepcopy(cfg) m["ingress"] = m["ingress"][:-1] mutants["catch-all removed"] = m for name, mut in mutants.items(): check(evaluate(mut), f"(6) self-test: the evaluator did NOT flag the mutant '{name}' — the " f"check is vacuous") # (4) the egress policy np_docs = docs(COMP / "networkpolicy.yaml") cnps = [d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy"] check(len(cnps) == 1, f"(4) exactly one CiliumNetworkPolicy in networkpolicy.yaml (found {len(cnps)})") for cnp in cnps: for i, rule in enumerate(cnp["spec"].get("egress") or []): ents = set(rule.get("toEntities") or []) check(not (ents & FORBIDDEN_ENTITIES), f"(4) egress rule {i} admits {sorted(ents & FORBIDDEN_ENTITIES)} — that reaches the " f"Kubernetes API or the nodes; the API stays local-only") check(not (rule.get("toCIDR") or rule.get("toCIDRSet")), f"(4) egress rule {i} uses a CIDR — a CIDR can name the API VIP; use entities") for ep in rule.get("toEndpoints") or []: notin = [e for e in ep.get("matchExpressions") or [] if e.get("key") == "io.kubernetes.pod.namespace" and e.get("operator") == "NotIn"] missing = set(EXCLUDED_NAMESPACES) - set(notin[0]["values"] if notin else []) check(not missing, f"(4) backend selector in egress rule {i} does not exclude {sorted(missing)}") for d in np_docs: if d.get("kind") == "NetworkPolicy" and d["metadata"]["name"] == "default-deny": check(set(d["spec"].get("policyTypes", [])) == {"Ingress", "Egress"}, "(4) default-deny covers both directions") break else: check(False, "(4) networkpolicy.yaml ships no default-deny baseline for the namespace") # (5) Flux isolation fk = next(d for d in docs(ROOT / "clusters" / "base" / "cloudflared.yaml") if d.get("kind") == "Kustomization") check(fk["spec"].get("wait") is False, "(5) clusters/base/cloudflared.yaml must be wait: false — its pods are legitimately not-Ready " "until the tunnel is enrolled") check(fk["spec"].get("path") == "./platform/cloudflared", "(5) the Kustomization reconciles ./platform/cloudflared") check("cloudflared.yaml" in (ROOT / "clusters" / "base" / "kustomization.yaml").read_text(), "(5) clusters/base/kustomization.yaml lists cloudflared.yaml") check("cloudflared" not in (ROOT / "platform" / "kustomization.yaml").read_text(), "(5) platform/kustomization.yaml must not include cloudflared — that is the gating tier") for f in sorted((ROOT / "clusters").rglob("*.yaml")): for d in docs(f): for dep in (d.get("spec") or {}).get("dependsOn") or []: check(not (isinstance(dep, dict) and dep.get("name") == "cloudflared"), f"(5) {f.relative_to(ROOT)} {d.get('metadata', {}).get('name')} dependsOn " f"cloudflared — that hands an unenrolled tunnel the power to stall it") for env in ("dev", "prod"): text = (ROOT / "clusters" / env / "kustomization.yaml").read_text() check(re.search(r"name:\s*cloudflared\s*\n\s*namespace:\s*flux-system\s*\n\s*spec:\s*\n" r"\s*decryption:", text) is not None, f"(5) clusters/{env}/kustomization.yaml carries the cloudflared SOPS decryption patch") dep = next(d for d in docs(COMP / "deployment.yaml") if d.get("kind") == "Deployment") check(dep["spec"].get("replicas", 1) >= 2, "(5) the Deployment runs at least 2 replicas") check(dep["spec"]["template"]["metadata"]["labels"].get("homelab.keithjackson.dev/egress-class") == "internet", "(4) the pods are egress-class=internet — vpn would put the edge connections behind PIA") check(dep["spec"]["template"]["spec"].get("automountServiceAccountToken") is False, "(4) the pods mount no ServiceAccount token") # (7) the credentials Secret is readable by the run-as identity def unreadable_secret_volumes(pod_spec): """Return a description per Secret volume the containers mount but cannot read.""" bad = [] psc = pod_spec.get("securityContext") or {} vols = {v["name"]: v for v in pod_spec.get("volumes") or [] if "secret" in v} for c in (pod_spec.get("initContainers") or []) + (pod_spec.get("containers") or []): csc = c.get("securityContext") or {} uid = csc.get("runAsUser", psc.get("runAsUser")) gid = csc.get("runAsGroup", psc.get("runAsGroup")) fs_group = psc.get("fsGroup") groups = {g for g in [gid, fs_group, *(psc.get("supplementalGroups") or [])] if g is not None} for vm in c.get("volumeMounts") or []: v = vols.get(vm["name"]) if v is None: continue sec = v["secret"] if uid is None: bad.append(f"container {c['name']} mounts Secret volume {vm['name']} but sets no " f"runAsUser (pod or container) — readability cannot be proven offline") continue default = sec.get("defaultMode", 0o644) modes = [it.get("mode", default) for it in sec.get("items") or []] or [default] owner_gid = fs_group if fs_group is not None else 0 for mode in modes: readable = ((uid == 0 and mode & 0o400) or (owner_gid in groups and mode & 0o040) or mode & 0o004) if not readable: bad.append(f"container {c['name']} (uid {uid}, groups {sorted(groups)}) cannot " f"read Secret volume {vm['name']} ({sec.get('secretName')}): files " f"are 0:{owner_gid} mode {mode:04o} — set pod fsGroup to a group " f"the container runs with, keeping 0440") return bad pod_spec = dep["spec"]["template"]["spec"] creds_vol = [v for v in pod_spec.get("volumes") or [] if (v.get("secret") or {}).get("secretName") == "cloudflared-credentials"] check(len(creds_vol) == 1, "(7) the Deployment mounts Secret cloudflared-credentials as a volume") for v in unreadable_secret_volumes(pod_spec): check(False, f"(7) {v}") sec_mutants = {} m = copy.deepcopy(pod_spec) m["securityContext"].pop("fsGroup", None) m["securityContext"].pop("supplementalGroups", None) sec_mutants["fsGroup removed"] = m m = copy.deepcopy(pod_spec) for v in m["volumes"]: if "secret" in v: v["secret"]["defaultMode"] = 0o400 for it in v["secret"].get("items") or []: it["mode"] = 0o400 sec_mutants["mode 0400"] = m m = copy.deepcopy(pod_spec) m["securityContext"].pop("runAsUser", None) for c in m["containers"]: (c.get("securityContext") or {}).pop("runAsUser", None) sec_mutants["runAsUser unset"] = m for name, mut in sec_mutants.items(): check(unreadable_secret_volumes(mut), f"(7) self-test: the readability model did NOT flag the mutant '{name}' — vacuous") if CHECKS < 25: FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated") if FAILURES: print(f"CHECK FAIL (cloudflared-ingress-rules): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr) for f in FAILURES: print(f" - {f}", file=sys.stderr) raise SystemExit(1) print(f"PASS (cloudflared-ingress-rules): {CHECKS} assertions") PY