#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — cloudflared-ingress-rules # Serves: R-NET-01 / R-NET-02 (default-deny; allow rules declarative and scoped), and the four # operator decisions of 2026-11-08 recorded in decisions/2026-11-08-cloudflare-tunnel.md. # # WHY THIS EXISTS. platform/cloudflared/config.yaml is the whole of what the internet can reach # through the Cloudflare Tunnel. A one-line edit there — a rule pointing at the Kubernetes API, a # LAN-only hostname moved below the wildcard, a dropped catch-all — publishes something the operator # decided must never be public, and nothing in the cluster turns red: the tunnel stays connected and # the Kustomization stays Ready. So the decisions are asserted here, on every gate run. # # GIVEN the repository as authored, # THEN (1) every ingress rule's `service` is `http_status:404` or EXACTLY the shared Gateway's # in-cluster Service, derived from platform/gateway/gateway.yaml (so a Gateway rename # fails here rather than in production) — which by construction excludes the # Kubernetes API, and a named deny-list says so explicitly when it is violated; # (2) each LAN/VPN-only hostname (plex, plex-vip, jackett, pihole, pihole-dns, semaphore, # qbittorrent, radarr, sonarr, headlamp, unifi, crafty-mc) has a rule mapping it to http_status:404 that comes BEFORE any # Gateway-bound rule (cloudflared matches first-rule-wins); # (3) the apex and the wildcard are the Gateway rules, with matchSNItoHost, and nothing # disables origin TLS verification; the last rule is a hostname-less 404 catch-all; # (4) the egress CiliumNetworkPolicy admits none of kube-apiserver/host/remote-node/ # cluster/all and no CIDR rule, and its backend selector excludes the LAN-only # namespaces (the second, independent layer behind (2)); # (5) the Flux tier is ISOLATED: its own Kustomization, wait:false, not listed in # platform/kustomization.yaml, nothing anywhere dependsOn it, and both overlays carry its # SOPS decryption patch — so a missing or invalid credential can stall nothing else; # AND (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config # (an API rule added, plex moved below the wildcard, the catch-all removed) and must # flag each one. # # NOT proven (static): that Cloudflare routes only the hostnames whose DNS points at the tunnel, # that the Gateway's certificate validates from cloudflared, or that Envoy enforces (4) — those are # runtime properties, verified by the operator per runbooks/cloudflare-tunnel.md. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root exec python3 - "$ROOT" <<'PY' import copy import re import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("CHECK FAIL (cloudflared-ingress-rules): PyYAML is required", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) FAILURES, CHECKS = [], 0 def check(cond, desc): global CHECKS CHECKS += 1 if not cond: FAILURES.append(desc) def docs(path): return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)] COMP = ROOT / "platform" / "cloudflared" CONFIG = COMP / "config.yaml" if not CONFIG.is_file(): print(f"CHECK FAIL (cloudflared-ingress-rules): {CONFIG.relative_to(ROOT)} is absent — the " "component this check guards was moved or deleted; re-point or delete this check in the " "SAME change", file=sys.stderr) raise SystemExit(1) # The hostnames the operator decided never go through the tunnel (decisions 2026-11-08). # headlamp: operator decision 2026-11-08 (LAN/VPN-only). qbittorrent/radarr/sonarr: catalog # `exposure: internal`, like jackett/pihole/semaphore. EXCLUDED_HOSTS = ("plex", "plex-vip", "jackett", "pihole", "pihole-dns", "semaphore", "qbittorrent", "radarr", "sonarr", "headlamp", "unifi", "crafty-mc") # Their Gateway-backend namespaces (unifi / crafty-mc are L4 VIPs, not Gateway backends). EXCLUDED_NAMESPACES = ("plex", "jackett", "pihole", "semaphore", "qbittorrent", "radarr", "sonarr", "headlamp") FORBIDDEN_ENTITIES = {"kube-apiserver", "host", "remote-node", "cluster", "all"} # Any of these in a `service` means a route toward the Kubernetes API (or a raw-TCP/SSH bastion). API_MARKERS = ("kubernetes", "kube-apiserver", "apiserver", ":6443", "10.43.0.1", "tcp://", "ssh://", "unix:", "bastion") # ── substitution: ${BASE_DOMAIN} from cluster-vars, the ONLY token allowed ────────────────────── cv = next(d for d in docs(ROOT / "clusters" / "base" / "cluster-vars.yaml") if d.get("kind") == "ConfigMap") base_domain = cv["data"]["BASE_DOMAIN"] raw = CONFIG.read_text() tokens = set(re.findall(r"\$\{([A-Za-z0-9_]+)\}", raw)) check(tokens == {"BASE_DOMAIN"}, f"(0) config.yaml uses exactly the ${{BASE_DOMAIN}} token (found {sorted(tokens)}); any other " f"token is substituted from cluster-vars unseen by this check") check(re.search(r"\$(?!\{BASE_DOMAIN\})", raw) is None, "(0) config.yaml contains a bare `$` — Flux envsubst would rewrite or empty it") cfg = yaml.safe_load(raw.replace("${BASE_DOMAIN}", base_domain)) # ── the Gateway Service, derived rather than hard-coded ───────────────────────────────────────── gw = next(d for d in docs(ROOT / "platform" / "gateway" / "gateway.yaml") if d.get("kind") == "Gateway") GATEWAY_SVC = (f"https://cilium-gateway-{gw['metadata']['name']}." f"{gw['metadata']['namespace']}.svc.cluster.local:443") def evaluate(cfg): """Return the list of rule violations for a parsed cloudflared config.""" bad = [] rules = cfg.get("ingress") or [] if len(rules) < len(EXCLUDED_HOSTS) + 3: bad.append(f"only {len(rules)} ingress rules — the scan is vacuous") return bad for i, r in enumerate(rules): svc = str(r.get("service", "")) if any(m in svc.lower() for m in API_MARKERS): bad.append(f"rule {i} ({r.get('hostname', '')}) targets {svc!r}, which " f"is a route toward the Kubernetes API / a raw bastion — the API stays " f"local-only (operator decision 2026-11-08)") if svc not in ("http_status:404", GATEWAY_SVC): bad.append(f"rule {i} ({r.get('hostname', '')}) targets {svc!r}; only " f"http_status:404 or {GATEWAY_SVC} are permitted") if (r.get("originRequest") or {}).get("noTLSVerify"): bad.append(f"rule {i} disables origin TLS verification (noTLSVerify)") if (cfg.get("originRequest") or {}).get("noTLSVerify"): bad.append("top-level originRequest disables origin TLS verification (noTLSVerify)") gw_idx = [i for i, r in enumerate(rules) if r.get("service") == GATEWAY_SVC] first_gw = min(gw_idx) if gw_idx else len(rules) for h in EXCLUDED_HOSTS: fqdn = f"{h}.{base_domain}" hits = [i for i, r in enumerate(rules) if r.get("hostname") == fqdn] if not hits: bad.append(f"no rule for LAN/VPN-only host {fqdn}; it would fall through to the " f"wildcard and reach the Gateway") continue i = hits[0] if rules[i].get("service") != "http_status:404": bad.append(f"{fqdn} maps to {rules[i].get('service')!r}, not http_status:404") if i > first_gw: bad.append(f"{fqdn}'s 404 rule (#{i}) comes AFTER the first Gateway rule (#{first_gw}); " f"cloudflared is first-match, so the Gateway rule wins") gw_hosts = {rules[i].get("hostname") for i in gw_idx} if gw_hosts != {base_domain, f"*.{base_domain}"}: bad.append(f"the Gateway-bound hostnames are {sorted(map(str, gw_hosts))}; expected " f"exactly the apex and the wildcard") for i in gw_idx: if (rules[i].get("originRequest") or {}).get("matchSNItoHost") is not True: bad.append(f"Gateway rule #{i} lacks originRequest.matchSNItoHost: true — the Gateway " f"would not see the SNI that selects its wildcard certificate") last = rules[-1] if "hostname" in last or last.get("service") != "http_status:404": bad.append("the last rule is not a hostname-less http_status:404 catch-all") return bad # (1)-(3) the real config for v in evaluate(cfg): check(False, f"(1-3) {v}") check(str(cfg.get("credentials-file", "")).startswith("/etc/cloudflared/creds/"), "(3) credentials-file points into the mounted Secret volume") # (6) mutation self-test — the evaluator must flag each broken variant mutants = {} m = copy.deepcopy(cfg) m["ingress"].insert(0, {"hostname": f"k8s.{base_domain}", "service": "https://kubernetes.default.svc:443"}) mutants["API rule added"] = m m = copy.deepcopy(cfg) plex = next(r for r in m["ingress"] if r.get("hostname") == f"plex.{base_domain}") m["ingress"].remove(plex) m["ingress"].insert(len(m["ingress"]) - 1, plex) mutants["plex moved below the wildcard"] = m m = copy.deepcopy(cfg) m["ingress"] = m["ingress"][:-1] mutants["catch-all removed"] = m for name, mut in mutants.items(): check(evaluate(mut), f"(6) self-test: the evaluator did NOT flag the mutant '{name}' — the " f"check is vacuous") # (4) the egress policy np_docs = docs(COMP / "networkpolicy.yaml") cnps = [d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy"] check(len(cnps) == 1, f"(4) exactly one CiliumNetworkPolicy in networkpolicy.yaml (found {len(cnps)})") for cnp in cnps: for i, rule in enumerate(cnp["spec"].get("egress") or []): ents = set(rule.get("toEntities") or []) check(not (ents & FORBIDDEN_ENTITIES), f"(4) egress rule {i} admits {sorted(ents & FORBIDDEN_ENTITIES)} — that reaches the " f"Kubernetes API or the nodes; the API stays local-only") check(not (rule.get("toCIDR") or rule.get("toCIDRSet")), f"(4) egress rule {i} uses a CIDR — a CIDR can name the API VIP; use entities") for ep in rule.get("toEndpoints") or []: notin = [e for e in ep.get("matchExpressions") or [] if e.get("key") == "io.kubernetes.pod.namespace" and e.get("operator") == "NotIn"] missing = set(EXCLUDED_NAMESPACES) - set(notin[0]["values"] if notin else []) check(not missing, f"(4) backend selector in egress rule {i} does not exclude {sorted(missing)}") for d in np_docs: if d.get("kind") == "NetworkPolicy" and d["metadata"]["name"] == "default-deny": check(set(d["spec"].get("policyTypes", [])) == {"Ingress", "Egress"}, "(4) default-deny covers both directions") break else: check(False, "(4) networkpolicy.yaml ships no default-deny baseline for the namespace") # (5) Flux isolation fk = next(d for d in docs(ROOT / "clusters" / "base" / "cloudflared.yaml") if d.get("kind") == "Kustomization") check(fk["spec"].get("wait") is False, "(5) clusters/base/cloudflared.yaml must be wait: false — its pods are legitimately not-Ready " "until the tunnel is enrolled") check(fk["spec"].get("path") == "./platform/cloudflared", "(5) the Kustomization reconciles ./platform/cloudflared") check("cloudflared.yaml" in (ROOT / "clusters" / "base" / "kustomization.yaml").read_text(), "(5) clusters/base/kustomization.yaml lists cloudflared.yaml") check("cloudflared" not in (ROOT / "platform" / "kustomization.yaml").read_text(), "(5) platform/kustomization.yaml must not include cloudflared — that is the gating tier") for f in sorted((ROOT / "clusters").rglob("*.yaml")): for d in docs(f): for dep in (d.get("spec") or {}).get("dependsOn") or []: check(not (isinstance(dep, dict) and dep.get("name") == "cloudflared"), f"(5) {f.relative_to(ROOT)} {d.get('metadata', {}).get('name')} dependsOn " f"cloudflared — that hands an unenrolled tunnel the power to stall it") for env in ("dev", "prod"): text = (ROOT / "clusters" / env / "kustomization.yaml").read_text() check(re.search(r"name:\s*cloudflared\s*\n\s*namespace:\s*flux-system\s*\n\s*spec:\s*\n" r"\s*decryption:", text) is not None, f"(5) clusters/{env}/kustomization.yaml carries the cloudflared SOPS decryption patch") dep = next(d for d in docs(COMP / "deployment.yaml") if d.get("kind") == "Deployment") check(dep["spec"].get("replicas", 1) >= 2, "(5) the Deployment runs at least 2 replicas") check(dep["spec"]["template"]["metadata"]["labels"].get("homelab.keithjackson.dev/egress-class") == "internet", "(4) the pods are egress-class=internet — vpn would put the edge connections behind PIA") check(dep["spec"]["template"]["spec"].get("automountServiceAccountToken") is False, "(4) the pods mount no ServiceAccount token") if CHECKS < 25: FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated") if FAILURES: print(f"CHECK FAIL (cloudflared-ingress-rules): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr) for f in FAILURES: print(f" - {f}", file=sys.stderr) raise SystemExit(1) print(f"PASS (cloudflared-ingress-rules): {CHECKS} assertions") PY