home-network-k3s-impl/conformance/tests/l0-static/cloudflared-ingress-rules/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — cloudflared-ingress-rules
# Serves: R-NET-01 / R-NET-02 (default-deny; allow rules declarative and scoped), and the four
#         operator decisions of 2026-11-08 recorded in decisions/2026-11-08-cloudflare-tunnel.md.
#
# WHY THIS EXISTS. platform/cloudflared/config.yaml is the whole of what the internet can reach
# through the Cloudflare Tunnel. A one-line edit there — a rule pointing at the Kubernetes API, a
# LAN-only hostname moved below the wildcard, a dropped catch-all — publishes something the operator
# decided must never be public, and nothing in the cluster turns red: the tunnel stays connected and
# the Kustomization stays Ready. So the decisions are asserted here, on every gate run.
#
# GIVEN the repository as authored,
# THEN  (1) every ingress rule's `service` is `http_status:404` or EXACTLY the shared Gateway's
#           in-cluster Service, derived from platform/gateway/gateway.yaml (so a Gateway rename
#           fails here rather than in production) — which by construction excludes the
#           Kubernetes API, and a named deny-list says so explicitly when it is violated;
#       (2) each LAN/VPN-only hostname (plex, plex-vip, jackett, pihole, pihole-dns, semaphore,
#           qbittorrent, radarr, sonarr, headlamp, unifi, crafty-mc) has a rule mapping it to http_status:404 that comes BEFORE any
#           Gateway-bound rule (cloudflared matches first-rule-wins);
#       (3) the apex and the wildcard are the Gateway rules, with matchSNItoHost, and nothing
#           disables origin TLS verification; the last rule is a hostname-less 404 catch-all;
#       (4) the egress CiliumNetworkPolicy admits none of kube-apiserver/host/remote-node/
#           cluster/all and no CIDR rule, and its backend selector excludes the LAN-only
#           namespaces (the second, independent layer behind (2));
#       (5) the Flux tier is ISOLATED: its own Kustomization, wait:false, not listed in
#           platform/kustomization.yaml, nothing anywhere dependsOn it, and both overlays carry its
#           SOPS decryption patch — so a missing or invalid credential can stall nothing else;
#   AND (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config
#           (an API rule added, plex moved below the wildcard, the catch-all removed) and must
#           flag each one.
#
# NOT proven (static): that Cloudflare routes only the hostnames whose DNS points at the tunnel,
# that the Gateway's certificate validates from cloudflared, or that Envoy enforces (4) — those are
# runtime properties, verified by the operator per runbooks/cloudflare-tunnel.md.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
exec python3 - "$ROOT" <<'PY'
import copy
import re
import sys
from pathlib import Path

try:
    import yaml
except ImportError:                                   # pragma: no cover
    print("CHECK FAIL (cloudflared-ingress-rules): PyYAML is required", file=sys.stderr)
    raise SystemExit(1)

ROOT = Path(sys.argv[1])
FAILURES, CHECKS = [], 0


def check(cond, desc):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(desc)


def docs(path):
    return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]


COMP = ROOT / "platform" / "cloudflared"
CONFIG = COMP / "config.yaml"
if not CONFIG.is_file():
    print(f"CHECK FAIL (cloudflared-ingress-rules): {CONFIG.relative_to(ROOT)} is absent — the "
          "component this check guards was moved or deleted; re-point or delete this check in the "
          "SAME change", file=sys.stderr)
    raise SystemExit(1)

# The hostnames the operator decided never go through the tunnel (decisions 2026-11-08).
# headlamp: operator decision 2026-11-08 (LAN/VPN-only). qbittorrent/radarr/sonarr: catalog
# `exposure: internal`, like jackett/pihole/semaphore.
EXCLUDED_HOSTS = ("plex", "plex-vip", "jackett", "pihole", "pihole-dns", "semaphore",
                  "qbittorrent", "radarr", "sonarr", "headlamp", "unifi", "crafty-mc")
# Their Gateway-backend namespaces (unifi / crafty-mc are L4 VIPs, not Gateway backends).
EXCLUDED_NAMESPACES = ("plex", "jackett", "pihole", "semaphore", "qbittorrent", "radarr",
                       "sonarr", "headlamp")
FORBIDDEN_ENTITIES = {"kube-apiserver", "host", "remote-node", "cluster", "all"}
# Any of these in a `service` means a route toward the Kubernetes API (or a raw-TCP/SSH bastion).
API_MARKERS = ("kubernetes", "kube-apiserver", "apiserver", ":6443", "10.43.0.1", "tcp://",
               "ssh://", "unix:", "bastion")

# ── substitution: ${BASE_DOMAIN} from cluster-vars, the ONLY token allowed ──────────────────────
cv = next(d for d in docs(ROOT / "clusters" / "base" / "cluster-vars.yaml")
          if d.get("kind") == "ConfigMap")
base_domain = cv["data"]["BASE_DOMAIN"]
raw = CONFIG.read_text()
tokens = set(re.findall(r"\$\{([A-Za-z0-9_]+)\}", raw))
check(tokens == {"BASE_DOMAIN"},
      f"(0) config.yaml uses exactly the ${{BASE_DOMAIN}} token (found {sorted(tokens)}); any other "
      f"token is substituted from cluster-vars unseen by this check")
check(re.search(r"\$(?!\{BASE_DOMAIN\})", raw) is None,
      "(0) config.yaml contains a bare `$` — Flux envsubst would rewrite or empty it")
cfg = yaml.safe_load(raw.replace("${BASE_DOMAIN}", base_domain))

# ── the Gateway Service, derived rather than hard-coded ─────────────────────────────────────────
gw = next(d for d in docs(ROOT / "platform" / "gateway" / "gateway.yaml") if d.get("kind") == "Gateway")
GATEWAY_SVC = (f"https://cilium-gateway-{gw['metadata']['name']}."
               f"{gw['metadata']['namespace']}.svc.cluster.local:443")


def evaluate(cfg):
    """Return the list of rule violations for a parsed cloudflared config."""
    bad = []
    rules = cfg.get("ingress") or []
    if len(rules) < len(EXCLUDED_HOSTS) + 3:
        bad.append(f"only {len(rules)} ingress rules — the scan is vacuous")
        return bad
    for i, r in enumerate(rules):
        svc = str(r.get("service", ""))
        if any(m in svc.lower() for m in API_MARKERS):
            bad.append(f"rule {i} ({r.get('hostname', '<catch-all>')}) targets {svc!r}, which "
                       f"is a route toward the Kubernetes API / a raw bastion — the API stays "
                       f"local-only (operator decision 2026-11-08)")
        if svc not in ("http_status:404", GATEWAY_SVC):
            bad.append(f"rule {i} ({r.get('hostname', '<catch-all>')}) targets {svc!r}; only "
                       f"http_status:404 or {GATEWAY_SVC} are permitted")
        if (r.get("originRequest") or {}).get("noTLSVerify"):
            bad.append(f"rule {i} disables origin TLS verification (noTLSVerify)")
    if (cfg.get("originRequest") or {}).get("noTLSVerify"):
        bad.append("top-level originRequest disables origin TLS verification (noTLSVerify)")

    gw_idx = [i for i, r in enumerate(rules) if r.get("service") == GATEWAY_SVC]
    first_gw = min(gw_idx) if gw_idx else len(rules)
    for h in EXCLUDED_HOSTS:
        fqdn = f"{h}.{base_domain}"
        hits = [i for i, r in enumerate(rules) if r.get("hostname") == fqdn]
        if not hits:
            bad.append(f"no rule for LAN/VPN-only host {fqdn}; it would fall through to the "
                       f"wildcard and reach the Gateway")
            continue
        i = hits[0]
        if rules[i].get("service") != "http_status:404":
            bad.append(f"{fqdn} maps to {rules[i].get('service')!r}, not http_status:404")
        if i > first_gw:
            bad.append(f"{fqdn}'s 404 rule (#{i}) comes AFTER the first Gateway rule (#{first_gw}); "
                       f"cloudflared is first-match, so the Gateway rule wins")

    gw_hosts = {rules[i].get("hostname") for i in gw_idx}
    if gw_hosts != {base_domain, f"*.{base_domain}"}:
        bad.append(f"the Gateway-bound hostnames are {sorted(map(str, gw_hosts))}; expected "
                   f"exactly the apex and the wildcard")
    for i in gw_idx:
        if (rules[i].get("originRequest") or {}).get("matchSNItoHost") is not True:
            bad.append(f"Gateway rule #{i} lacks originRequest.matchSNItoHost: true — the Gateway "
                       f"would not see the SNI that selects its wildcard certificate")
    last = rules[-1]
    if "hostname" in last or last.get("service") != "http_status:404":
        bad.append("the last rule is not a hostname-less http_status:404 catch-all")
    return bad


# (1)-(3) the real config
for v in evaluate(cfg):
    check(False, f"(1-3) {v}")
check(str(cfg.get("credentials-file", "")).startswith("/etc/cloudflared/creds/"),
      "(3) credentials-file points into the mounted Secret volume")

# (6) mutation self-test — the evaluator must flag each broken variant
mutants = {}
m = copy.deepcopy(cfg)
m["ingress"].insert(0, {"hostname": f"k8s.{base_domain}",
                        "service": "https://kubernetes.default.svc:443"})
mutants["API rule added"] = m
m = copy.deepcopy(cfg)
plex = next(r for r in m["ingress"] if r.get("hostname") == f"plex.{base_domain}")
m["ingress"].remove(plex)
m["ingress"].insert(len(m["ingress"]) - 1, plex)
mutants["plex moved below the wildcard"] = m
m = copy.deepcopy(cfg)
m["ingress"] = m["ingress"][:-1]
mutants["catch-all removed"] = m
for name, mut in mutants.items():
    check(evaluate(mut), f"(6) self-test: the evaluator did NOT flag the mutant '{name}' — the "
                         f"check is vacuous")

# (4) the egress policy
np_docs = docs(COMP / "networkpolicy.yaml")
cnps = [d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy"]
check(len(cnps) == 1, f"(4) exactly one CiliumNetworkPolicy in networkpolicy.yaml (found {len(cnps)})")
for cnp in cnps:
    for i, rule in enumerate(cnp["spec"].get("egress") or []):
        ents = set(rule.get("toEntities") or [])
        check(not (ents & FORBIDDEN_ENTITIES),
              f"(4) egress rule {i} admits {sorted(ents & FORBIDDEN_ENTITIES)} — that reaches the "
              f"Kubernetes API or the nodes; the API stays local-only")
        check(not (rule.get("toCIDR") or rule.get("toCIDRSet")),
              f"(4) egress rule {i} uses a CIDR — a CIDR can name the API VIP; use entities")
        for ep in rule.get("toEndpoints") or []:
            notin = [e for e in ep.get("matchExpressions") or []
                     if e.get("key") == "io.kubernetes.pod.namespace" and e.get("operator") == "NotIn"]
            missing = set(EXCLUDED_NAMESPACES) - set(notin[0]["values"] if notin else [])
            check(not missing,
                  f"(4) backend selector in egress rule {i} does not exclude {sorted(missing)}")
for d in np_docs:
    if d.get("kind") == "NetworkPolicy" and d["metadata"]["name"] == "default-deny":
        check(set(d["spec"].get("policyTypes", [])) == {"Ingress", "Egress"},
              "(4) default-deny covers both directions")
        break
else:
    check(False, "(4) networkpolicy.yaml ships no default-deny baseline for the namespace")

# (5) Flux isolation
fk = next(d for d in docs(ROOT / "clusters" / "base" / "cloudflared.yaml")
          if d.get("kind") == "Kustomization")
check(fk["spec"].get("wait") is False,
      "(5) clusters/base/cloudflared.yaml must be wait: false — its pods are legitimately not-Ready "
      "until the tunnel is enrolled")
check(fk["spec"].get("path") == "./platform/cloudflared", "(5) the Kustomization reconciles ./platform/cloudflared")
check("cloudflared.yaml" in (ROOT / "clusters" / "base" / "kustomization.yaml").read_text(),
      "(5) clusters/base/kustomization.yaml lists cloudflared.yaml")
check("cloudflared" not in (ROOT / "platform" / "kustomization.yaml").read_text(),
      "(5) platform/kustomization.yaml must not include cloudflared — that is the gating tier")
for f in sorted((ROOT / "clusters").rglob("*.yaml")):
    for d in docs(f):
        for dep in (d.get("spec") or {}).get("dependsOn") or []:
            check(not (isinstance(dep, dict) and dep.get("name") == "cloudflared"),
                  f"(5) {f.relative_to(ROOT)} {d.get('metadata', {}).get('name')} dependsOn "
                  f"cloudflared — that hands an unenrolled tunnel the power to stall it")
for env in ("dev", "prod"):
    text = (ROOT / "clusters" / env / "kustomization.yaml").read_text()
    check(re.search(r"name:\s*cloudflared\s*\n\s*namespace:\s*flux-system\s*\n\s*spec:\s*\n"
                    r"\s*decryption:", text) is not None,
          f"(5) clusters/{env}/kustomization.yaml carries the cloudflared SOPS decryption patch")

dep = next(d for d in docs(COMP / "deployment.yaml") if d.get("kind") == "Deployment")
check(dep["spec"].get("replicas", 1) >= 2, "(5) the Deployment runs at least 2 replicas")
check(dep["spec"]["template"]["metadata"]["labels"].get("homelab.keithjackson.dev/egress-class")
      == "internet",
      "(4) the pods are egress-class=internet — vpn would put the edge connections behind PIA")
check(dep["spec"]["template"]["spec"].get("automountServiceAccountToken") is False,
      "(4) the pods mount no ServiceAccount token")

if CHECKS < 25:
    FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated")
if FAILURES:
    print(f"CHECK FAIL (cloudflared-ingress-rules): {len(FAILURES)} of {CHECKS} assertions failed",
          file=sys.stderr)
    for f in FAILURES:
        print(f"  - {f}", file=sys.stderr)
    raise SystemExit(1)
print(f"PASS (cloudflared-ingress-rules): {CHECKS} assertions")
PY