jakman85andClaude Opus 5.5
b85d1e3ed7
Hand-authored platform/vuln-scanner (non-gating Flux tier, digest-pinned Trivy 0.76.1 and python 3.14.9-alpine, read-only RBAC, per-pod oauth2-proxy, internal-only cve.<domain>), five alerts with promtool tests, R-VULN-01..09 registry entries, and vuln-* L0 checks with mutation self-tests. src/ and the bot workflow are integrated separately. Co-Authored-By: Claude Opus 5.5 <[email protected]>
322 lines
18 KiB
Bash
Executable File
322 lines
18 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# L0 static check (offline, no cluster) — cloudflared-ingress-rules
|
|
# Serves: R-NET-01 / R-NET-02 (default-deny; allow rules declarative and scoped), and the four
|
|
# operator decisions of 2026-11-08 recorded in decisions/2026-11-08-cloudflare-tunnel.md.
|
|
#
|
|
# WHY THIS EXISTS. platform/cloudflared/config.yaml is the whole of what the internet can reach
|
|
# through the Cloudflare Tunnel. A one-line edit there — a rule pointing at the Kubernetes API, a
|
|
# LAN-only hostname moved below the wildcard, a dropped catch-all — publishes something the operator
|
|
# decided must never be public, and nothing in the cluster turns red: the tunnel stays connected and
|
|
# the Kustomization stays Ready. So the decisions are asserted here, on every gate run.
|
|
#
|
|
# GIVEN the repository as authored,
|
|
# THEN (1) every ingress rule's `service` is `http_status:404` or EXACTLY the shared Gateway's
|
|
# in-cluster Service, derived from platform/gateway/gateway.yaml (so a Gateway rename
|
|
# fails here rather than in production) — which by construction excludes the
|
|
# Kubernetes API, and a named deny-list says so explicitly when it is violated;
|
|
# (2) each LAN/VPN-only hostname (plex, plex-vip, jackett, pihole, pihole-dns, semaphore,
|
|
# qbittorrent, radarr, sonarr, headlamp, unifi, crafty-mc) has a rule mapping it to http_status:404 that comes BEFORE any
|
|
# Gateway-bound rule (cloudflared matches first-rule-wins);
|
|
# (3) the apex and the wildcard are the Gateway rules, with matchSNItoHost, and nothing
|
|
# disables origin TLS verification; the last rule is a hostname-less 404 catch-all;
|
|
# (4) the egress CiliumNetworkPolicy admits none of kube-apiserver/host/remote-node/
|
|
# cluster/all and no CIDR rule, and its backend selector excludes the LAN-only
|
|
# namespaces (the second, independent layer behind (2));
|
|
# (5) the Flux tier is ISOLATED: its own Kustomization, wait:false, not listed in
|
|
# platform/kustomization.yaml, nothing anywhere dependsOn it, and both overlays carry its
|
|
# SOPS decryption patch — so a missing or invalid credential can stall nothing else;
|
|
# (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config
|
|
# (an API rule added, plex moved below the wildcard, the catch-all removed) and must
|
|
# flag each one;
|
|
# AND (7) every Secret volume the container mounts — the tunnel credentials above all — is
|
|
# READABLE by the container's run-as identity, modelled the way the kubelet lays it out:
|
|
# files owned by uid 0, group-owned by fsGroup if set (else gid 0), with the item mode or
|
|
# defaultMode (API default 0644). The identity is the container securityContext over
|
|
# the pod's; the process's groups are runAsGroup + fsGroup + supplementalGroups. An
|
|
# unset runAsUser is a FAIL, not a pass: the image default cannot be read offline. The
|
|
# model is deliberately conservative — it does not credit the kubelet's habit of
|
|
# OR-ing group-read into fsGroup-owned volumes, so 0400+fsGroup fails here though it
|
|
# would work. Self-tested by mutation (fsGroup removed; mode 0400; runAsUser unset).
|
|
# This is the 2026-11-08 CrashLoop: runAsUser 65532, no fsGroup, 0440 root:root.
|
|
#
|
|
# NOT proven (static): that Cloudflare routes only the hostnames whose DNS points at the tunnel,
|
|
# that the Gateway's certificate validates from cloudflared, or that Envoy enforces (4) — those are
|
|
# runtime properties, verified by the operator per runbooks/cloudflare-tunnel.md.
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
|
exec python3 - "$ROOT" <<'PY'
|
|
import copy
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError: # pragma: no cover
|
|
print("CHECK FAIL (cloudflared-ingress-rules): PyYAML is required", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
ROOT = Path(sys.argv[1])
|
|
FAILURES, CHECKS = [], 0
|
|
|
|
|
|
def check(cond, desc):
|
|
global CHECKS
|
|
CHECKS += 1
|
|
if not cond:
|
|
FAILURES.append(desc)
|
|
|
|
|
|
def docs(path):
|
|
return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]
|
|
|
|
|
|
COMP = ROOT / "platform" / "cloudflared"
|
|
CONFIG = COMP / "config.yaml"
|
|
if not CONFIG.is_file():
|
|
print(f"CHECK FAIL (cloudflared-ingress-rules): {CONFIG.relative_to(ROOT)} is absent — the "
|
|
"component this check guards was moved or deleted; re-point or delete this check in the "
|
|
"SAME change", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
# The hostnames the operator decided never go through the tunnel (decisions 2026-11-08).
|
|
# headlamp: operator decision 2026-11-08 (LAN/VPN-only). qbittorrent/radarr/sonarr: catalog
|
|
# `exposure: internal`, like jackett/pihole/semaphore.
|
|
EXCLUDED_HOSTS = ("plex", "plex-vip", "jackett", "pihole", "pihole-dns", "semaphore",
|
|
"qbittorrent", "radarr", "sonarr", "headlamp", "cve", "unifi", "crafty-mc")
|
|
# Their Gateway-backend namespaces (unifi / crafty-mc are L4 VIPs, not Gateway backends).
|
|
EXCLUDED_NAMESPACES = ("plex", "jackett", "pihole", "semaphore", "qbittorrent", "radarr",
|
|
"sonarr", "headlamp", "vuln-scanner")
|
|
FORBIDDEN_ENTITIES = {"kube-apiserver", "host", "remote-node", "cluster", "all"}
|
|
# Any of these in a `service` means a route toward the Kubernetes API (or a raw-TCP/SSH bastion).
|
|
API_MARKERS = ("kubernetes", "kube-apiserver", "apiserver", ":6443", "10.43.0.1", "tcp://",
|
|
"ssh://", "unix:", "bastion")
|
|
|
|
# ── substitution: ${BASE_DOMAIN} from cluster-vars, the ONLY token allowed ──────────────────────
|
|
cv = next(d for d in docs(ROOT / "clusters" / "base" / "cluster-vars.yaml")
|
|
if d.get("kind") == "ConfigMap")
|
|
base_domain = cv["data"]["BASE_DOMAIN"]
|
|
raw = CONFIG.read_text()
|
|
tokens = set(re.findall(r"\$\{([A-Za-z0-9_]+)\}", raw))
|
|
check(tokens == {"BASE_DOMAIN"},
|
|
f"(0) config.yaml uses exactly the ${{BASE_DOMAIN}} token (found {sorted(tokens)}); any other "
|
|
f"token is substituted from cluster-vars unseen by this check")
|
|
check(re.search(r"\$(?!\{BASE_DOMAIN\})", raw) is None,
|
|
"(0) config.yaml contains a bare `$` — Flux envsubst would rewrite or empty it")
|
|
cfg = yaml.safe_load(raw.replace("${BASE_DOMAIN}", base_domain))
|
|
|
|
# ── the Gateway Service, derived rather than hard-coded ─────────────────────────────────────────
|
|
gw = next(d for d in docs(ROOT / "platform" / "gateway" / "gateway.yaml") if d.get("kind") == "Gateway")
|
|
GATEWAY_SVC = (f"https://cilium-gateway-{gw['metadata']['name']}."
|
|
f"{gw['metadata']['namespace']}.svc.cluster.local:443")
|
|
|
|
|
|
def evaluate(cfg):
|
|
"""Return the list of rule violations for a parsed cloudflared config."""
|
|
bad = []
|
|
rules = cfg.get("ingress") or []
|
|
if len(rules) < len(EXCLUDED_HOSTS) + 3:
|
|
bad.append(f"only {len(rules)} ingress rules — the scan is vacuous")
|
|
return bad
|
|
for i, r in enumerate(rules):
|
|
svc = str(r.get("service", ""))
|
|
if any(m in svc.lower() for m in API_MARKERS):
|
|
bad.append(f"rule {i} ({r.get('hostname', '<catch-all>')}) targets {svc!r}, which "
|
|
f"is a route toward the Kubernetes API / a raw bastion — the API stays "
|
|
f"local-only (operator decision 2026-11-08)")
|
|
if svc not in ("http_status:404", GATEWAY_SVC):
|
|
bad.append(f"rule {i} ({r.get('hostname', '<catch-all>')}) targets {svc!r}; only "
|
|
f"http_status:404 or {GATEWAY_SVC} are permitted")
|
|
if (r.get("originRequest") or {}).get("noTLSVerify"):
|
|
bad.append(f"rule {i} disables origin TLS verification (noTLSVerify)")
|
|
if (cfg.get("originRequest") or {}).get("noTLSVerify"):
|
|
bad.append("top-level originRequest disables origin TLS verification (noTLSVerify)")
|
|
|
|
gw_idx = [i for i, r in enumerate(rules) if r.get("service") == GATEWAY_SVC]
|
|
first_gw = min(gw_idx) if gw_idx else len(rules)
|
|
for h in EXCLUDED_HOSTS:
|
|
fqdn = f"{h}.{base_domain}"
|
|
hits = [i for i, r in enumerate(rules) if r.get("hostname") == fqdn]
|
|
if not hits:
|
|
bad.append(f"no rule for LAN/VPN-only host {fqdn}; it would fall through to the "
|
|
f"wildcard and reach the Gateway")
|
|
continue
|
|
i = hits[0]
|
|
if rules[i].get("service") != "http_status:404":
|
|
bad.append(f"{fqdn} maps to {rules[i].get('service')!r}, not http_status:404")
|
|
if i > first_gw:
|
|
bad.append(f"{fqdn}'s 404 rule (#{i}) comes AFTER the first Gateway rule (#{first_gw}); "
|
|
f"cloudflared is first-match, so the Gateway rule wins")
|
|
|
|
gw_hosts = {rules[i].get("hostname") for i in gw_idx}
|
|
if gw_hosts != {base_domain, f"*.{base_domain}"}:
|
|
bad.append(f"the Gateway-bound hostnames are {sorted(map(str, gw_hosts))}; expected "
|
|
f"exactly the apex and the wildcard")
|
|
for i in gw_idx:
|
|
if (rules[i].get("originRequest") or {}).get("matchSNItoHost") is not True:
|
|
bad.append(f"Gateway rule #{i} lacks originRequest.matchSNItoHost: true — the Gateway "
|
|
f"would not see the SNI that selects its wildcard certificate")
|
|
last = rules[-1]
|
|
if "hostname" in last or last.get("service") != "http_status:404":
|
|
bad.append("the last rule is not a hostname-less http_status:404 catch-all")
|
|
return bad
|
|
|
|
|
|
# (1)-(3) the real config
|
|
for v in evaluate(cfg):
|
|
check(False, f"(1-3) {v}")
|
|
check(str(cfg.get("credentials-file", "")).startswith("/etc/cloudflared/creds/"),
|
|
"(3) credentials-file points into the mounted Secret volume")
|
|
|
|
# (6) mutation self-test — the evaluator must flag each broken variant
|
|
mutants = {}
|
|
m = copy.deepcopy(cfg)
|
|
m["ingress"].insert(0, {"hostname": f"k8s.{base_domain}",
|
|
"service": "https://kubernetes.default.svc:443"})
|
|
mutants["API rule added"] = m
|
|
m = copy.deepcopy(cfg)
|
|
plex = next(r for r in m["ingress"] if r.get("hostname") == f"plex.{base_domain}")
|
|
m["ingress"].remove(plex)
|
|
m["ingress"].insert(len(m["ingress"]) - 1, plex)
|
|
mutants["plex moved below the wildcard"] = m
|
|
m = copy.deepcopy(cfg)
|
|
m["ingress"] = m["ingress"][:-1]
|
|
mutants["catch-all removed"] = m
|
|
for name, mut in mutants.items():
|
|
check(evaluate(mut), f"(6) self-test: the evaluator did NOT flag the mutant '{name}' — the "
|
|
f"check is vacuous")
|
|
|
|
# (4) the egress policy
|
|
np_docs = docs(COMP / "networkpolicy.yaml")
|
|
cnps = [d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy"]
|
|
check(len(cnps) == 1, f"(4) exactly one CiliumNetworkPolicy in networkpolicy.yaml (found {len(cnps)})")
|
|
for cnp in cnps:
|
|
for i, rule in enumerate(cnp["spec"].get("egress") or []):
|
|
ents = set(rule.get("toEntities") or [])
|
|
check(not (ents & FORBIDDEN_ENTITIES),
|
|
f"(4) egress rule {i} admits {sorted(ents & FORBIDDEN_ENTITIES)} — that reaches the "
|
|
f"Kubernetes API or the nodes; the API stays local-only")
|
|
check(not (rule.get("toCIDR") or rule.get("toCIDRSet")),
|
|
f"(4) egress rule {i} uses a CIDR — a CIDR can name the API VIP; use entities")
|
|
for ep in rule.get("toEndpoints") or []:
|
|
notin = [e for e in ep.get("matchExpressions") or []
|
|
if e.get("key") == "io.kubernetes.pod.namespace" and e.get("operator") == "NotIn"]
|
|
missing = set(EXCLUDED_NAMESPACES) - set(notin[0]["values"] if notin else [])
|
|
check(not missing,
|
|
f"(4) backend selector in egress rule {i} does not exclude {sorted(missing)}")
|
|
for d in np_docs:
|
|
if d.get("kind") == "NetworkPolicy" and d["metadata"]["name"] == "default-deny":
|
|
check(set(d["spec"].get("policyTypes", [])) == {"Ingress", "Egress"},
|
|
"(4) default-deny covers both directions")
|
|
break
|
|
else:
|
|
check(False, "(4) networkpolicy.yaml ships no default-deny baseline for the namespace")
|
|
|
|
# (5) Flux isolation
|
|
fk = next(d for d in docs(ROOT / "clusters" / "base" / "cloudflared.yaml")
|
|
if d.get("kind") == "Kustomization")
|
|
check(fk["spec"].get("wait") is False,
|
|
"(5) clusters/base/cloudflared.yaml must be wait: false — its pods are legitimately not-Ready "
|
|
"until the tunnel is enrolled")
|
|
check(fk["spec"].get("path") == "./platform/cloudflared", "(5) the Kustomization reconciles ./platform/cloudflared")
|
|
check("cloudflared.yaml" in (ROOT / "clusters" / "base" / "kustomization.yaml").read_text(),
|
|
"(5) clusters/base/kustomization.yaml lists cloudflared.yaml")
|
|
check("cloudflared" not in (ROOT / "platform" / "kustomization.yaml").read_text(),
|
|
"(5) platform/kustomization.yaml must not include cloudflared — that is the gating tier")
|
|
for f in sorted((ROOT / "clusters").rglob("*.yaml")):
|
|
for d in docs(f):
|
|
for dep in (d.get("spec") or {}).get("dependsOn") or []:
|
|
check(not (isinstance(dep, dict) and dep.get("name") == "cloudflared"),
|
|
f"(5) {f.relative_to(ROOT)} {d.get('metadata', {}).get('name')} dependsOn "
|
|
f"cloudflared — that hands an unenrolled tunnel the power to stall it")
|
|
for env in ("dev", "prod"):
|
|
text = (ROOT / "clusters" / env / "kustomization.yaml").read_text()
|
|
check(re.search(r"name:\s*cloudflared\s*\n\s*namespace:\s*flux-system\s*\n\s*spec:\s*\n"
|
|
r"\s*decryption:", text) is not None,
|
|
f"(5) clusters/{env}/kustomization.yaml carries the cloudflared SOPS decryption patch")
|
|
|
|
dep = next(d for d in docs(COMP / "deployment.yaml") if d.get("kind") == "Deployment")
|
|
check(dep["spec"].get("replicas", 1) >= 2, "(5) the Deployment runs at least 2 replicas")
|
|
check(dep["spec"]["template"]["metadata"]["labels"].get("homelab.keithjackson.dev/egress-class")
|
|
== "internet",
|
|
"(4) the pods are egress-class=internet — vpn would put the edge connections behind PIA")
|
|
check(dep["spec"]["template"]["spec"].get("automountServiceAccountToken") is False,
|
|
"(4) the pods mount no ServiceAccount token")
|
|
|
|
# (7) the credentials Secret is readable by the run-as identity
|
|
def unreadable_secret_volumes(pod_spec):
|
|
"""Return a description per Secret volume the containers mount but cannot read."""
|
|
bad = []
|
|
psc = pod_spec.get("securityContext") or {}
|
|
vols = {v["name"]: v for v in pod_spec.get("volumes") or [] if "secret" in v}
|
|
for c in (pod_spec.get("initContainers") or []) + (pod_spec.get("containers") or []):
|
|
csc = c.get("securityContext") or {}
|
|
uid = csc.get("runAsUser", psc.get("runAsUser"))
|
|
gid = csc.get("runAsGroup", psc.get("runAsGroup"))
|
|
fs_group = psc.get("fsGroup")
|
|
groups = {g for g in [gid, fs_group, *(psc.get("supplementalGroups") or [])]
|
|
if g is not None}
|
|
for vm in c.get("volumeMounts") or []:
|
|
v = vols.get(vm["name"])
|
|
if v is None:
|
|
continue
|
|
sec = v["secret"]
|
|
if uid is None:
|
|
bad.append(f"container {c['name']} mounts Secret volume {vm['name']} but sets no "
|
|
f"runAsUser (pod or container) — readability cannot be proven offline")
|
|
continue
|
|
default = sec.get("defaultMode", 0o644)
|
|
modes = [it.get("mode", default) for it in sec.get("items") or []] or [default]
|
|
owner_gid = fs_group if fs_group is not None else 0
|
|
for mode in modes:
|
|
readable = ((uid == 0 and mode & 0o400) or (owner_gid in groups and mode & 0o040)
|
|
or mode & 0o004)
|
|
if not readable:
|
|
bad.append(f"container {c['name']} (uid {uid}, groups {sorted(groups)}) cannot "
|
|
f"read Secret volume {vm['name']} ({sec.get('secretName')}): files "
|
|
f"are 0:{owner_gid} mode {mode:04o} — set pod fsGroup to a group "
|
|
f"the container runs with, keeping 0440")
|
|
return bad
|
|
|
|
|
|
pod_spec = dep["spec"]["template"]["spec"]
|
|
creds_vol = [v for v in pod_spec.get("volumes") or []
|
|
if (v.get("secret") or {}).get("secretName") == "cloudflared-credentials"]
|
|
check(len(creds_vol) == 1, "(7) the Deployment mounts Secret cloudflared-credentials as a volume")
|
|
for v in unreadable_secret_volumes(pod_spec):
|
|
check(False, f"(7) {v}")
|
|
sec_mutants = {}
|
|
m = copy.deepcopy(pod_spec)
|
|
m["securityContext"].pop("fsGroup", None)
|
|
m["securityContext"].pop("supplementalGroups", None)
|
|
sec_mutants["fsGroup removed"] = m
|
|
m = copy.deepcopy(pod_spec)
|
|
for v in m["volumes"]:
|
|
if "secret" in v:
|
|
v["secret"]["defaultMode"] = 0o400
|
|
for it in v["secret"].get("items") or []:
|
|
it["mode"] = 0o400
|
|
sec_mutants["mode 0400"] = m
|
|
m = copy.deepcopy(pod_spec)
|
|
m["securityContext"].pop("runAsUser", None)
|
|
for c in m["containers"]:
|
|
(c.get("securityContext") or {}).pop("runAsUser", None)
|
|
sec_mutants["runAsUser unset"] = m
|
|
for name, mut in sec_mutants.items():
|
|
check(unreadable_secret_volumes(mut),
|
|
f"(7) self-test: the readability model did NOT flag the mutant '{name}' — vacuous")
|
|
|
|
if CHECKS < 25:
|
|
FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated")
|
|
if FAILURES:
|
|
print(f"CHECK FAIL (cloudflared-ingress-rules): {len(FAILURES)} of {CHECKS} assertions failed",
|
|
file=sys.stderr)
|
|
for f in FAILURES:
|
|
print(f" - {f}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
print(f"PASS (cloudflared-ingress-rules): {CHECKS} assertions")
|
|
PY
|