#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — c2-root-surface # # Serves (spec docs/00-vision-and-scope.md; detail in docs/12-command-center-ui.md): # # R-C2-01 "The root `home.keithjackson.dev` MUST serve a UI that surfaces the # services running in the cluster and links to them." # R-C2-02 "The C2 UI MUST surface platform health (cluster, storage, GitOps, # alerts) and provide controlled operational actions where safe." # # WHY THIS IS AN L0 STATIC CHECK AND NOT AN L2 PROBE # -------------------------------------------------- # The REACHABILITY half of R-C2-01 is already exercised against a live cluster: # l2-behavioral/gateway-ingress-reachable dials the Gateway and l2-behavioral/trusted-tls # verifies the certificate it serves. What no live probe can decide is the half the # requirement is actually about — whether the UI at the root surfaces THE SERVICES RUNNING # IN THE CLUSTER. Deciding that is a comparison of two SETS, and both are repository facts: # what git declares the cluster serves (every in-repo HTTPRoute hostname, every # hostname-publishing L4 Service, every catalog ServiceDefinition) against what the # dashboard's ConfigMap links. A probe of the root gets HTTP 200 from a dashboard that hides # half the estate and cannot tell the difference. # # It is also the assertion that ROTS, in the direction nothing else notices: onboard a # service, forget the dashboard, and every cluster test stays green while the operator's # front door hides a running service. That is not a styling preference — it is exactly the # R-C2-01 failure mode, and it was a LIVE defect on main when this check was written # (platform/crafty served crafty.${BASE_DOMAIN} and nothing on the root linked to it). # # THE RULE THIS IMPLEMENTS, in one sentence: every home-zone hostname the repository routes # over HTTP must be linked from the apex dashboard's configuration, every home-zone link the # apex dashboard offers must resolve to something the repository actually serves, and the two # R-C2-02 surfaces (health data + a per-user-authorized console) must be linked, reconciled # and backed by real alert rules. # # GIVEN the repository as git tracks it, # WHEN the apex ingress chain, the dashboard's declared link set, every in-repo HTTPRoute, # every hostname-publishing L4 Service, the versioned catalog, the platform-health # alert rules and the console's RBAC posture are read and compared, # THEN — R-C2-01 — # (0) the INPUT SURFACE is intact: every Flux-reconciled path lies inside the roots # this check scans, and every manifest in them parses. A root this check does not # scan, or a file it cannot parse, is a service it cannot see — which would # silently weaken every assertion below; # (1) EXACTLY ONE HTTPRoute claims the bare apex (an ambiguous root is not a root), # and it belongs to the landing-dashboard component; # (2) that route pins by sectionName the ONE Gateway listener whose hostname is the # bare apex, and that listener is HTTPS — the wildcard `*.${BASE_DOMAIN}` listener # does NOT intersect the bare apex, so without the sectionName the root binds # nothing and serves nothing (asserted against the wildcard listener's existence, # so the claim is proven rather than asserted from prose); # (3) the apex is a SAN on the Certificate whose Secret that listener terminates with; # (4) the dashboard is RECONCILED, not merely authored: one Flux Kustomization points # at its path and is listed in the base entrypoint set; # (5) that Kustomization carries postBuild.substituteFrom, because the dashboard's # hrefs and its route's hostname are written as `${BASE_DOMAIN}` tokens — without # substitution the apex route binds a literal `${BASE_DOMAIN}` and EVERY tile on # the root links to a nonexistent host, while Flux still reports Ready; # (6) the component's own kustomization ships config + workload + Service + route; # (7) the dashboard is a service CATALOG: grouped entries carrying hrefs; # (8) FORWARD — every home-zone hostname an in-repo HTTPRoute serves, other than the # apex itself, is LINKED from the root. This is the rot-catcher; # (9) FORWARD — every catalog `kind: container` ServiceDefinition whose declared # ingress host is routed is likewise linked, tying the C2 to the VERSIONED # onboarding contract and not only to whatever is rendered today; # (10) REVERSE — every home-zone link the root offers resolves to something this repo # actually serves: an HTTPRoute (L7) or a hostname-publishing LoadBalancer Service # (L4). A dead tile is a link the root offers to nothing; # (11) the off-cluster escape hatch is not a blanket one: every name declared in the # ConfigMap's `off-cluster-links` annotation is a specific home-zone hostname and # none of them shadows a hostname the repo DOES serve; # (12) no href names a LITERAL base-domain host. Every hostname is the `${BASE_DOMAIN}` # token, so prod's root cannot link dev's services — the 2026-09-04 incident class # (two clusters authored to describe the same names), applied to the front door; # — R-C2-02 — # (13) both health surfaces docs/12 names are linked: the Kubernetes console # (cluster/storage/GitOps state) and the dashboards/alerts surface; # (14) both are RECONCILED by Flux, so the links lead to something running; # (15) the health DATA those surfaces render exists as code — PrometheusRule groups for # each axis R-C2-02 enumerates (cluster/nodes, storage, GitOps reconciliation) # plus certificate expiry — and those rule files are reconciled; # (16) actions are CONTROLLED, not cluster-admin-in-a-browser (docs/12: "never a raw # cluster-admin kubeconfig in a browser"): the console chart is explicitly told NOT # to bind its ServiceAccount to a ClusterRole, so there is no shared privileged # identity to act as; # (17) actions are instead PER-USER: the console is configured for native OIDC with # credentials from a named (SOPS) Secret, not inline; # (18) cluster rights arrive through OIDC GROUPS at MORE THAN ONE privilege level (an # admin group and a lesser one) — least-privilege, not one tier for everybody; # (19) the C2 root DECLARES the authenticated-group posture docs/12 requires, on its own # route (declared, NOT enforced — see the gap note below). # # WHAT THIS DOES *NOT* PROVE — honest gaps, the same ones the Test declaration names: # - That the apex answers HTTP 200, or that any tile's status ping is green. That is L2; the # Gateway/TLS half is covered under R-DNS-01 / R-TLS-01. # - That the C2 is access-CONTROLLED in fact. (19) reads a DECLARED posture, because the # Gateway ext-authz attachment that would ENFORCE forward-auth is not built anywhere in # this repo (platform/homepage/httproute.yaml says so in its own header, and every # forward-auth app shares the gap). Declared, not enforced, and labelled as such in the # assertion text itself. # - That an operational action in the console succeeds, or that it is audited. (16)-(18) # prove there is no shared cluster-admin identity to act with and that rights arrive # per-user; the audit trail is a runtime property. # - Whether a tile is USEFUL. Homepage service-widget API keys are a deliberate follow-up # (platform/homepage/configmap.yaml); link presence is what R-C2-01 asks for. # - That an L4-ONLY endpoint is surfaced. A raw TCP/UDP endpoint (a Minecraft game port) has # no URL a browser can follow, so the FORWARD direction quantifies over HTTP routes; # L4-published hostnames are listed as ADVISORIES and are honoured in the REVERSE # direction (that is how the UniFi tile, an L4 LoadBalancer on :11443, resolves). # - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06) and the # requirement text above is quoted, not parsed. # # STATUS: EXECUTABLE. Pure offline: `git ls-files` + PyYAML (the dependency ten neighbouring # l0-static checks already take). No cluster, no network, no spec repo. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root PYTHON="${PYTHON:-python3}" # Honest SKIP (never green, and rc 3 is UNWIRED for a scope-declaring test) if the landing # dashboard component is absent entirely — there is then no C2 root to audit, which is a # report about the tree, not a pass. if [ ! -f "${ROOT}/platform/homepage/configmap.yaml" ]; then echo "check SKIPPED: platform/homepage/configmap.yaml absent — no C2 landing dashboard to audit" >&2 exit 3 fi cd "${ROOT}" "${PYTHON}" - "${ROOT}" <<'PY' """Static audit: the C2 root surfaces every service the repository serves, plus health. Every parse contract is documented at its parser. PyYAML rather than regex because the subjects are genuinely structured: multi-document apps/ bundles, and a Homepage config that is nested YAML *inside* a ConfigMap string value. """ from __future__ import annotations import re import subprocess import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree FAILURES: list[str] = [] ADVISORIES: list[str] = [] CHECKS = 0 def check(cond, msg): global CHECKS CHECKS += 1 if cond: print(f" ok {msg}") else: print(f" FAIL {msg}", file=sys.stderr) FAILURES.append(msg) def read(p: Path) -> str: try: return p.read_text() except (OSError, UnicodeDecodeError): return "" # ============================================================ the hostname vocabulary # BASE_DOMAIN is a cluster-vars knob (R-ENV-02): every public hostname the cluster serves is # `.${BASE_DOMAIN}`, and dev/prod differ ONLY in that value. So a hostname in this repo # is canonically the TOKEN form, and the literal value below exists only to be FORBIDDEN in # (12): a literal is how prod came to be authored to serve dev's hostnames (2026-09-04, # clusters/base/cluster-vars.yaml's own header). TOKEN = "${BASE_DOMAIN}" _vars = yaml.safe_load(read(ROOT / "clusters" / "base" / "cluster-vars.yaml")) or {} LITERAL_BASE = ((_vars.get("data") or {}).get("BASE_DOMAIN") or "").strip() def strip_port(host: str) -> str: return host.split(":", 1)[0] def zone_label(host: str): """The sub-label of a home-zone hostname, "" for the bare apex, or None if not in zone. Accepts only the TOKEN form; a literal-base hostname returns None here and is caught separately by (12), so a literal can never silently satisfy a set comparison. """ h = strip_port(host) if h == TOKEN: return "" if h.endswith("." + TOKEN): return h[: -len("." + TOKEN)] return None def is_literal_base(host: str) -> bool: h = strip_port(host) return bool(LITERAL_BASE) and (h == LITERAL_BASE or h.endswith("." + LITERAL_BASE)) # ============================================================ (0) the input surface # DELIVERY ROOTS — the subtrees Flux reconciles, plus catalog/ (the versioned source the # apps/ manifests are rendered FROM). conformance/ is deliberately EXCLUDED: its fixtures and # goldens declare synthetic HTTPRoutes (`paas-demo.…`, `gw-declared-conformance.…`) that are # test data, not services the cluster serves, and counting them would make (8) assert that # the operator's dashboard links this harness's fixtures. DELIVERY_ROOTS = ("apps", "platform", "platform-config", "platform-config-cdi", "runbooks", "clusters", "bootstrap", "catalog") # NARROW EXEMPTION to (0), for reconciled trees that deliberately serve NOTHING. # `conformance/incluster` (the L2/L3 runner + CI-trigger identities, R-SEC-02) is reconciled # by clusters/base/conformance.yaml but contains only a Namespace, ServiceAccounts, RBAC and # an admission-bounded Job template. Adding "conformance" to DELIVERY_ROOTS instead would be # wrong for the reason stated above: it would sweep in this harness's synthetic fixture # routes and make (8) demand the operator's dashboard link them. # # THE EXEMPTION POLICES ITSELF. An unchecked allowlist is how (0) would rot into a fail-open — # add a tree here, later put an HTTPRoute in it, and the forward direction goes quietly green # on a hidden service. So each exempted tree is ASSERTED below to declare no route and no # hostname-publishing Service. If that ever stops being true the check fails and the tree must # become a real delivery root. NON_SERVING_RECONCILED = ("conformance/incluster",) def tracked_yaml() -> list[Path]: out = subprocess.run(["git", "ls-files", "-z", "--", "*.yaml", "*.yml"], cwd=ROOT, capture_output=True, text=True, check=True) return [ROOT / p for p in out.stdout.split("\0") if p] ALL_TRACKED = tracked_yaml() DELIVERY = [p for p in ALL_TRACKED if p.relative_to(ROOT).parts[0] in DELIVERY_ROOTS] parse_failed: list[str] = [] DOCS: dict[Path, list[dict]] = {} for p in DELIVERY: try: DOCS[p] = [d for d in yaml.safe_load_all(read(p)) if isinstance(d, dict)] except yaml.YAMLError: DOCS[p] = [] parse_failed.append(str(p.relative_to(ROOT))) def docs_all(): for p, ds in DOCS.items(): for d in ds: yield p, d check(len(DELIVERY) >= 100, f"(0) the delivery tree is present and scanned ({len(DELIVERY)} tracked manifest(s) " f"under {'/, '.join(DELIVERY_ROOTS)}/)") check(not parse_failed, f"(0) every manifest in the scanned roots PARSES — an unparseable one is a service " f"this check cannot see, so it would weaken every set comparison below " f"{parse_failed if parse_failed else ''}") # Every Flux-reconciled path must lie inside a root this check scans. This is the guard on # the guard: if a future tier reconciles, say, ./services, the forward direction would stop # seeing its routes and would go quietly green on a dashboard that hides all of them. flux_ks = [(p, d) for p, d in docs_all() if d.get("kind") == "Kustomization" and str(d.get("apiVersion", "")).startswith("kustomize.toolkit.fluxcd.io")] # A RECONCILING Kustomization is one carrying a sourceRef. The pathless documents under # bootstrap/flux/patches/ are strategic-merge PATCH FRAGMENTS (they add a decryption stanza to # a Kustomization declared elsewhere); they name no source and reconcile nothing, so requiring # a path of them would fail on a file that is correct. reconcilers = [(p, d) for p, d in flux_ks if (d.get("spec", {}) or {}).get("sourceRef")] def _reconciled_path(d): return (d.get("spec", {}) or {}).get("path", "").lstrip("./").rstrip("/") unscanned = sorted({ f"{p.relative_to(ROOT)}:{(d.get('spec', {}) or {}).get('path', '') or '(no path)'}" for p, d in reconcilers if _reconciled_path(d).split("/")[0] not in DELIVERY_ROOTS and _reconciled_path(d) not in NON_SERVING_RECONCILED }) # The exemption's own guard: an exempted tree must actually serve nothing. exempt_serving = sorted({ f"{p.relative_to(ROOT)} ({d.get('kind')})" for p, d in docs_all() if any(str(p.relative_to(ROOT)).startswith(t + "/") for t in NON_SERVING_RECONCILED) and (d.get("kind") == "HTTPRoute" or (d.get("kind") == "Service" and any("hostname" in str(k).lower() for k in ((d.get("metadata", {}) or {}).get("annotations", {}) or {})))) }) check(not exempt_serving, f"(0b) every NON_SERVING_RECONCILED tree really serves nothing — an exempted tree that " f"gains a route would hide a service from this check entirely " f"{exempt_serving if exempt_serving else ''}") check(len(reconcilers) >= 10 and not unscanned, f"(0) all {len(reconcilers)} Flux-reconciled path(s) lie inside a scanned root — a " f"reconciled tree this check does not read is a service it cannot require a link for " f"{unscanned if unscanned else ''}") # ============================================================ what the repo SERVES # L7: every hostname an in-repo HTTPRoute claims, and the component that owns it. routes: dict[str, list[tuple[Path, dict]]] = {} apex_inheriting: list[Path] = [] for p, d in docs_all(): if d.get("kind") != "HTTPRoute": continue hosts = (d.get("spec", {}) or {}).get("hostnames") or [] if not hosts: # An HTTPRoute with NO hostnames inherits the hostname of every listener it attaches # to (runbooks/serving/httproute.yaml binds `https-apex` only, so it serves a PATH on # the apex). It introduces no new hostname, so it owes no tile. apex_inheriting.append(p) continue for h in hosts: routes.setdefault(str(h), []).append((p, d)) routed_labels = {} # sub-label -> owning component dir for h, owners in routes.items(): lbl = zone_label(h) if lbl: # "" is the apex; None is out of zone routed_labels[lbl] = owners[0][0].parent # L4: a Service that publishes a home-zone hostname to DNS is also "served by this repo" — # it is how UniFi (a KubeVirt VM behind a pinned Cilium LoadBalancer VIP on :11443) reaches # the LAN. external-dns's annotation IS the declaration; there is no HTTPRoute to find. DNS_ANN = "external-dns.alpha.kubernetes.io/hostname" l4_labels: dict[str, Path] = {} for p, d in docs_all(): if d.get("kind") != "Service": continue ann = (d.get("metadata", {}) or {}).get("annotations", {}) or {} for h in re.split(r"[,\s]+", str(ann.get(DNS_ANN, ""))): lbl = zone_label(h) if h else None if lbl: l4_labels[lbl] = p check(len(routed_labels) >= 10, f"(0) the repo's north-south HTTP surface is discovered " f"({len(routed_labels)} routed home-zone hostname(s), " f"{len(apex_inheriting)} apex-path route(s), {len(l4_labels)} L4-published hostname(s))") # ============================================================ (1) the apex is unambiguous apex_claims = routes.get(TOKEN, []) check(len(apex_claims) == 1, f"(1) exactly ONE HTTPRoute claims the bare apex {TOKEN} — an ambiguous root is not a " f"root (found {len(apex_claims)}: {[str(p.relative_to(ROOT)) for p, _ in apex_claims]})") apex_path, apex_route = (apex_claims[0] if apex_claims else (None, {})) DASH = apex_path.parent if apex_path is not None else None check(DASH is not None and DASH.name == "homepage", f"(1) the apex route belongs to the landing-dashboard component " f"(found {DASH.name if DASH else None!r})") # ============================================================ (2) it binds the apex listener parents = (apex_route.get("spec", {}) or {}).get("parentRefs") or [] section = next((pr.get("sectionName") for pr in parents if pr.get("sectionName")), None) gw_name = next((pr.get("name") for pr in parents), None) gw_ns = next((pr.get("namespace") for pr in parents), None) check(section is not None, "(2) the apex route pins a Gateway listener by sectionName — the wildcard listener's " "hostname cannot match a bare apex, so an unpinned apex route binds NOTHING") gw = next((d for _, d in docs_all() if d.get("kind") == "Gateway" and (d.get("metadata", {}) or {}).get("name") == gw_name and (gw_ns is None or (d.get("metadata", {}) or {}).get("namespace") == gw_ns)), None) check(gw is not None, f"(2) the parent Gateway '{gw_name}' is declared in the repo") listeners = ((gw or {}).get("spec", {}) or {}).get("listeners") or [] apex_listener = next((l for l in listeners if l.get("name") == section), None) check(apex_listener is not None, f"(2) the Gateway declares the '{section}' listener the apex route binds") check(apex_listener is not None and apex_listener.get("hostname") == TOKEN, f"(2) that listener's hostname is EXACTLY the bare apex " f"(found {(apex_listener or {}).get('hostname')!r})") check(apex_listener is not None and apex_listener.get("protocol") == "HTTPS", "(2) the apex listener terminates HTTPS (docs/12 acceptance gate: trusted TLS on the root)") # Prove the sectionName is LOAD-BEARING rather than decorative: a wildcard listener exists, # it is a DIFFERENT listener, and its hostname does not equal the apex. wildcard = [l for l in listeners if str(l.get("hostname", "")) == "*." + TOKEN] check(bool(wildcard) and all(l.get("name") != section for l in wildcard), f"(2) the wildcard listener(s) {[l.get('name') for l in wildcard]} are DISTINCT from " f"'{section}' — which is why the apex needs its own listener, and why dropping the " f"sectionName would leave the root unserved") # ============================================================ (3) trusted TLS on the root cert_secret = next((ref.get("name") for ref in ((apex_listener or {}).get("tls", {}) or {}).get("certificateRefs") or []), None) cert = next((d for _, d in docs_all() if d.get("kind") == "Certificate" and (d.get("spec", {}) or {}).get("secretName") == cert_secret), None) check(cert is not None, f"(3) a cert-manager Certificate produces the apex listener's Secret ({cert_secret!r})") check(cert is not None and TOKEN in ((cert.get("spec", {}) or {}).get("dnsNames") or []), "(3) the bare apex is a SAN on that Certificate — the wildcard alone does NOT cover it, " "so without this SAN the root serves an untrusted certificate") # ============================================================ (4)(5) it is RECONCILED dash_rel = f"./{DASH.relative_to(ROOT)}" if DASH else None dash_ks = [(p, d) for p, d in flux_ks if (d.get("spec", {}) or {}).get("path", "").rstrip("/") == dash_rel] check(len(dash_ks) == 1, f"(4) exactly one Flux Kustomization reconciles the dashboard ({dash_rel}) — authored " f"but unreconciled is the 'looks done, is absent' class this repo keeps hitting " f"(found {len(dash_ks)})") if dash_ks: base_txt = read(ROOT / "clusters" / "base" / "kustomization.yaml") entry = dash_ks[0][0].name check(re.search(rf"^\s*-\s*{re.escape(entry)}\s*(#.*)?$", base_txt, re.M) is not None, f"(4) that Kustomization is listed in the base entrypoint set ({entry})") # (5) The dashboard's hrefs and its route's hostname are ${BASE_DOMAIN} TOKENS. Without # postBuild.substituteFrom the token reaches the cluster LITERALLY: the apex route binds # the host `${BASE_DOMAIN}` (matching no listener) and every tile links to a host that # does not exist — and Flux still reports the Kustomization Ready. It fails OPEN. subs = (((dash_ks[0][1].get("spec", {}) or {}).get("postBuild") or {}).get("substituteFrom") or []) check(any((s.get("name") == "cluster-vars") for s in subs), f"(5) {dash_ks[0][0].name} carries postBuild.substituteFrom cluster-vars. It MUST: " f"this component's route hostname and EVERY tile href are ${{BASE_DOMAIN}} tokens, " f"and an unsubstituted token is applied literally while Flux still reports Ready — " f"a root bound to no listener and a dashboard of dead links, with a green tier") # ============================================================ (6) it ships the UI dash_kust = yaml.safe_load(read(DASH / "kustomization.yaml")) if DASH else None dash_res = set((dash_kust or {}).get("resources") or []) for want in ("configmap.yaml", "deployment.yaml", "service.yaml", "httproute.yaml"): check(want in dash_res, f"(6) the dashboard kustomization ships {want}") # ================================================== the dashboard's DECLARED link set # Homepage's config is nested YAML inside ConfigMap string values (services.yaml, # bookmarks.yaml). Both are read: a bookmark is a link the root offers just as a tile is. cm = next((d for d in (DOCS.get(DASH / "configmap.yaml") or []) if d.get("kind") == "ConfigMap"), {}) if DASH else {} cm_data = cm.get("data") or {} def embedded(key): try: return yaml.safe_load(cm_data.get(key) or "") or [] except yaml.YAMLError: return None svc_cfg = embedded("services.yaml") bm_cfg = embedded("bookmarks.yaml") check(svc_cfg is not None and bm_cfg is not None, "(7) the dashboard's embedded services.yaml / bookmarks.yaml parse as YAML — the app " "mounts these verbatim, so a malformed one is a dashboard that renders nothing") def hrefs(node): """Every href anywhere in Homepage's nested group/entry structure.""" if isinstance(node, dict): for k, v in node.items(): if k == "href" and isinstance(v, str): yield v else: yield from hrefs(v) elif isinstance(node, list): for v in node: yield from hrefs(v) def host_of(url): m = re.match(r"^https?://([^/]+)", url or "") return m.group(1) if m else None tile_urls = list(hrefs(svc_cfg or [])) all_urls = tile_urls + list(hrefs(bm_cfg or [])) all_hosts = [h for h in (host_of(u) for u in all_urls) if h] linked = {lbl for lbl in (zone_label(h) for h in all_hosts) if lbl} tile_labels = {lbl for lbl in (zone_label(h) for h in (host_of(u) for u in tile_urls) if h) if lbl} groups = [g for g in (svc_cfg or []) if isinstance(g, dict)] check(len(groups) >= 1 and len(tile_labels) >= 5, f"(7) the root is a grouped service catalog with links " f"({len(groups)} group(s), {len(tile_labels)} linked home-zone service(s))") # ============================================ (8) FORWARD: nothing routed is hidden unsurfaced = sorted(set(routed_labels) - linked) check(not unsurfaced, "(8) every service the repo ROUTES over HTTP is surfaced on the C2 root. Unsurfaced: " + ", ".join(f"{n}.{TOKEN} ({routed_labels[n].relative_to(ROOT)})" for n in unsurfaced) if unsurfaced else f"(8) every service the repo ROUTES over HTTP is surfaced on the C2 root " f"({len(routed_labels)} hostname(s) compared)") # L4-only endpoints are reported, never required: a raw TCP/UDP port has no browsable URL. for lbl in sorted(set(l4_labels) - linked): ADVISORIES.append( f"{lbl}.{TOKEN} is published to DNS by an L4 Service " f"({l4_labels[lbl].relative_to(ROOT)}) and is not linked from the root. Not a " f"failure: a raw TCP/UDP endpoint has no URL a browser can follow. Verify the " f"service it belongs to is reachable through a tile that IS linked") # ================================= (9) FORWARD: the VERSIONED onboarding contract cat_dir = ROOT / "catalog" cat_seen, cat_missing, cat_unrouted = 0, [], [] for p in sorted(cat_dir.glob("*.yaml")): sd = next((d for d in (DOCS.get(p) or [])), None) if not isinstance(sd, dict) or sd.get("kind") != "container": continue # kind: vm has no north-south HTTP surface (docs/09) for r in sd.get("ingressRoutes") or []: lbl = zone_label(str(r.get("host", ""))) if not lbl: continue if lbl not in routed_labels: # Declared in the catalog but not rendered into a route: an apps/ render drift # problem owned by R-GITOPS-02, reported here rather than silently skipped. cat_unrouted.append(f"{sd.get('name')}:{lbl}") continue cat_seen += 1 if lbl not in linked: cat_missing.append(f"{sd.get('name')}:{lbl}") check(cat_seen >= 5, f"(9) the versioned catalog contributes routed HTTP services ({cat_seen} found) — the " f"C2 is tied to the onboarding contract, not only to today's rendered output") check(not cat_missing, f"(9) every catalog `kind: container` service with a route is linked from the C2 root " f"{cat_missing if cat_missing else ''}") for u in cat_unrouted: ADVISORIES.append(f"catalog declares an ingress host with no rendered HTTPRoute: {u} " f"(render drift — R-GITOPS-02 owns it, not R-C2-01)") # ============================ (10)(11) REVERSE: no link points at nothing ann = (cm.get("metadata", {}) or {}).get("annotations", {}) or {} OFF_ANN = "homelab.keithjackson.dev/off-cluster-links" declared_off_raw = [t for t in re.split(r"[,\s]+", str(ann.get(OFF_ANN, ""))) if t] declared_off = {zone_label(t) for t in declared_off_raw} served = set(routed_labels) | set(l4_labels) dead = sorted(linked - served - declared_off) check(not dead, f"(10) every home-zone link the root offers resolves to something this repo serves — " f"an HTTPRoute (L7) or a hostname-publishing LoadBalancer Service (L4). Dead: " f"{[f'{d}.{TOKEN}' for d in dead] if dead else ''}") # The escape hatch is a DECLARATION, not a hole: same doctrine as an explicit `NONE` in # ci/touched-requirements.sh's path map. Empty today, and these two hold vacuously when it is # — they exist so it cannot be widened into a blanket without failing. check(all(zone_label(t) for t in declared_off_raw), f"(11) every declared off-cluster link is a specific home-zone hostname in TOKEN form " f"({declared_off_raw})") shadowed = sorted(declared_off & served) check(not shadowed, f"(11) no off-cluster declaration shadows a hostname the repo DOES serve " f"{shadowed if shadowed else ''}") # ============================ (12) tokens, never literals literal_hosts = sorted({h for h in all_hosts if is_literal_base(h)}) check(not literal_hosts, f"(12) no link on the root names a LITERAL '{LITERAL_BASE}' host — every hostname is the " f"${{BASE_DOMAIN}} token, so prod's front door cannot link dev's services (the " f"2026-09-04 class: two clusters authored to describe the same names). Literals: " f"{literal_hosts if literal_hosts else ''}") # ===================================================================== # R-C2-02 — platform health, and controlled operational actions # ===================================================================== # docs/12 names the surfaces the C2 composes: Headlamp is "the R-C2-02 'controlled # operational actions where safe' surface"; Grafana is the dashboards/alerts surface. Each is # identified by the component that OWNS ITS ROUTE, not by a magic string in the config. CONSOLE, DASHBOARDS = "headlamp", "grafana" check(CONSOLE in linked, f"(13) the C2 root links the Kubernetes console ({CONSOLE}.{TOKEN}) — cluster, storage " f"and GitOps state, and the R-C2-02 operational-actions surface (docs/12)") check(DASHBOARDS in linked, f"(13) the C2 root links the dashboards/alerts surface ({DASHBOARDS}.{TOKEN})") AGG = ("\n".join(read(ROOT / t / "kustomization.yaml") for t in ("platform", "platform-config", "platform-config-cdi"))) for label, name in (("console", CONSOLE), ("dashboards", DASHBOARDS)): comp = routed_labels.get(name) check(comp is not None, f"(14) the {label} surface has an in-repo HTTPRoute") if comp is None: continue rel = f"./{comp.relative_to(ROOT)}" reconciled = any((d.get("spec", {}) or {}).get("path", "").rstrip("/") == rel for _, d in flux_ks) if not reconciled: # A component may reconcile as a MEMBER of an aggregate tier's kustomize tree # (platform / platform-config) instead of owning a Flux Kustomization. Both count; # being in NEITHER is "authored, never applied". reconciled = (f"\n - {comp.name}" in AGG or f"../{comp.relative_to(ROOT)}/" in AGG) check(reconciled, f"(14) the {label} surface ({comp.name}) is reconciled by Flux, not merely authored " f"— a link to an unreconciled component is a link to nothing") # (15) the health DATA exists as code, for every axis R-C2-02 enumerates. rule_files = sorted({p for p, d in docs_all() if d.get("kind") == "PrometheusRule"}) rule_groups = {g.get("name", "") for p, d in docs_all() if d.get("kind") == "PrometheusRule" for g in ((d.get("spec", {}) or {}).get("groups") or [])} for axis, needle in (("cluster/nodes", "nodes"), ("storage", "longhorn"), ("GitOps reconciliation", "flux"), ("certificate expiry", "certificates")): check(any(needle in g for g in rule_groups), f"(15) platform-health alert rules exist for {axis} (a '*{needle}*' rule group) — " f"the C2 cannot surface health that is not measured") pc_txt = read(ROOT / "platform-config" / "kustomization.yaml") reconciled_rules = [p for p in rule_files if f"../{p.relative_to(ROOT)}" in pc_txt] check(len(reconciled_rules) >= 2, f"(15) those alert rules are RECONCILED — PrometheusRule is a CR of a CRD the platform " f"tier installs, so the rules are applied by platform-config " f"({len(reconciled_rules)}/{len(rule_files)} rule file(s) referenced there)") # (16)(17) controlled actions: no shared cluster-admin identity; per-user OIDC instead. console_dir = routed_labels.get(CONSOLE) hr = next((d for p in (sorted(console_dir.glob("*.yaml")) if console_dir else []) for d in (DOCS.get(p) or []) if d.get("kind") == "HelmRelease"), None) check(hr is not None, "(16) the Kubernetes console is deployed by a HelmRelease") hv = ((hr or {}).get("spec", {}) or {}).get("values", {}) or {} check((hv.get("clusterRoleBinding") or {}).get("create") is False, "(16) the console chart is told NOT to bind its ServiceAccount to a ClusterRole. The " "chart's DEFAULT binds it to cluster-admin, which is exactly the 'raw cluster-admin " "kubeconfig in a browser' docs/12 forbids — every visitor would act as one shared " "omnipotent identity") oidc = ((hv.get("config") or {}).get("oidc") or {}) check(bool(oidc), "(17) the console is configured for native OIDC login, so an action carries the " "operator's OWN identity (docs/12: Headlamp/Grafana use native OIDC)") check(bool((oidc.get("externalSecret") or {}).get("name")), "(17) its OIDC client credentials come from a NAMED (SOPS) Secret, not inline in the " "HelmRelease values — R-SEC-01") # (18) rights arrive through GROUPS, at more than one privilege level. group_bindings = [] for p, d in docs_all(): if d.get("kind") not in ("ClusterRoleBinding", "RoleBinding"): continue for s in d.get("subjects") or []: if s.get("kind") == "Group" and str(s.get("name", "")).startswith("oidc:"): group_bindings.append(((d.get("metadata", {}) or {}).get("name"), s.get("name"), (d.get("roleRef") or {}).get("name"))) check(len(group_bindings) >= 2, f"(18) cluster rights are bound to OIDC GROUPS, so access follows Keycloak group " f"membership rather than a per-user grant somebody must remember to revoke " f"({[(g, r) for _, g, r in group_bindings]})") check(any(r == "cluster-admin" for _, _, r in group_bindings) and any(r != "cluster-admin" for _, _, r in group_bindings), "(18) MORE THAN ONE privilege level exists (an admin group AND a lesser one) — " "least-privilege, not one tier for everybody (docs/12 access control, R-IDP-06)") # (19) the root DECLARES the authenticated-group posture. DECLARED, NOT ENFORCED. route_ann = (apex_route.get("metadata", {}) or {}).get("annotations", {}) or {} check(route_ann.get("homelab.keithjackson.dev/auth-mode") in ("forward-auth", "oidc", "oauth2-proxy"), f"(19) the C2 root's route DECLARES an authentication mode " f"(found {route_ann.get('homelab.keithjackson.dev/auth-mode')!r}) " f"[DECLARED, NOT ENFORCED — no Gateway ext-authz attachment exists in this repo]") check(bool(route_ann.get("homelab.keithjackson.dev/auth-groups")), "(19) it DECLARES the authorized group(s) — docs/12 requires an authenticated user IN " "AN AUTHORIZED GROUP, not merely logged in " "[DECLARED, NOT ENFORCED — see the header's gap note]") # ======================================================================== report print() print(f"C2 surface inventory (apex {TOKEN}, literal base default '{LITERAL_BASE}'):") print(f" routed over HTTP : {len(routed_labels)} -> {', '.join(sorted(routed_labels))}") print(f" L4-published : {len(l4_labels)} -> {', '.join(sorted(l4_labels)) or '(none)'}") print(f" linked from root : {len(linked)} -> {', '.join(sorted(linked))}") print(f" apex-path routes : {len(apex_inheriting)} -> " f"{', '.join(str(p.relative_to(ROOT)) for p in apex_inheriting) or '(none)'}") print(f" catalog services compared: {cat_seen} off-cluster declarations: " f"{sorted(declared_off_raw) or '(none)'}") if ADVISORIES: print("\nADVISORIES (not failures):") for a in ADVISORIES: print(f" - {a}") print() if not FAILURES: print(f"check OK: c2-root-surface — {CHECKS} assertions (R-C2-01, R-C2-02)") raise SystemExit(0) print(f"CHECK FAIL (c2-root-surface): {len(FAILURES)} failed, " f"{CHECKS - len(FAILURES)} passed", file=sys.stderr) for f in FAILURES: print(f" FAIL {f}", file=sys.stderr) raise SystemExit(1) PY