home-network-k3s-impl/conformance/tests/l0-static/c2-root-surface/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — c2-root-surface
#
# Serves (spec docs/00-vision-and-scope.md; detail in docs/12-command-center-ui.md):
#
#   R-C2-01  "The root `home.keithjackson.dev` MUST serve a UI that surfaces the
#             services running in the cluster and links to them."
#   R-C2-02  "The C2 UI MUST surface platform health (cluster, storage, GitOps,
#             alerts) and provide controlled operational actions where safe."
#
# WHY THIS IS AN L0 STATIC CHECK AND NOT AN L2 PROBE
# --------------------------------------------------
# The REACHABILITY half of R-C2-01 is already exercised against a live cluster:
# l2-behavioral/gateway-ingress-reachable dials the Gateway and l2-behavioral/trusted-tls
# verifies the certificate it serves. What no live probe can decide is the half the
# requirement is actually about — whether the UI at the root surfaces THE SERVICES RUNNING
# IN THE CLUSTER. Deciding that is a comparison of two SETS, and both are repository facts:
# what git declares the cluster serves (every in-repo HTTPRoute hostname, every
# hostname-publishing L4 Service, every catalog ServiceDefinition) against what the
# dashboard's ConfigMap links. A probe of the root gets HTTP 200 from a dashboard that hides
# half the estate and cannot tell the difference.
#
# It is also the assertion that ROTS, in the direction nothing else notices: onboard a
# service, forget the dashboard, and every cluster test stays green while the operator's
# front door hides a running service. That is not a styling preference — it is exactly the
# R-C2-01 failure mode, and it was a LIVE defect on main when this check was written
# (platform/crafty served crafty.${BASE_DOMAIN} and nothing on the root linked to it).
#
# THE RULE THIS IMPLEMENTS, in one sentence: every home-zone hostname the repository routes
# over HTTP must be linked from the apex dashboard's configuration, every home-zone link the
# apex dashboard offers must resolve to something the repository actually serves, and the two
# R-C2-02 surfaces (health data + a per-user-authorized console) must be linked, reconciled
# and backed by real alert rules.
#
# GIVEN the repository as git tracks it,
# WHEN  the apex ingress chain, the dashboard's declared link set, every in-repo HTTPRoute,
#       every hostname-publishing L4 Service, the versioned catalog, the platform-health
#       alert rules and the console's RBAC posture are read and compared,
# THEN — R-C2-01 —
#       (0)  the INPUT SURFACE is intact: every Flux-reconciled path lies inside the roots
#            this check scans, and every manifest in them parses. A root this check does not
#            scan, or a file it cannot parse, is a service it cannot see — which would
#            silently weaken every assertion below;
#       (1)  EXACTLY ONE HTTPRoute claims the bare apex (an ambiguous root is not a root),
#            and it belongs to the landing-dashboard component;
#       (2)  that route pins by sectionName the ONE Gateway listener whose hostname is the
#            bare apex, and that listener is HTTPS — the wildcard `*.${BASE_DOMAIN}` listener
#            does NOT intersect the bare apex, so without the sectionName the root binds
#            nothing and serves nothing (asserted against the wildcard listener's existence,
#            so the claim is proven rather than asserted from prose);
#       (3)  the apex is a SAN on the Certificate whose Secret that listener terminates with;
#       (4)  the dashboard is RECONCILED, not merely authored: one Flux Kustomization points
#            at its path and is listed in the base entrypoint set;
#       (5)  that Kustomization carries postBuild.substituteFrom, because the dashboard's
#            hrefs and its route's hostname are written as `${BASE_DOMAIN}` tokens — without
#            substitution the apex route binds a literal `${BASE_DOMAIN}` and EVERY tile on
#            the root links to a nonexistent host, while Flux still reports Ready;
#       (6)  the component's own kustomization ships config + workload + Service + route;
#       (7)  the dashboard is a service CATALOG: grouped entries carrying hrefs;
#       (8)  FORWARD — every home-zone hostname an in-repo HTTPRoute serves, other than the
#            apex itself, is LINKED from the root. This is the rot-catcher;
#       (9)  FORWARD — every catalog `kind: container` ServiceDefinition whose declared
#            ingress host is routed is likewise linked, tying the C2 to the VERSIONED
#            onboarding contract and not only to whatever is rendered today;
#       (10) REVERSE — every home-zone link the root offers resolves to something this repo
#            actually serves: an HTTPRoute (L7) or a hostname-publishing LoadBalancer Service
#            (L4). A dead tile is a link the root offers to nothing;
#       (11) the off-cluster escape hatch is not a blanket one: every name declared in the
#            ConfigMap's `off-cluster-links` annotation is a specific home-zone hostname and
#            none of them shadows a hostname the repo DOES serve;
#       (12) no href names a LITERAL base-domain host. Every hostname is the `${BASE_DOMAIN}`
#            token, so prod's root cannot link dev's services — the 2026-09-04 incident class
#            (two clusters authored to describe the same names), applied to the front door;
#     — R-C2-02 —
#       (13) both health surfaces docs/12 names are linked: the Kubernetes console
#            (cluster/storage/GitOps state) and the dashboards/alerts surface;
#       (14) both are RECONCILED by Flux, so the links lead to something running;
#       (15) the health DATA those surfaces render exists as code — PrometheusRule groups for
#            each axis R-C2-02 enumerates (cluster/nodes, storage, GitOps reconciliation)
#            plus certificate expiry — and those rule files are reconciled;
#       (16) actions are CONTROLLED, not cluster-admin-in-a-browser (docs/12: "never a raw
#            cluster-admin kubeconfig in a browser"): the console chart is explicitly told NOT
#            to bind its ServiceAccount to a ClusterRole, so there is no shared privileged
#            identity to act as;
#       (17) actions are instead PER-USER: the console is configured for native OIDC with
#            credentials from a named (SOPS) Secret, not inline;
#       (18) cluster rights arrive through OIDC GROUPS at MORE THAN ONE privilege level (an
#            admin group and a lesser one) — least-privilege, not one tier for everybody;
#       (19) the C2 root DECLARES the authenticated-group posture docs/12 requires, on its own
#            route (declared, NOT enforced — see the gap note below).
#
# WHAT THIS DOES *NOT* PROVE — honest gaps, the same ones the Test declaration names:
#   - That the apex answers HTTP 200, or that any tile's status ping is green. That is L2; the
#     Gateway/TLS half is covered under R-DNS-01 / R-TLS-01.
#   - That the C2 is access-CONTROLLED in fact. (19) reads a DECLARED posture, because the
#     Gateway ext-authz attachment that would ENFORCE forward-auth is not built anywhere in
#     this repo (platform/homepage/httproute.yaml says so in its own header, and every
#     forward-auth app shares the gap). Declared, not enforced, and labelled as such in the
#     assertion text itself.
#   - That an operational action in the console succeeds, or that it is audited. (16)-(18)
#     prove there is no shared cluster-admin identity to act with and that rights arrive
#     per-user; the audit trail is a runtime property.
#   - Whether a tile is USEFUL. Homepage service-widget API keys are a deliberate follow-up
#     (platform/homepage/configmap.yaml); link presence is what R-C2-01 asks for.
#   - That an L4-ONLY endpoint is surfaced. A raw TCP/UDP endpoint (a Minecraft game port) has
#     no URL a browser can follow, so the FORWARD direction quantifies over HTTP routes;
#     L4-published hostnames are listed as ADVISORIES and are honoured in the REVERSE
#     direction (that is how the UniFi tile, an L4 LoadBalancer on :11443, resolves).
#   - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06) and the
#     requirement text above is quoted, not parsed.
#
# STATUS: EXECUTABLE. Pure offline: `git ls-files` + PyYAML (the dependency ten neighbouring
# l0-static checks already take). No cluster, no network, no spec repo.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"

# Honest SKIP (never green, and rc 3 is UNWIRED for a scope-declaring test) if the landing
# dashboard component is absent entirely — there is then no C2 root to audit, which is a
# report about the tree, not a pass.
if [ ! -f "${ROOT}/platform/homepage/configmap.yaml" ]; then
  echo "check SKIPPED: platform/homepage/configmap.yaml absent — no C2 landing dashboard to audit" >&2
  exit 3
fi

cd "${ROOT}"
"${PYTHON}" - "${ROOT}" <<'PY'
"""Static audit: the C2 root surfaces every service the repository serves, plus health.

Every parse contract is documented at its parser. PyYAML rather than regex because the
subjects are genuinely structured: multi-document apps/ bundles, and a Homepage config that
is nested YAML *inside* a ConfigMap string value.
"""
from __future__ import annotations

import re
import subprocess
import sys
from pathlib import Path

try:
    import yaml
except ImportError:                                  # pragma: no cover
    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
          file=sys.stderr)
    raise SystemExit(1)

ROOT = Path(sys.argv[1])
sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree

FAILURES: list[str] = []
ADVISORIES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if cond:
        print(f"  ok   {msg}")
    else:
        print(f"  FAIL {msg}", file=sys.stderr)
        FAILURES.append(msg)


def read(p: Path) -> str:
    try:
        return p.read_text()
    except (OSError, UnicodeDecodeError):
        return ""


# ============================================================ the hostname vocabulary
# BASE_DOMAIN is a cluster-vars knob (R-ENV-02): every public hostname the cluster serves is
# `<svc>.${BASE_DOMAIN}`, and dev/prod differ ONLY in that value. So a hostname in this repo
# is canonically the TOKEN form, and the literal value below exists only to be FORBIDDEN in
# (12): a literal is how prod came to be authored to serve dev's hostnames (2026-09-04,
# clusters/base/cluster-vars.yaml's own header).
TOKEN = "${BASE_DOMAIN}"
_vars = yaml.safe_load(read(ROOT / "clusters" / "base" / "cluster-vars.yaml")) or {}
LITERAL_BASE = ((_vars.get("data") or {}).get("BASE_DOMAIN") or "").strip()


def strip_port(host: str) -> str:
    return host.split(":", 1)[0]


def zone_label(host: str):
    """The sub-label of a home-zone hostname, "" for the bare apex, or None if not in zone.

    Accepts only the TOKEN form; a literal-base hostname returns None here and is caught
    separately by (12), so a literal can never silently satisfy a set comparison.
    """
    h = strip_port(host)
    if h == TOKEN:
        return ""
    if h.endswith("." + TOKEN):
        return h[: -len("." + TOKEN)]
    return None


def is_literal_base(host: str) -> bool:
    h = strip_port(host)
    return bool(LITERAL_BASE) and (h == LITERAL_BASE or h.endswith("." + LITERAL_BASE))


# ============================================================ (0) the input surface
# DELIVERY ROOTS — the subtrees Flux reconciles, plus catalog/ (the versioned source the
# apps/ manifests are rendered FROM). conformance/ is deliberately EXCLUDED: its fixtures and
# goldens declare synthetic HTTPRoutes (`paas-demo.…`, `gw-declared-conformance.…`) that are
# test data, not services the cluster serves, and counting them would make (8) assert that
# the operator's dashboard links this harness's fixtures.
DELIVERY_ROOTS = ("apps", "platform", "platform-config", "platform-config-cdi",
                  "runbooks", "clusters", "bootstrap", "catalog")

# NARROW EXEMPTION to (0), for reconciled trees that deliberately serve NOTHING.
# `conformance/incluster` (the L2/L3 runner + CI-trigger identities, R-SEC-02) is reconciled
# by clusters/base/conformance.yaml but contains only a Namespace, ServiceAccounts, RBAC and
# an admission-bounded Job template. Adding "conformance" to DELIVERY_ROOTS instead would be
# wrong for the reason stated above: it would sweep in this harness's synthetic fixture
# routes and make (8) demand the operator's dashboard link them.
#
# THE EXEMPTION POLICES ITSELF. An unchecked allowlist is how (0) would rot into a fail-open —
# add a tree here, later put an HTTPRoute in it, and the forward direction goes quietly green
# on a hidden service. So each exempted tree is ASSERTED below to declare no route and no
# hostname-publishing Service. If that ever stops being true the check fails and the tree must
# become a real delivery root.
NON_SERVING_RECONCILED = ("conformance/incluster",)


def tracked_yaml() -> list[Path]:
    out = subprocess.run(["git", "ls-files", "-z", "--", "*.yaml", "*.yml"],
                         cwd=ROOT, capture_output=True, text=True, check=True)
    return [ROOT / p for p in out.stdout.split("\0") if p]


ALL_TRACKED = tracked_yaml()
DELIVERY = [p for p in ALL_TRACKED
            if p.relative_to(ROOT).parts[0] in DELIVERY_ROOTS]

parse_failed: list[str] = []
DOCS: dict[Path, list[dict]] = {}
for p in DELIVERY:
    try:
        DOCS[p] = [d for d in yaml.safe_load_all(read(p)) if isinstance(d, dict)]
    except yaml.YAMLError:
        DOCS[p] = []
        parse_failed.append(str(p.relative_to(ROOT)))


def docs_all():
    for p, ds in DOCS.items():
        for d in ds:
            yield p, d


check(len(DELIVERY) >= 100,
      f"(0) the delivery tree is present and scanned ({len(DELIVERY)} tracked manifest(s) "
      f"under {'/, '.join(DELIVERY_ROOTS)}/)")
check(not parse_failed,
      f"(0) every manifest in the scanned roots PARSES — an unparseable one is a service "
      f"this check cannot see, so it would weaken every set comparison below "
      f"{parse_failed if parse_failed else ''}")

# Every Flux-reconciled path must lie inside a root this check scans. This is the guard on
# the guard: if a future tier reconciles, say, ./services, the forward direction would stop
# seeing its routes and would go quietly green on a dashboard that hides all of them.
flux_ks = [(p, d) for p, d in docs_all()
           if d.get("kind") == "Kustomization"
           and str(d.get("apiVersion", "")).startswith("kustomize.toolkit.fluxcd.io")]
# A RECONCILING Kustomization is one carrying a sourceRef. The pathless documents under
# bootstrap/flux/patches/ are strategic-merge PATCH FRAGMENTS (they add a decryption stanza to
# a Kustomization declared elsewhere); they name no source and reconcile nothing, so requiring
# a path of them would fail on a file that is correct.
reconcilers = [(p, d) for p, d in flux_ks if (d.get("spec", {}) or {}).get("sourceRef")]
def _reconciled_path(d):
    return (d.get("spec", {}) or {}).get("path", "").lstrip("./").rstrip("/")


unscanned = sorted({
    f"{p.relative_to(ROOT)}:{(d.get('spec', {}) or {}).get('path', '') or '(no path)'}"
    for p, d in reconcilers
    if _reconciled_path(d).split("/")[0] not in DELIVERY_ROOTS
    and _reconciled_path(d) not in NON_SERVING_RECONCILED
})

# The exemption's own guard: an exempted tree must actually serve nothing.
exempt_serving = sorted({
    f"{p.relative_to(ROOT)} ({d.get('kind')})"
    for p, d in docs_all()
    if any(str(p.relative_to(ROOT)).startswith(t + "/") for t in NON_SERVING_RECONCILED)
    and (d.get("kind") == "HTTPRoute"
         or (d.get("kind") == "Service"
             and any("hostname" in str(k).lower()
                     for k in ((d.get("metadata", {}) or {}).get("annotations", {}) or {}))))
})
check(not exempt_serving,
      f"(0b) every NON_SERVING_RECONCILED tree really serves nothing — an exempted tree that "
      f"gains a route would hide a service from this check entirely "
      f"{exempt_serving if exempt_serving else ''}")
check(len(reconcilers) >= 10 and not unscanned,
      f"(0) all {len(reconcilers)} Flux-reconciled path(s) lie inside a scanned root — a "
      f"reconciled tree this check does not read is a service it cannot require a link for "
      f"{unscanned if unscanned else ''}")

# ============================================================ what the repo SERVES
# L7: every hostname an in-repo HTTPRoute claims, and the component that owns it.
routes: dict[str, list[tuple[Path, dict]]] = {}
apex_inheriting: list[Path] = []
for p, d in docs_all():
    if d.get("kind") != "HTTPRoute":
        continue
    hosts = (d.get("spec", {}) or {}).get("hostnames") or []
    if not hosts:
        # An HTTPRoute with NO hostnames inherits the hostname of every listener it attaches
        # to (runbooks/serving/httproute.yaml binds `https-apex` only, so it serves a PATH on
        # the apex). It introduces no new hostname, so it owes no tile.
        apex_inheriting.append(p)
        continue
    for h in hosts:
        routes.setdefault(str(h), []).append((p, d))

routed_labels = {}          # sub-label -> owning component dir
for h, owners in routes.items():
    lbl = zone_label(h)
    if lbl:                 # "" is the apex; None is out of zone
        routed_labels[lbl] = owners[0][0].parent

# L4: a Service that publishes a home-zone hostname to DNS is also "served by this repo" —
# it is how UniFi (a KubeVirt VM behind a pinned Cilium LoadBalancer VIP on :11443) reaches
# the LAN. external-dns's annotation IS the declaration; there is no HTTPRoute to find.
DNS_ANN = "external-dns.alpha.kubernetes.io/hostname"
l4_labels: dict[str, Path] = {}
for p, d in docs_all():
    if d.get("kind") != "Service":
        continue
    ann = (d.get("metadata", {}) or {}).get("annotations", {}) or {}
    for h in re.split(r"[,\s]+", str(ann.get(DNS_ANN, ""))):
        lbl = zone_label(h) if h else None
        if lbl:
            l4_labels[lbl] = p

check(len(routed_labels) >= 10,
      f"(0) the repo's north-south HTTP surface is discovered "
      f"({len(routed_labels)} routed home-zone hostname(s), "
      f"{len(apex_inheriting)} apex-path route(s), {len(l4_labels)} L4-published hostname(s))")

# ============================================================ (1) the apex is unambiguous
apex_claims = routes.get(TOKEN, [])
check(len(apex_claims) == 1,
      f"(1) exactly ONE HTTPRoute claims the bare apex {TOKEN} — an ambiguous root is not a "
      f"root (found {len(apex_claims)}: {[str(p.relative_to(ROOT)) for p, _ in apex_claims]})")
apex_path, apex_route = (apex_claims[0] if apex_claims else (None, {}))
DASH = apex_path.parent if apex_path is not None else None
check(DASH is not None and DASH.name == "homepage",
      f"(1) the apex route belongs to the landing-dashboard component "
      f"(found {DASH.name if DASH else None!r})")

# ============================================================ (2) it binds the apex listener
parents = (apex_route.get("spec", {}) or {}).get("parentRefs") or []
section = next((pr.get("sectionName") for pr in parents if pr.get("sectionName")), None)
gw_name = next((pr.get("name") for pr in parents), None)
gw_ns = next((pr.get("namespace") for pr in parents), None)
check(section is not None,
      "(2) the apex route pins a Gateway listener by sectionName — the wildcard listener's "
      "hostname cannot match a bare apex, so an unpinned apex route binds NOTHING")

gw = next((d for _, d in docs_all()
           if d.get("kind") == "Gateway"
           and (d.get("metadata", {}) or {}).get("name") == gw_name
           and (gw_ns is None or (d.get("metadata", {}) or {}).get("namespace") == gw_ns)), None)
check(gw is not None, f"(2) the parent Gateway '{gw_name}' is declared in the repo")
listeners = ((gw or {}).get("spec", {}) or {}).get("listeners") or []
apex_listener = next((l for l in listeners if l.get("name") == section), None)
check(apex_listener is not None,
      f"(2) the Gateway declares the '{section}' listener the apex route binds")
check(apex_listener is not None and apex_listener.get("hostname") == TOKEN,
      f"(2) that listener's hostname is EXACTLY the bare apex "
      f"(found {(apex_listener or {}).get('hostname')!r})")
check(apex_listener is not None and apex_listener.get("protocol") == "HTTPS",
      "(2) the apex listener terminates HTTPS (docs/12 acceptance gate: trusted TLS on the root)")
# Prove the sectionName is LOAD-BEARING rather than decorative: a wildcard listener exists,
# it is a DIFFERENT listener, and its hostname does not equal the apex.
wildcard = [l for l in listeners if str(l.get("hostname", "")) == "*." + TOKEN]
check(bool(wildcard) and all(l.get("name") != section for l in wildcard),
      f"(2) the wildcard listener(s) {[l.get('name') for l in wildcard]} are DISTINCT from "
      f"'{section}' — which is why the apex needs its own listener, and why dropping the "
      f"sectionName would leave the root unserved")

# ============================================================ (3) trusted TLS on the root
cert_secret = next((ref.get("name")
                    for ref in ((apex_listener or {}).get("tls", {}) or {}).get("certificateRefs") or []),
                   None)
cert = next((d for _, d in docs_all()
             if d.get("kind") == "Certificate"
             and (d.get("spec", {}) or {}).get("secretName") == cert_secret), None)
check(cert is not None,
      f"(3) a cert-manager Certificate produces the apex listener's Secret ({cert_secret!r})")
check(cert is not None and TOKEN in ((cert.get("spec", {}) or {}).get("dnsNames") or []),
      "(3) the bare apex is a SAN on that Certificate — the wildcard alone does NOT cover it, "
      "so without this SAN the root serves an untrusted certificate")

# ============================================================ (4)(5) it is RECONCILED
dash_rel = f"./{DASH.relative_to(ROOT)}" if DASH else None
dash_ks = [(p, d) for p, d in flux_ks
           if (d.get("spec", {}) or {}).get("path", "").rstrip("/") == dash_rel]
check(len(dash_ks) == 1,
      f"(4) exactly one Flux Kustomization reconciles the dashboard ({dash_rel}) — authored "
      f"but unreconciled is the 'looks done, is absent' class this repo keeps hitting "
      f"(found {len(dash_ks)})")
if dash_ks:
    base_txt = read(ROOT / "clusters" / "base" / "kustomization.yaml")
    entry = dash_ks[0][0].name
    check(re.search(rf"^\s*-\s*{re.escape(entry)}\s*(#.*)?$", base_txt, re.M) is not None,
          f"(4) that Kustomization is listed in the base entrypoint set ({entry})")
    # (5) The dashboard's hrefs and its route's hostname are ${BASE_DOMAIN} TOKENS. Without
    # postBuild.substituteFrom the token reaches the cluster LITERALLY: the apex route binds
    # the host `${BASE_DOMAIN}` (matching no listener) and every tile links to a host that
    # does not exist — and Flux still reports the Kustomization Ready. It fails OPEN.
    subs = (((dash_ks[0][1].get("spec", {}) or {}).get("postBuild") or {}).get("substituteFrom") or [])
    check(any((s.get("name") == "cluster-vars") for s in subs),
          f"(5) {dash_ks[0][0].name} carries postBuild.substituteFrom cluster-vars. It MUST: "
          f"this component's route hostname and EVERY tile href are ${{BASE_DOMAIN}} tokens, "
          f"and an unsubstituted token is applied literally while Flux still reports Ready — "
          f"a root bound to no listener and a dashboard of dead links, with a green tier")

# ============================================================ (6) it ships the UI
dash_kust = yaml.safe_load(read(DASH / "kustomization.yaml")) if DASH else None
dash_res = set((dash_kust or {}).get("resources") or [])
for want in ("configmap.yaml", "deployment.yaml", "service.yaml", "httproute.yaml"):
    check(want in dash_res, f"(6) the dashboard kustomization ships {want}")

# ================================================== the dashboard's DECLARED link set
# Homepage's config is nested YAML inside ConfigMap string values (services.yaml,
# bookmarks.yaml). Both are read: a bookmark is a link the root offers just as a tile is.
cm = next((d for d in (DOCS.get(DASH / "configmap.yaml") or []) if d.get("kind") == "ConfigMap"),
          {}) if DASH else {}
cm_data = cm.get("data") or {}


def embedded(key):
    try:
        return yaml.safe_load(cm_data.get(key) or "") or []
    except yaml.YAMLError:
        return None


svc_cfg = embedded("services.yaml")
bm_cfg = embedded("bookmarks.yaml")
check(svc_cfg is not None and bm_cfg is not None,
      "(7) the dashboard's embedded services.yaml / bookmarks.yaml parse as YAML — the app "
      "mounts these verbatim, so a malformed one is a dashboard that renders nothing")


def hrefs(node):
    """Every href anywhere in Homepage's nested group/entry structure."""
    if isinstance(node, dict):
        for k, v in node.items():
            if k == "href" and isinstance(v, str):
                yield v
            else:
                yield from hrefs(v)
    elif isinstance(node, list):
        for v in node:
            yield from hrefs(v)


def host_of(url):
    m = re.match(r"^https?://([^/]+)", url or "")
    return m.group(1) if m else None


tile_urls = list(hrefs(svc_cfg or []))
all_urls = tile_urls + list(hrefs(bm_cfg or []))
all_hosts = [h for h in (host_of(u) for u in all_urls) if h]
linked = {lbl for lbl in (zone_label(h) for h in all_hosts) if lbl}
tile_labels = {lbl for lbl in (zone_label(h) for h in (host_of(u) for u in tile_urls) if h) if lbl}

groups = [g for g in (svc_cfg or []) if isinstance(g, dict)]
check(len(groups) >= 1 and len(tile_labels) >= 5,
      f"(7) the root is a grouped service catalog with links "
      f"({len(groups)} group(s), {len(tile_labels)} linked home-zone service(s))")

# ============================================ (8) FORWARD: nothing routed is hidden
unsurfaced = sorted(set(routed_labels) - linked)
check(not unsurfaced,
      "(8) every service the repo ROUTES over HTTP is surfaced on the C2 root. Unsurfaced: "
      + ", ".join(f"{n}.{TOKEN} ({routed_labels[n].relative_to(ROOT)})" for n in unsurfaced)
      if unsurfaced else
      f"(8) every service the repo ROUTES over HTTP is surfaced on the C2 root "
      f"({len(routed_labels)} hostname(s) compared)")

# L4-only endpoints are reported, never required: a raw TCP/UDP port has no browsable URL.
for lbl in sorted(set(l4_labels) - linked):
    ADVISORIES.append(
        f"{lbl}.{TOKEN} is published to DNS by an L4 Service "
        f"({l4_labels[lbl].relative_to(ROOT)}) and is not linked from the root. Not a "
        f"failure: a raw TCP/UDP endpoint has no URL a browser can follow. Verify the "
        f"service it belongs to is reachable through a tile that IS linked")

# ================================= (9) FORWARD: the VERSIONED onboarding contract
cat_dir = ROOT / "catalog"
cat_seen, cat_missing, cat_unrouted = 0, [], []
for p in sorted(cat_dir.glob("*.yaml")):
    sd = next((d for d in (DOCS.get(p) or [])), None)
    if not isinstance(sd, dict) or sd.get("kind") != "container":
        continue                      # kind: vm has no north-south HTTP surface (docs/09)
    for r in sd.get("ingressRoutes") or []:
        lbl = zone_label(str(r.get("host", "")))
        if not lbl:
            continue
        if lbl not in routed_labels:
            # Declared in the catalog but not rendered into a route: an apps/ render drift
            # problem owned by R-GITOPS-02, reported here rather than silently skipped.
            cat_unrouted.append(f"{sd.get('name')}:{lbl}")
            continue
        cat_seen += 1
        if lbl not in linked:
            cat_missing.append(f"{sd.get('name')}:{lbl}")
check(cat_seen >= 5,
      f"(9) the versioned catalog contributes routed HTTP services ({cat_seen} found) — the "
      f"C2 is tied to the onboarding contract, not only to today's rendered output")
check(not cat_missing,
      f"(9) every catalog `kind: container` service with a route is linked from the C2 root "
      f"{cat_missing if cat_missing else ''}")
for u in cat_unrouted:
    ADVISORIES.append(f"catalog declares an ingress host with no rendered HTTPRoute: {u} "
                      f"(render drift — R-GITOPS-02 owns it, not R-C2-01)")

# ============================ (10)(11) REVERSE: no link points at nothing
ann = (cm.get("metadata", {}) or {}).get("annotations", {}) or {}
OFF_ANN = "homelab.keithjackson.dev/off-cluster-links"
declared_off_raw = [t for t in re.split(r"[,\s]+", str(ann.get(OFF_ANN, ""))) if t]
declared_off = {zone_label(t) for t in declared_off_raw}
served = set(routed_labels) | set(l4_labels)
dead = sorted(linked - served - declared_off)
check(not dead,
      f"(10) every home-zone link the root offers resolves to something this repo serves — "
      f"an HTTPRoute (L7) or a hostname-publishing LoadBalancer Service (L4). Dead: "
      f"{[f'{d}.{TOKEN}' for d in dead] if dead else ''}")
# The escape hatch is a DECLARATION, not a hole: same doctrine as an explicit `NONE` in
# ci/touched-requirements.sh's path map. Empty today, and these two hold vacuously when it is
# — they exist so it cannot be widened into a blanket without failing.
check(all(zone_label(t) for t in declared_off_raw),
      f"(11) every declared off-cluster link is a specific home-zone hostname in TOKEN form "
      f"({declared_off_raw})")
shadowed = sorted(declared_off & served)
check(not shadowed,
      f"(11) no off-cluster declaration shadows a hostname the repo DOES serve "
      f"{shadowed if shadowed else ''}")

# ============================ (12) tokens, never literals
literal_hosts = sorted({h for h in all_hosts if is_literal_base(h)})
check(not literal_hosts,
      f"(12) no link on the root names a LITERAL '{LITERAL_BASE}' host — every hostname is the "
      f"${{BASE_DOMAIN}} token, so prod's front door cannot link dev's services (the "
      f"2026-09-04 class: two clusters authored to describe the same names). Literals: "
      f"{literal_hosts if literal_hosts else ''}")

# =====================================================================
# R-C2-02 — platform health, and controlled operational actions
# =====================================================================
# docs/12 names the surfaces the C2 composes: Headlamp is "the R-C2-02 'controlled
# operational actions where safe' surface"; Grafana is the dashboards/alerts surface. Each is
# identified by the component that OWNS ITS ROUTE, not by a magic string in the config.
CONSOLE, DASHBOARDS = "headlamp", "grafana"
check(CONSOLE in linked,
      f"(13) the C2 root links the Kubernetes console ({CONSOLE}.{TOKEN}) — cluster, storage "
      f"and GitOps state, and the R-C2-02 operational-actions surface (docs/12)")
check(DASHBOARDS in linked,
      f"(13) the C2 root links the dashboards/alerts surface ({DASHBOARDS}.{TOKEN})")

AGG = ("\n".join(read(ROOT / t / "kustomization.yaml")
                 for t in ("platform", "platform-config", "platform-config-cdi")))
for label, name in (("console", CONSOLE), ("dashboards", DASHBOARDS)):
    comp = routed_labels.get(name)
    check(comp is not None, f"(14) the {label} surface has an in-repo HTTPRoute")
    if comp is None:
        continue
    rel = f"./{comp.relative_to(ROOT)}"
    reconciled = any((d.get("spec", {}) or {}).get("path", "").rstrip("/") == rel
                     for _, d in flux_ks)
    if not reconciled:
        # A component may reconcile as a MEMBER of an aggregate tier's kustomize tree
        # (platform / platform-config) instead of owning a Flux Kustomization. Both count;
        # being in NEITHER is "authored, never applied".
        reconciled = (f"\n  - {comp.name}" in AGG
                      or f"../{comp.relative_to(ROOT)}/" in AGG)
    check(reconciled,
          f"(14) the {label} surface ({comp.name}) is reconciled by Flux, not merely authored "
          f"— a link to an unreconciled component is a link to nothing")

# (15) the health DATA exists as code, for every axis R-C2-02 enumerates.
rule_files = sorted({p for p, d in docs_all() if d.get("kind") == "PrometheusRule"})
rule_groups = {g.get("name", "")
               for p, d in docs_all() if d.get("kind") == "PrometheusRule"
               for g in ((d.get("spec", {}) or {}).get("groups") or [])}
for axis, needle in (("cluster/nodes", "nodes"),
                     ("storage", "longhorn"),
                     ("GitOps reconciliation", "flux"),
                     ("certificate expiry", "certificates")):
    check(any(needle in g for g in rule_groups),
          f"(15) platform-health alert rules exist for {axis} (a '*{needle}*' rule group) — "
          f"the C2 cannot surface health that is not measured")
pc_txt = read(ROOT / "platform-config" / "kustomization.yaml")
reconciled_rules = [p for p in rule_files if f"../{p.relative_to(ROOT)}" in pc_txt]
check(len(reconciled_rules) >= 2,
      f"(15) those alert rules are RECONCILED — PrometheusRule is a CR of a CRD the platform "
      f"tier installs, so the rules are applied by platform-config "
      f"({len(reconciled_rules)}/{len(rule_files)} rule file(s) referenced there)")

# (16)(17) controlled actions: no shared cluster-admin identity; per-user OIDC instead.
console_dir = routed_labels.get(CONSOLE)
hr = next((d for p in (sorted(console_dir.glob("*.yaml")) if console_dir else [])
           for d in (DOCS.get(p) or []) if d.get("kind") == "HelmRelease"), None)
check(hr is not None, "(16) the Kubernetes console is deployed by a HelmRelease")
hv = ((hr or {}).get("spec", {}) or {}).get("values", {}) or {}
check((hv.get("clusterRoleBinding") or {}).get("create") is False,
      "(16) the console chart is told NOT to bind its ServiceAccount to a ClusterRole. The "
      "chart's DEFAULT binds it to cluster-admin, which is exactly the 'raw cluster-admin "
      "kubeconfig in a browser' docs/12 forbids — every visitor would act as one shared "
      "omnipotent identity")
oidc = ((hv.get("config") or {}).get("oidc") or {})
check(bool(oidc),
      "(17) the console is configured for native OIDC login, so an action carries the "
      "operator's OWN identity (docs/12: Headlamp/Grafana use native OIDC)")
check(bool((oidc.get("externalSecret") or {}).get("name")),
      "(17) its OIDC client credentials come from a NAMED (SOPS) Secret, not inline in the "
      "HelmRelease values — R-SEC-01")

# (18) rights arrive through GROUPS, at more than one privilege level.
group_bindings = []
for p, d in docs_all():
    if d.get("kind") not in ("ClusterRoleBinding", "RoleBinding"):
        continue
    for s in d.get("subjects") or []:
        if s.get("kind") == "Group" and str(s.get("name", "")).startswith("oidc:"):
            group_bindings.append(((d.get("metadata", {}) or {}).get("name"),
                                   s.get("name"),
                                   (d.get("roleRef") or {}).get("name")))
check(len(group_bindings) >= 2,
      f"(18) cluster rights are bound to OIDC GROUPS, so access follows Keycloak group "
      f"membership rather than a per-user grant somebody must remember to revoke "
      f"({[(g, r) for _, g, r in group_bindings]})")
check(any(r == "cluster-admin" for _, _, r in group_bindings)
      and any(r != "cluster-admin" for _, _, r in group_bindings),
      "(18) MORE THAN ONE privilege level exists (an admin group AND a lesser one) — "
      "least-privilege, not one tier for everybody (docs/12 access control, R-IDP-06)")

# (19) the root DECLARES the authenticated-group posture. DECLARED, NOT ENFORCED.
route_ann = (apex_route.get("metadata", {}) or {}).get("annotations", {}) or {}
check(route_ann.get("homelab.keithjackson.dev/auth-mode") in ("forward-auth", "oidc", "oauth2-proxy"),
      f"(19) the C2 root's route DECLARES an authentication mode "
      f"(found {route_ann.get('homelab.keithjackson.dev/auth-mode')!r}) "
      f"[DECLARED, NOT ENFORCED — no Gateway ext-authz attachment exists in this repo]")
check(bool(route_ann.get("homelab.keithjackson.dev/auth-groups")),
      "(19) it DECLARES the authorized group(s) — docs/12 requires an authenticated user IN "
      "AN AUTHORIZED GROUP, not merely logged in "
      "[DECLARED, NOT ENFORCED — see the header's gap note]")

# ======================================================================== report
print()
print(f"C2 surface inventory (apex {TOKEN}, literal base default '{LITERAL_BASE}'):")
print(f"  routed over HTTP : {len(routed_labels)}  -> {', '.join(sorted(routed_labels))}")
print(f"  L4-published     : {len(l4_labels)}  -> {', '.join(sorted(l4_labels)) or '(none)'}")
print(f"  linked from root : {len(linked)}  -> {', '.join(sorted(linked))}")
print(f"  apex-path routes : {len(apex_inheriting)}  -> "
      f"{', '.join(str(p.relative_to(ROOT)) for p in apex_inheriting) or '(none)'}")
print(f"  catalog services compared: {cat_seen}   off-cluster declarations: "
      f"{sorted(declared_off_raw) or '(none)'}")

if ADVISORIES:
    print("\nADVISORIES (not failures):")
    for a in ADVISORIES:
        print(f"  - {a}")

print()
if not FAILURES:
    print(f"check OK: c2-root-surface — {CHECKS} assertions (R-C2-01, R-C2-02)")
    raise SystemExit(0)
print(f"CHECK FAIL (c2-root-surface): {len(FAILURES)} failed, "
      f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
for f in FAILURES:
    print(f"  FAIL {f}", file=sys.stderr)
raise SystemExit(1)
PY