home-network-k3s-impl/conformance/tests/l0-static/tls-lifecycle-automated/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — tls-lifecycle-automated
# Serves: R-TLS-02 ("Certificates MUST be automatically **issued, renewed, and revoked**
#         without manual steps." — spec docs/00-vision-and-scope.md; mechanism in
#         docs/08-tls-and-certificates.md, ADR-0006.)
#
# WHICH CLAUSE THIS LAYER CAN PROVE — AND WHICH IT CANNOT
# ------------------------------------------------------
# R-TLS-02 has one adverbial clause ("without manual steps") and three verbs
# ("issued", "renewed", "revoked"). In a GitOps platform the *without manual steps*
# clause IS a repository property, and it is the half that can be proven with no
# cluster: if a human had to mint, install, or approve a serving certificate, the
# evidence of that human step would be IN THE REPO — a committed `tls.crt`, a Gateway
# pointing at a Secret nothing declares, an issuer that no automated solver can
# satisfy, a per-cert credential handed over out of band, or an admission path that
# waits for a manual CertificateRequest approval. This check asserts that none of
# those exist and that the automated path is completely wired instead.
#
# The three VERBS are runtime events. They are asserted by the companion L2 test
# (conformance/tests/l2-behavioral/cert-lifecycle-automated), which observes an
# issuance, an unattended renewal, and the withdrawal of a removed certificate. This
# check does NOT claim them; see "WHAT THIS DOES *NOT* PROVE" below.
#
# GIVEN the repository as git tracks it,
# WHEN  every serving-TLS reference, Certificate, Issuer and the cert-manager release
#       are inspected offline,
# THEN  (1) at least one cert-manager `Certificate` is authored (vacuity floor: a repo
#           with no Certificate cannot pass this check by having nothing to check);
#       (2) every Gateway HTTPS listener that TERMINATES TLS references ONLY Secrets
#           that a tracked Certificate declares as its `secretName` — so the serving
#           credential is produced by the controller, never dropped in by hand;
#       (3) NO certificate material is tracked anywhere: no Secret manifest of type
#           kubernetes.io/tls, no tls.crt/tls.key/ca.crt key in any committed Secret
#           (plain OR SOPS-encrypted), and no PEM certificate body in any tracked
#           YAML — i.e. issuance is not, and cannot silently become, a human step;
#       (4) every Certificate's `issuerRef` resolves to a TRACKED issuer IN EVERY
#           ENVIRONMENT — the `${ACME_CLUSTER_ISSUER}` token is resolved against the
#           base cluster-vars default AND every overlay's value, so an environment
#           whose issuer does not exist is caught here rather than at prod bring-up;
#       (5) every issuer so referenced is an ACME issuer with a NON-EMPTY solver list
#           whose every solver is an automated challenge type (dns01 / http01) — the
#           same code path serves issuance AND renewal, so a solver a human must
#           complete would defeat both;
#       (6) each dns01 solver's provider credential is referenced BY NAME and that
#           Secret is delivered SOPS-encrypted from git via a kustomization — the
#           operator's one-time token hand-off (docs/08 OQ-4), never a per-cert step;
#       (7) cert-manager's automatic CertificateRequest approval is NOT disabled
#           (`disableAutoApproval`), and any `approveSignerNames` narrowing still
#           covers the ClusterIssuers actually used — a request that waits for a
#           human approver is the definition of a manual step;
#       (8) the whole chain is reconciled from git — cert-manager's HelmRelease, the
#           ClusterIssuers and the Certificate are each included in a kustomization
#           that a Flux Kustomization renders — so a deleted Certificate is
#           re-created and re-issued with no human action;
#       (9) no Certificate defeats renewal arithmetically (`renewBefore` >= `duration`
#           would mean the renewal instant never arrives).
#
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
#   - ISSUED: that an issuance actually happens. Asserted at L2 (step 2 of
#     l2-behavioral/cert-lifecycle-automated) and, for the real ACME/DNS-01 path,
#     evidenced by l2-behavioral/trusted-tls serving a publicly-trusted cert.
#   - RENEWED: that a renewal actually happens unattended. docs/08's acceptance gate
#     is explicit that this must be "observed across a renewal" — L2 step 3 does that
#     by fast-forwarding a certificate's renewal window; nothing static can.
#   - REVOKED: NOT PROVEN AT ANY LAYER TODAY, and not asserted here. Two distinct
#     readings, neither established: (a) ACME revocation — cert-manager does not
#     revoke certificates at all, so no repo state could show it; (b) the spec's own
#     reading in docs/08 ("removing/rotating a certificate is a declarative change")
#     — withdrawal of a REMOVED certificate's serving material requires cert-manager's
#     `enableCertificateOwnerRef`, which the tracked HelmRelease does NOT set (chart
#     default false). L2 step 4 asserts the withdrawal behaviour and is expected to
#     FAIL until that knob is set; escalated to the PM rather than softened (R-TEST-06).
#   - That the tracked Certificate is the one the live Gateway is actually serving
#     (cluster state), or that the ACME account/DNS credential is still valid.
#   - Certificate EXPIRY ALERTING, which docs/08 delegates to R-OBS-04 (see
#     l3-fault-injection/alerting).
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"

cd "${ROOT}"
"${PYTHON}" - "${ROOT}" <<'PY'
import re
import subprocess
import sys
from pathlib import Path

try:
    import yaml
except ImportError:                                  # pragma: no cover
    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
          file=sys.stderr)
    raise SystemExit(1)

ROOT = Path(sys.argv[1])
FAILURES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(msg)


def tracked() -> list[str]:
    out = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"],
                         capture_output=True, text=True, check=True).stdout
    return [p for p in out.split("\0") if p]


FILES = tracked()
check(len(FILES) > 50, f"git ls-files returned only {len(FILES)} paths — not a real checkout")
YAML_FILES = [f for f in FILES if re.search(r"\.(ya?ml)$", f)]


def docs_of(rel: str):
    """Parsed YAML documents of a tracked file, or [] when it is not parseable YAML."""
    try:
        return [d for d in yaml.safe_load_all((ROOT / rel).read_text(errors="replace"))
                if isinstance(d, dict)]
    except Exception:
        return []


DOCS: list[tuple[str, dict]] = [(rel, d) for rel in YAML_FILES for d in docs_of(rel)]


def of_kind(kind: str, group_prefix: str = "") -> list[tuple[str, dict]]:
    out = []
    for rel, d in DOCS:
        if d.get("kind") != kind:
            continue
        if group_prefix and not str(d.get("apiVersion", "")).startswith(group_prefix):
            continue
        out.append((rel, d))
    return out


def ns_of(d: dict, default: str = "default") -> str:
    return (d.get("metadata") or {}).get("namespace") or default


def name_of(d: dict) -> str:
    return (d.get("metadata") or {}).get("name") or "<unnamed>"


# ------------------------------------------------------------ cluster-vars resolution
# Every environment's value for a ${TOKEN}: the base default plus each overlay's JSON6902
# `- op: replace / path: /data/KEY / value: ...` patch on the cluster-vars ConfigMap. This
# is the same mechanism flux-postbuild-substitution audits; here we need the VALUES, so an
# issuer name can be resolved per environment instead of being asserted for dev only.
def env_values(key: str) -> dict[str, str]:
    vals: dict[str, str] = {}
    base = ROOT / "clusters" / "base" / "cluster-vars.yaml"
    if base.is_file():
        for d in yaml.safe_load_all(base.read_text()):
            if isinstance(d, dict) and d.get("kind") == "ConfigMap" \
                    and name_of(d) == "cluster-vars":
                v = (d.get("data") or {}).get(key)
                if v is not None:
                    vals["base"] = str(v)
    for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
        env = overlay.parent.name
        if env == "base":
            continue
        try:
            k = yaml.safe_load(overlay.read_text()) or {}
        except Exception:
            continue
        for p in (k.get("patches") or []):
            tgt = p.get("target") or {}
            if tgt.get("kind") != "ConfigMap" or tgt.get("name") != "cluster-vars":
                continue
            try:
                ops = yaml.safe_load(p.get("patch") or "") or []
            except Exception:
                continue
            for op in ops if isinstance(ops, list) else []:
                if isinstance(op, dict) and str(op.get("path", "")) == f"/data/{key}":
                    vals[env] = str(op.get("value"))
    return vals


TOKEN_RE = re.compile(r"^\$\{([A-Z0-9_]+)\}$")


def resolve_per_env(value: str) -> dict[str, str]:
    """A literal resolves to itself in every environment; a ${TOKEN} to each env's value."""
    m = TOKEN_RE.match(value.strip())
    if not m:
        return {"(literal)": value.strip()}
    return env_values(m.group(1))


# --------------------------------------------- what Flux actually renders (reachability)
# A manifest is DELIVERED only if it is reachable from a Flux Kustomization's `path` through
# the kustomize resource graph: <path>/kustomization.yaml -> resources -> (file | directory
# -> that directory's kustomization.yaml -> ...). Membership is deliberately computed by
# walking that graph rather than by asking "is this file's DIRECTORY mentioned somewhere":
# the loose form silently passes when an entry is deleted from the very kustomization.yaml
# that the directory reference renders (verified — it made two assertions here no-ops).
KUST_RESOURCES: dict[str, list[str]] = {}
for rel in [f for f in YAML_FILES if f.endswith("kustomization.yaml")]:
    refs: list[str] = []
    for d in docs_of(rel):
        for r in (d.get("resources") or []) + (d.get("components") or []):
            if not isinstance(r, str) or "://" in r:      # skip remote bases
                continue
            try:
                p = (ROOT / rel).parent.joinpath(r).resolve()
                refs.append(str(p.relative_to(ROOT)))
            except Exception:
                continue
    KUST_RESOURCES[rel] = refs

FLUX_PATHS: set[str] = set()
for _rel, _d in DOCS:
    if _d.get("kind") == "Kustomization" and \
            str(_d.get("apiVersion", "")).startswith("kustomize.toolkit.fluxcd.io/"):
        p = str((_d.get("spec") or {}).get("path", "")).lstrip("./").rstrip("/")
        if p:
            FLUX_PATHS.add(p)


def _walk(entry: str, seen: set[str], out: set[str]) -> None:
    if entry in seen:
        return
    seen.add(entry)
    kust = entry if entry.endswith("kustomization.yaml") else f"{entry}/kustomization.yaml"
    if kust in KUST_RESOURCES:                            # a directory (or a kustomization)
        for r in KUST_RESOURCES[kust]:
            _walk(r, seen, out)
        return
    out.add(entry)                                        # a leaf manifest


RENDERED: set[str] = set()
for _p in FLUX_PATHS:
    _walk(_p, set(), RENDERED)


def delivered(rel: str) -> bool:
    """True iff Flux renders this file from one of its Kustomization paths."""
    return rel in RENDERED


# --------------------------------------------------- (1) there is something to check
CERTS = of_kind("Certificate", "cert-manager.io/")
check(len(CERTS) >= 1,
      "no cert-manager.io Certificate is authored anywhere in the tracked tree — every "
      "assertion below would be vacuous, so this is a FAILURE, not a pass (R-TLS-02)")

ISSUERS = of_kind("ClusterIssuer", "cert-manager.io/") + of_kind("Issuer", "cert-manager.io/")
check(len(ISSUERS) >= 1,
      "no cert-manager ClusterIssuer/Issuer is authored — nothing could issue automatically")

# ---------------------------- (2) every terminating listener serves a controller cert
CERT_SECRETS: dict[tuple[str, str], str] = {           # (namespace, secretName) -> file
    (ns_of(d, "default"), str((d.get("spec") or {}).get("secretName", ""))): rel
    for rel, d in CERTS
}
listeners_checked = 0
for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"):
    gw_ns = ns_of(gw)
    for lis in ((gw.get("spec") or {}).get("listeners") or []):
        tls = lis.get("tls") or {}
        if str(tls.get("mode", "")) != "Terminate":
            continue
        listeners_checked += 1
        refs = tls.get("certificateRefs") or []
        check(len(refs) > 0,
              f"{rel}: Gateway {gw_ns}/{name_of(gw)} listener {lis.get('name')!r} terminates "
              "TLS with NO certificateRefs — it can only serve a certificate someone puts "
              "there by hand")
        for r in refs:
            kind = str(r.get("kind", "Secret"))
            key = (str(r.get("namespace") or gw_ns), str(r.get("name")))
            check(kind == "Secret",
                  f"{rel}: listener {lis.get('name')!r} certificateRef kind is {kind!r}; only "
                  "Secret refs can be traced to a cert-manager Certificate")
            check(key in CERT_SECRETS,
                  f"{rel}: listener {lis.get('name')!r} serves Secret {key[0]}/{key[1]}, which "
                  "NO tracked cert-manager Certificate declares as its secretName — that "
                  "credential would have to be created by hand (R-TLS-02 'without manual steps')")
check(listeners_checked > 0,
      "no Gateway listener with tls.mode=Terminate found in the tracked tree — there is no "
      "serving-TLS reference to trace, so (2) proved nothing")

# --------------------------------- (3) no certificate material is tracked, anywhere
TLS_KEYS = {"tls.crt", "tls.key", "ca.crt", "tls-combined.pem"}
for rel, d in DOCS:
    if d.get("kind") != "Secret":
        continue
    stype = str(d.get("type", ""))
    check(stype != "kubernetes.io/tls",
          f"{rel}: a Secret of type kubernetes.io/tls is COMMITTED (metadata.name="
          f"{name_of(d)}) — a serving certificate authored by a human, not issued")
    for field in ("data", "stringData"):
        for k in (d.get(field) or {}):
            check(str(k) not in TLS_KEYS,
                  f"{rel}: committed Secret {name_of(d)} carries {field}.{k} — certificate "
                  "material in git means a manual issuance step exists (R-TLS-02)")

PEM_RE = re.compile(r"-----BEGIN (?:[A-Z ]+ )?CERTIFICATE-----")
for rel in YAML_FILES:
    body = (ROOT / rel).read_text(errors="replace")
    # Scoped to YAML manifests deliberately: prose/docs may legitimately SHOW a PEM header
    # in an example, and a .md file is not applied to any cluster. A manifest cannot.
    check(not PEM_RE.search(body),
          f"{rel}: contains a PEM CERTIFICATE body — a hand-minted certificate in a manifest")

# --------------------------- (4)+(5)+(6) the issuer chain is automated in EVERY env
# The cert-manager release's namespace IS the controller's `--cluster-resource-namespace`
# (chart default), i.e. where a ClusterIssuer's solver credential must live. Derived from the
# HelmRelease, never written down here, so this holds if the release namespace ever changes.
cm_releases = [(rel, d) for rel, d in of_kind("HelmRelease") if name_of(d) == "cert-manager"]
CM_NS = ns_of(cm_releases[0][1], "cert-manager") if cm_releases else "cert-manager"

ISSUER_INDEX = {(d.get("kind"), name_of(d)): (rel, d) for rel, d in ISSUERS}
solver_creds: list[tuple[str, str, str, str]] = []     # (issuer, ns, secretName, ref field)
issuers_verified: set[str] = set()
for rel, cert in CERTS:
    spec = cert.get("spec") or {}
    ref = spec.get("issuerRef") or {}
    kind = str(ref.get("kind", "Issuer"))
    per_env = resolve_per_env(str(ref.get("name", "")))
    check(len(per_env) > 0,
          f"{rel}: Certificate {name_of(cert)} issuerRef.name is a token with NO value in "
          "clusters/base/cluster-vars.yaml or any overlay — it would render empty and never issue")
    for env, iname in sorted(per_env.items()):
        hit = ISSUER_INDEX.get((kind, iname))
        check(hit is not None,
              f"{rel}: Certificate {name_of(cert)} resolves issuerRef to {kind}/{iname} in "
              f"environment {env!r}, but no such issuer is tracked — that environment cannot "
              "issue anything without a hand-created issuer (R-TLS-02, R-ENV-02)")
        if hit is None:
            continue
        irel, issuer = hit
        issuers_verified.add(iname)
        ispec = issuer.get("spec") or {}
        acme = ispec.get("acme")
        check(isinstance(acme, dict),
              f"{irel}: {kind}/{iname} (used by {name_of(cert)} in env {env!r}) is not an ACME "
              f"issuer — its issuance/renewal path is {sorted(ispec)}; docs/08/ADR-0006 mandate "
              "ACME so both are unattended")
        if not isinstance(acme, dict):
            continue
        check(str(acme.get("server", "")).startswith("https://"),
              f"{irel}: {kind}/{iname} has no https ACME directory endpoint")
        check(bool((acme.get("privateKeySecretRef") or {}).get("name")),
              f"{irel}: {kind}/{iname} declares no privateKeySecretRef — the ACME account key "
              "would have to be provisioned by hand")
        solvers = acme.get("solvers") or []
        check(len(solvers) > 0,
              f"{irel}: {kind}/{iname} declares NO solvers — every order would sit unsolved "
              "until a human completed the challenge")
        for s in solvers:
            types = [t for t in ("dns01", "http01") if t in s]
            check(len(types) == 1,
                  f"{irel}: {kind}/{iname} has a solver with challenge types {sorted(s)} — only "
                  "the automated dns01/http01 solvers satisfy 'without manual steps'")
            for provider, cfg in (s.get("dns01") or {}).items():
                if not isinstance(cfg, dict):
                    continue
                for field, val in cfg.items():
                    if isinstance(val, dict) and str(field).endswith(("SecretRef", "secretRef")):
                        n = str(val.get("name", ""))
                        check(bool(n),
                              f"{irel}: {kind}/{iname} dns01 {provider}.{field} has no secret name")
                        if n:
                            # A ClusterIssuer resolves secret refs in the controller's
                            # cluster-resource-namespace; a namespaced Issuer in its own.
                            want_ns = CM_NS if kind == "ClusterIssuer" else ns_of(issuer, CM_NS)
                            solver_creds.append((iname, want_ns, n, f"{provider}.{field}"))

check(len(issuers_verified) > 0, "no issuer was reachable from any Certificate — (5) proved nothing")

# (6) the solver credential is delivered, SOPS-encrypted, from git — never per-cert by hand.
check(len(solver_creds) > 0,
      "no dns01 solver credential reference found — docs/08 (OQ-4) requires the scoped "
      "Cloudflare token to drive DNS-01, so this check would otherwise be vacuous")
# Keyed by (namespace, name): a same-named Secret in ANOTHER namespace is NOT this credential.
# (Verified: keying by name alone let the external-dns copy of `cloudflare-api-token` satisfy
# the cert-manager reference, so deleting cert-manager's own copy passed — a silent no-op.)
SECRET_DOCS = {(ns_of(d, CM_NS), name_of(d)): (rel, d)
               for rel, d in DOCS if d.get("kind") == "Secret"}
for iname, sns, sname, field in solver_creds:
    hit = SECRET_DOCS.get((sns, sname))
    check(hit is not None,
          f"{iname}: dns01 {field} references Secret {sname!r}, which no tracked manifest "
          f"delivers into namespace {sns!r} (the issuer's secret-resolution namespace) — the "
          "credential would be created by hand in-cluster (R-TLS-02)")
    if hit is None:
        continue
    srel, sdoc = hit
    check("sops" in sdoc,
          f"{srel}: Secret {sname} is the DNS-01 credential but is not SOPS-encrypted (R-SEC-01)")
    check(delivered(srel),
          f"{srel}: Secret {sname} is not rendered by any Flux Kustomization path — Flux would "
          "never deliver it, so issuance would wait on a manual kubectl apply")

# ------------------------------- (7) automatic CertificateRequest approval is intact
check(len(cm_releases) == 1,
      f"expected exactly one cert-manager HelmRelease, found {[r for r, _ in cm_releases]} — "
      "the controller that issues and renews must itself be GitOps-managed")
for rel, hr in cm_releases:
    values = ((hr.get("spec") or {}).get("values") or {})
    check(values.get("disableAutoApproval") in (None, False),
          f"{rel}: cert-manager values set disableAutoApproval={values.get('disableAutoApproval')!r} "
          "— every CertificateRequest would then wait for an approver, which is a manual step")
    signers = values.get("approveSignerNames")
    if signers is not None:
        pats = [str(s) for s in (signers if isinstance(signers, list) else [signers])]
        for iname in sorted(issuers_verified):
            ok = any(p in ("*", "*/*") or "clusterissuers.cert-manager.io" in p and
                     (p.endswith("/*") or p.endswith(f"/{iname}")) for p in pats)
            check(ok,
                  f"{rel}: approveSignerNames={pats} does not cover ClusterIssuer {iname} — "
                  "requests for it would never be auto-approved")
    # The chart's own CRD install must be on, else the CRs below cannot even be applied.
    check((values.get("crds") or {}).get("enabled") is True,
          f"{rel}: cert-manager values do not enable CRD installation — the Certificate CRD "
          "would have to be installed by hand")

# ------------------------------- (8) the whole chain is continuously reconciled from git
check(len(FLUX_PATHS) > 0 and len(RENDERED) > 20,
      f"the kustomize resource graph resolved {len(RENDERED)} manifest(s) from {len(FLUX_PATHS)} "
      "Flux Kustomization path(s) — too few to be a real graph, so (8) would prove nothing")
for rel, _ in cm_releases + CERTS + ISSUERS:
    check(delivered(rel),
          f"{rel} is not rendered by any Flux Kustomization path — nothing reconciles it, so a "
          "deleted Certificate/issuer/controller would stay deleted until a human re-applied it")

# --------------------------------------------- (9) renewal is not defeated arithmetically
def to_seconds(v) -> int | None:
    m = re.fullmatch(r"\s*(\d+)\s*([smh])\s*", str(v))
    if not m:
        return None
    return int(m.group(1)) * {"s": 1, "m": 60, "h": 3600}[m.group(2)]


renew_pairs = 0
for rel, cert in CERTS:
    spec = cert.get("spec") or {}
    dur, rb = to_seconds(spec.get("duration")), to_seconds(spec.get("renewBefore"))
    if dur is None or rb is None:
        continue
    renew_pairs += 1
    check(rb < dur,
          f"{rel}: Certificate {name_of(cert)} sets renewBefore={spec.get('renewBefore')} >= "
          f"duration={spec.get('duration')} — the renewal instant never arrives")

# --------------------------------------------------------------------------- verdict
# NO-OP GUARD: this repo has been bitten by checks that degrade to a successful no-op.
# The floor below is a count of assertions that MUST have run for the verdict to mean
# anything; it is not a style rule.
check(CHECKS >= 20,
      f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
      "would be meaningless")

if FAILURES:
    print(f"CHECK FAIL (tls-lifecycle-automated): {len(FAILURES)} of {CHECKS} assertions failed",
          file=sys.stderr)
    for f in FAILURES:
        print(f"  - {f}", file=sys.stderr)
    raise SystemExit(1)
print(f"check OK: tls-lifecycle-automated — {CHECKS} assertions; {len(CERTS)} Certificate(s), "
      f"{len(ISSUERS)} issuer(s), {len(issuers_verified)} issuer(s) reached from a Certificate "
      f"across every environment, {listeners_checked} terminating listener(s), "
      f"{len(solver_creds)} dns01 credential ref(s), {renew_pairs} duration/renewBefore pair(s). "
      "R-TLS-02 'without manual steps' clause only — issued/renewed/revoked are RUNTIME verbs "
      "(see l2-behavioral/cert-lifecycle-automated; 'revoked' is NOT proven at any layer).")
PY