#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — tls-lifecycle-automated # Serves: R-TLS-02 ("Certificates MUST be automatically **issued, renewed, and revoked** # without manual steps." — spec docs/00-vision-and-scope.md; mechanism in # docs/08-tls-and-certificates.md, ADR-0006.) # # WHICH CLAUSE THIS LAYER CAN PROVE — AND WHICH IT CANNOT # ------------------------------------------------------ # R-TLS-02 has one adverbial clause ("without manual steps") and three verbs # ("issued", "renewed", "revoked"). In a GitOps platform the *without manual steps* # clause IS a repository property, and it is the half that can be proven with no # cluster: if a human had to mint, install, or approve a serving certificate, the # evidence of that human step would be IN THE REPO — a committed `tls.crt`, a Gateway # pointing at a Secret nothing declares, an issuer that no automated solver can # satisfy, a per-cert credential handed over out of band, or an admission path that # waits for a manual CertificateRequest approval. This check asserts that none of # those exist and that the automated path is completely wired instead. # # The three VERBS are runtime events. They are asserted by the companion L2 test # (conformance/tests/l2-behavioral/cert-lifecycle-automated), which observes an # issuance, an unattended renewal, and the withdrawal of a removed certificate. This # check does NOT claim them; see "WHAT THIS DOES *NOT* PROVE" below. # # GIVEN the repository as git tracks it, # WHEN every serving-TLS reference, Certificate, Issuer and the cert-manager release # are inspected offline, # THEN (1) at least one cert-manager `Certificate` is authored (vacuity floor: a repo # with no Certificate cannot pass this check by having nothing to check); # (2) every Gateway HTTPS listener that TERMINATES TLS references ONLY Secrets # that a tracked Certificate declares as its `secretName` — so the serving # credential is produced by the controller, never dropped in by hand; # (3) NO certificate material is tracked anywhere: no Secret manifest of type # kubernetes.io/tls, no tls.crt/tls.key/ca.crt key in any committed Secret # (plain OR SOPS-encrypted), and no PEM certificate body in any tracked # YAML — i.e. issuance is not, and cannot silently become, a human step; # (4) every Certificate's `issuerRef` resolves to a TRACKED issuer IN EVERY # ENVIRONMENT — the `${ACME_CLUSTER_ISSUER}` token is resolved against the # base cluster-vars default AND every overlay's value, so an environment # whose issuer does not exist is caught here rather than at prod bring-up; # (5) every issuer so referenced is an ACME issuer with a NON-EMPTY solver list # whose every solver is an automated challenge type (dns01 / http01) — the # same code path serves issuance AND renewal, so a solver a human must # complete would defeat both; # (6) each dns01 solver's provider credential is referenced BY NAME and that # Secret is delivered SOPS-encrypted from git via a kustomization — the # operator's one-time token hand-off (docs/08 OQ-4), never a per-cert step; # (7) cert-manager's automatic CertificateRequest approval is NOT disabled # (`disableAutoApproval`), and any `approveSignerNames` narrowing still # covers the ClusterIssuers actually used — a request that waits for a # human approver is the definition of a manual step; # (8) the whole chain is reconciled from git — cert-manager's HelmRelease, the # ClusterIssuers and the Certificate are each included in a kustomization # that a Flux Kustomization renders — so a deleted Certificate is # re-created and re-issued with no human action; # (9) no Certificate defeats renewal arithmetically (`renewBefore` >= `duration` # would mean the renewal instant never arrives). # # WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them): # - ISSUED: that an issuance actually happens. Asserted at L2 (step 2 of # l2-behavioral/cert-lifecycle-automated) and, for the real ACME/DNS-01 path, # evidenced by l2-behavioral/trusted-tls serving a publicly-trusted cert. # - RENEWED: that a renewal actually happens unattended. docs/08's acceptance gate # is explicit that this must be "observed across a renewal" — L2 step 3 does that # by fast-forwarding a certificate's renewal window; nothing static can. # - REVOKED: NOT PROVEN AT ANY LAYER TODAY, and not asserted here. Two distinct # readings, neither established: (a) ACME revocation — cert-manager does not # revoke certificates at all, so no repo state could show it; (b) the spec's own # reading in docs/08 ("removing/rotating a certificate is a declarative change") # — withdrawal of a REMOVED certificate's serving material requires cert-manager's # `enableCertificateOwnerRef`, which the tracked HelmRelease does NOT set (chart # default false). L2 step 4 asserts the withdrawal behaviour and is expected to # FAIL until that knob is set; escalated to the PM rather than softened (R-TEST-06). # - That the tracked Certificate is the one the live Gateway is actually serving # (cluster state), or that the ACME account/DNS credential is still valid. # - Certificate EXPIRY ALERTING, which docs/08 delegates to R-OBS-04 (see # l3-fault-injection/alerting). # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root PYTHON="${PYTHON:-python3}" cd "${ROOT}" "${PYTHON}" - "${ROOT}" <<'PY' import re import subprocess import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) FAILURES: list[str] = [] CHECKS = 0 def check(cond, msg): global CHECKS CHECKS += 1 if not cond: FAILURES.append(msg) def tracked() -> list[str]: out = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"], capture_output=True, text=True, check=True).stdout return [p for p in out.split("\0") if p] FILES = tracked() check(len(FILES) > 50, f"git ls-files returned only {len(FILES)} paths — not a real checkout") YAML_FILES = [f for f in FILES if re.search(r"\.(ya?ml)$", f)] def docs_of(rel: str): """Parsed YAML documents of a tracked file, or [] when it is not parseable YAML.""" try: return [d for d in yaml.safe_load_all((ROOT / rel).read_text(errors="replace")) if isinstance(d, dict)] except Exception: return [] DOCS: list[tuple[str, dict]] = [(rel, d) for rel in YAML_FILES for d in docs_of(rel)] def of_kind(kind: str, group_prefix: str = "") -> list[tuple[str, dict]]: out = [] for rel, d in DOCS: if d.get("kind") != kind: continue if group_prefix and not str(d.get("apiVersion", "")).startswith(group_prefix): continue out.append((rel, d)) return out def ns_of(d: dict, default: str = "default") -> str: return (d.get("metadata") or {}).get("namespace") or default def name_of(d: dict) -> str: return (d.get("metadata") or {}).get("name") or "" # ------------------------------------------------------------ cluster-vars resolution # Every environment's value for a ${TOKEN}: the base default plus each overlay's JSON6902 # `- op: replace / path: /data/KEY / value: ...` patch on the cluster-vars ConfigMap. This # is the same mechanism flux-postbuild-substitution audits; here we need the VALUES, so an # issuer name can be resolved per environment instead of being asserted for dev only. def env_values(key: str) -> dict[str, str]: vals: dict[str, str] = {} base = ROOT / "clusters" / "base" / "cluster-vars.yaml" if base.is_file(): for d in yaml.safe_load_all(base.read_text()): if isinstance(d, dict) and d.get("kind") == "ConfigMap" \ and name_of(d) == "cluster-vars": v = (d.get("data") or {}).get(key) if v is not None: vals["base"] = str(v) for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")): env = overlay.parent.name if env == "base": continue try: k = yaml.safe_load(overlay.read_text()) or {} except Exception: continue for p in (k.get("patches") or []): tgt = p.get("target") or {} if tgt.get("kind") != "ConfigMap" or tgt.get("name") != "cluster-vars": continue try: ops = yaml.safe_load(p.get("patch") or "") or [] except Exception: continue for op in ops if isinstance(ops, list) else []: if isinstance(op, dict) and str(op.get("path", "")) == f"/data/{key}": vals[env] = str(op.get("value")) return vals TOKEN_RE = re.compile(r"^\$\{([A-Z0-9_]+)\}$") def resolve_per_env(value: str) -> dict[str, str]: """A literal resolves to itself in every environment; a ${TOKEN} to each env's value.""" m = TOKEN_RE.match(value.strip()) if not m: return {"(literal)": value.strip()} return env_values(m.group(1)) # --------------------------------------------- what Flux actually renders (reachability) # A manifest is DELIVERED only if it is reachable from a Flux Kustomization's `path` through # the kustomize resource graph: /kustomization.yaml -> resources -> (file | directory # -> that directory's kustomization.yaml -> ...). Membership is deliberately computed by # walking that graph rather than by asking "is this file's DIRECTORY mentioned somewhere": # the loose form silently passes when an entry is deleted from the very kustomization.yaml # that the directory reference renders (verified — it made two assertions here no-ops). KUST_RESOURCES: dict[str, list[str]] = {} for rel in [f for f in YAML_FILES if f.endswith("kustomization.yaml")]: refs: list[str] = [] for d in docs_of(rel): for r in (d.get("resources") or []) + (d.get("components") or []): if not isinstance(r, str) or "://" in r: # skip remote bases continue try: p = (ROOT / rel).parent.joinpath(r).resolve() refs.append(str(p.relative_to(ROOT))) except Exception: continue KUST_RESOURCES[rel] = refs FLUX_PATHS: set[str] = set() for _rel, _d in DOCS: if _d.get("kind") == "Kustomization" and \ str(_d.get("apiVersion", "")).startswith("kustomize.toolkit.fluxcd.io/"): p = str((_d.get("spec") or {}).get("path", "")).lstrip("./").rstrip("/") if p: FLUX_PATHS.add(p) def _walk(entry: str, seen: set[str], out: set[str]) -> None: if entry in seen: return seen.add(entry) kust = entry if entry.endswith("kustomization.yaml") else f"{entry}/kustomization.yaml" if kust in KUST_RESOURCES: # a directory (or a kustomization) for r in KUST_RESOURCES[kust]: _walk(r, seen, out) return out.add(entry) # a leaf manifest RENDERED: set[str] = set() for _p in FLUX_PATHS: _walk(_p, set(), RENDERED) def delivered(rel: str) -> bool: """True iff Flux renders this file from one of its Kustomization paths.""" return rel in RENDERED # --------------------------------------------------- (1) there is something to check CERTS = of_kind("Certificate", "cert-manager.io/") check(len(CERTS) >= 1, "no cert-manager.io Certificate is authored anywhere in the tracked tree — every " "assertion below would be vacuous, so this is a FAILURE, not a pass (R-TLS-02)") ISSUERS = of_kind("ClusterIssuer", "cert-manager.io/") + of_kind("Issuer", "cert-manager.io/") check(len(ISSUERS) >= 1, "no cert-manager ClusterIssuer/Issuer is authored — nothing could issue automatically") # ---------------------------- (2) every terminating listener serves a controller cert CERT_SECRETS: dict[tuple[str, str], str] = { # (namespace, secretName) -> file (ns_of(d, "default"), str((d.get("spec") or {}).get("secretName", ""))): rel for rel, d in CERTS } listeners_checked = 0 for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"): gw_ns = ns_of(gw) for lis in ((gw.get("spec") or {}).get("listeners") or []): tls = lis.get("tls") or {} if str(tls.get("mode", "")) != "Terminate": continue listeners_checked += 1 refs = tls.get("certificateRefs") or [] check(len(refs) > 0, f"{rel}: Gateway {gw_ns}/{name_of(gw)} listener {lis.get('name')!r} terminates " "TLS with NO certificateRefs — it can only serve a certificate someone puts " "there by hand") for r in refs: kind = str(r.get("kind", "Secret")) key = (str(r.get("namespace") or gw_ns), str(r.get("name"))) check(kind == "Secret", f"{rel}: listener {lis.get('name')!r} certificateRef kind is {kind!r}; only " "Secret refs can be traced to a cert-manager Certificate") check(key in CERT_SECRETS, f"{rel}: listener {lis.get('name')!r} serves Secret {key[0]}/{key[1]}, which " "NO tracked cert-manager Certificate declares as its secretName — that " "credential would have to be created by hand (R-TLS-02 'without manual steps')") check(listeners_checked > 0, "no Gateway listener with tls.mode=Terminate found in the tracked tree — there is no " "serving-TLS reference to trace, so (2) proved nothing") # --------------------------------- (3) no certificate material is tracked, anywhere TLS_KEYS = {"tls.crt", "tls.key", "ca.crt", "tls-combined.pem"} for rel, d in DOCS: if d.get("kind") != "Secret": continue stype = str(d.get("type", "")) check(stype != "kubernetes.io/tls", f"{rel}: a Secret of type kubernetes.io/tls is COMMITTED (metadata.name=" f"{name_of(d)}) — a serving certificate authored by a human, not issued") for field in ("data", "stringData"): for k in (d.get(field) or {}): check(str(k) not in TLS_KEYS, f"{rel}: committed Secret {name_of(d)} carries {field}.{k} — certificate " "material in git means a manual issuance step exists (R-TLS-02)") PEM_RE = re.compile(r"-----BEGIN (?:[A-Z ]+ )?CERTIFICATE-----") for rel in YAML_FILES: body = (ROOT / rel).read_text(errors="replace") # Scoped to YAML manifests deliberately: prose/docs may legitimately SHOW a PEM header # in an example, and a .md file is not applied to any cluster. A manifest cannot. check(not PEM_RE.search(body), f"{rel}: contains a PEM CERTIFICATE body — a hand-minted certificate in a manifest") # --------------------------- (4)+(5)+(6) the issuer chain is automated in EVERY env # The cert-manager release's namespace IS the controller's `--cluster-resource-namespace` # (chart default), i.e. where a ClusterIssuer's solver credential must live. Derived from the # HelmRelease, never written down here, so this holds if the release namespace ever changes. cm_releases = [(rel, d) for rel, d in of_kind("HelmRelease") if name_of(d) == "cert-manager"] CM_NS = ns_of(cm_releases[0][1], "cert-manager") if cm_releases else "cert-manager" ISSUER_INDEX = {(d.get("kind"), name_of(d)): (rel, d) for rel, d in ISSUERS} solver_creds: list[tuple[str, str, str, str]] = [] # (issuer, ns, secretName, ref field) issuers_verified: set[str] = set() for rel, cert in CERTS: spec = cert.get("spec") or {} ref = spec.get("issuerRef") or {} kind = str(ref.get("kind", "Issuer")) per_env = resolve_per_env(str(ref.get("name", ""))) check(len(per_env) > 0, f"{rel}: Certificate {name_of(cert)} issuerRef.name is a token with NO value in " "clusters/base/cluster-vars.yaml or any overlay — it would render empty and never issue") for env, iname in sorted(per_env.items()): hit = ISSUER_INDEX.get((kind, iname)) check(hit is not None, f"{rel}: Certificate {name_of(cert)} resolves issuerRef to {kind}/{iname} in " f"environment {env!r}, but no such issuer is tracked — that environment cannot " "issue anything without a hand-created issuer (R-TLS-02, R-ENV-02)") if hit is None: continue irel, issuer = hit issuers_verified.add(iname) ispec = issuer.get("spec") or {} acme = ispec.get("acme") check(isinstance(acme, dict), f"{irel}: {kind}/{iname} (used by {name_of(cert)} in env {env!r}) is not an ACME " f"issuer — its issuance/renewal path is {sorted(ispec)}; docs/08/ADR-0006 mandate " "ACME so both are unattended") if not isinstance(acme, dict): continue check(str(acme.get("server", "")).startswith("https://"), f"{irel}: {kind}/{iname} has no https ACME directory endpoint") check(bool((acme.get("privateKeySecretRef") or {}).get("name")), f"{irel}: {kind}/{iname} declares no privateKeySecretRef — the ACME account key " "would have to be provisioned by hand") solvers = acme.get("solvers") or [] check(len(solvers) > 0, f"{irel}: {kind}/{iname} declares NO solvers — every order would sit unsolved " "until a human completed the challenge") for s in solvers: types = [t for t in ("dns01", "http01") if t in s] check(len(types) == 1, f"{irel}: {kind}/{iname} has a solver with challenge types {sorted(s)} — only " "the automated dns01/http01 solvers satisfy 'without manual steps'") for provider, cfg in (s.get("dns01") or {}).items(): if not isinstance(cfg, dict): continue for field, val in cfg.items(): if isinstance(val, dict) and str(field).endswith(("SecretRef", "secretRef")): n = str(val.get("name", "")) check(bool(n), f"{irel}: {kind}/{iname} dns01 {provider}.{field} has no secret name") if n: # A ClusterIssuer resolves secret refs in the controller's # cluster-resource-namespace; a namespaced Issuer in its own. want_ns = CM_NS if kind == "ClusterIssuer" else ns_of(issuer, CM_NS) solver_creds.append((iname, want_ns, n, f"{provider}.{field}")) check(len(issuers_verified) > 0, "no issuer was reachable from any Certificate — (5) proved nothing") # (6) the solver credential is delivered, SOPS-encrypted, from git — never per-cert by hand. check(len(solver_creds) > 0, "no dns01 solver credential reference found — docs/08 (OQ-4) requires the scoped " "Cloudflare token to drive DNS-01, so this check would otherwise be vacuous") # Keyed by (namespace, name): a same-named Secret in ANOTHER namespace is NOT this credential. # (Verified: keying by name alone let the external-dns copy of `cloudflare-api-token` satisfy # the cert-manager reference, so deleting cert-manager's own copy passed — a silent no-op.) SECRET_DOCS = {(ns_of(d, CM_NS), name_of(d)): (rel, d) for rel, d in DOCS if d.get("kind") == "Secret"} for iname, sns, sname, field in solver_creds: hit = SECRET_DOCS.get((sns, sname)) check(hit is not None, f"{iname}: dns01 {field} references Secret {sname!r}, which no tracked manifest " f"delivers into namespace {sns!r} (the issuer's secret-resolution namespace) — the " "credential would be created by hand in-cluster (R-TLS-02)") if hit is None: continue srel, sdoc = hit check("sops" in sdoc, f"{srel}: Secret {sname} is the DNS-01 credential but is not SOPS-encrypted (R-SEC-01)") check(delivered(srel), f"{srel}: Secret {sname} is not rendered by any Flux Kustomization path — Flux would " "never deliver it, so issuance would wait on a manual kubectl apply") # ------------------------------- (7) automatic CertificateRequest approval is intact check(len(cm_releases) == 1, f"expected exactly one cert-manager HelmRelease, found {[r for r, _ in cm_releases]} — " "the controller that issues and renews must itself be GitOps-managed") for rel, hr in cm_releases: values = ((hr.get("spec") or {}).get("values") or {}) check(values.get("disableAutoApproval") in (None, False), f"{rel}: cert-manager values set disableAutoApproval={values.get('disableAutoApproval')!r} " "— every CertificateRequest would then wait for an approver, which is a manual step") signers = values.get("approveSignerNames") if signers is not None: pats = [str(s) for s in (signers if isinstance(signers, list) else [signers])] for iname in sorted(issuers_verified): ok = any(p in ("*", "*/*") or "clusterissuers.cert-manager.io" in p and (p.endswith("/*") or p.endswith(f"/{iname}")) for p in pats) check(ok, f"{rel}: approveSignerNames={pats} does not cover ClusterIssuer {iname} — " "requests for it would never be auto-approved") # The chart's own CRD install must be on, else the CRs below cannot even be applied. check((values.get("crds") or {}).get("enabled") is True, f"{rel}: cert-manager values do not enable CRD installation — the Certificate CRD " "would have to be installed by hand") # ------------------------------- (8) the whole chain is continuously reconciled from git check(len(FLUX_PATHS) > 0 and len(RENDERED) > 20, f"the kustomize resource graph resolved {len(RENDERED)} manifest(s) from {len(FLUX_PATHS)} " "Flux Kustomization path(s) — too few to be a real graph, so (8) would prove nothing") for rel, _ in cm_releases + CERTS + ISSUERS: check(delivered(rel), f"{rel} is not rendered by any Flux Kustomization path — nothing reconciles it, so a " "deleted Certificate/issuer/controller would stay deleted until a human re-applied it") # --------------------------------------------- (9) renewal is not defeated arithmetically def to_seconds(v) -> int | None: m = re.fullmatch(r"\s*(\d+)\s*([smh])\s*", str(v)) if not m: return None return int(m.group(1)) * {"s": 1, "m": 60, "h": 3600}[m.group(2)] renew_pairs = 0 for rel, cert in CERTS: spec = cert.get("spec") or {} dur, rb = to_seconds(spec.get("duration")), to_seconds(spec.get("renewBefore")) if dur is None or rb is None: continue renew_pairs += 1 check(rb < dur, f"{rel}: Certificate {name_of(cert)} sets renewBefore={spec.get('renewBefore')} >= " f"duration={spec.get('duration')} — the renewal instant never arrives") # --------------------------------------------------------------------------- verdict # NO-OP GUARD: this repo has been bitten by checks that degrade to a successful no-op. # The floor below is a count of assertions that MUST have run for the verdict to mean # anything; it is not a style rule. check(CHECKS >= 20, f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass " "would be meaningless") if FAILURES: print(f"CHECK FAIL (tls-lifecycle-automated): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr) for f in FAILURES: print(f" - {f}", file=sys.stderr) raise SystemExit(1) print(f"check OK: tls-lifecycle-automated — {CHECKS} assertions; {len(CERTS)} Certificate(s), " f"{len(ISSUERS)} issuer(s), {len(issuers_verified)} issuer(s) reached from a Certificate " f"across every environment, {listeners_checked} terminating listener(s), " f"{len(solver_creds)} dns01 credential ref(s), {renew_pairs} duration/renewBefore pair(s). " "R-TLS-02 'without manual steps' clause only — issued/renewed/revoked are RUNTIME verbs " "(see l2-behavioral/cert-lifecycle-automated; 'revoked' is NOT proven at any layer).") PY