#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — flux-postbuild-substitution # Serves: R-ENV-02 ("Same GitOps config promotes dev/test -> prod (authored once)." # — spec docs/02-environments-and-provisioning.md) # R-GITOPS-01 ("All desired state in git, reconciled by Flux." # — spec docs/05-gitops-and-delivery.md) # # WHY THIS EXISTS — Flux post-build substitution FAILS OPEN # -------------------------------------------------------- # R-ENV-02 is implemented by exactly one mechanism: `clusters/base/cluster-vars.yaml` # is injected into every `${TOKEN}` of the manifests each Flux Kustomization renders, # via `postBuild.substituteFrom`. That mechanism is silently lossy by default: an # UNDEFINED variable becomes the EMPTY STRING and the Kustomization still reports # Ready. On 2026-09-01 that emptied `k8sServiceHost` in the Cilium HelmRelease and put # cilium into Init:Error on 2 of 3 dev nodes (DECISIONS-LOG "substitution race that # fails OPEN"). "Authored once, promoted" is therefore only true while every token a # substituted path renders is either DEFINED or deliberately ESCAPED — and nothing in # the repo asserted that. This check does. # # ENVSUBST BEHAVIOUR ASSERTED HERE (empirically verified against the pinned toolchain, # `flux envsubst`, flux 2.4.0 — this is measured behaviour, not assumption): # ${DEFINED} -> substituted with the cluster-vars value. # ${UNDEFINED} -> EMPTY STRING, silently. `--strict` / the cluster-wide # StrictPostBuildSubstitutions feature gate turns this into a # loud Kustomization failure ("variable not set (strict mode)"). # ${UNDEFINED:-d} -> the DEFAULT is applied AT RECONCILE TIME. Strict mode does # NOT catch this (it exempts default-operator forms by design), # so a shell guard written `[ -z "${X:-}" ]` inside a rendered # manifest is pre-resolved to `[ -z "" ]` and its else-branch # becomes unreachable in the container. Silent, and permanent. # $${VAR}, $${V:-d}, $${ARR[0]} -> emitted as the LITERAL ${...} for a downstream # shell/cloud-init consumer. Escaping is total: it neutralises # even the indexed form. # $VAR, $(cmd) -> left alone by envsubst; they need no escaping. # ${...}, ${a b}, ${ARR[0]} (UNESCAPED) -> HARD PARSE ERROR ("unable to parse # variable name" / "missing closing brace") which fails the # WHOLE Kustomization — every resource in the path, not just # the offending one. # The escape contract this enforces is the one already documented in # `clusters/base/apps-vms.yaml`; this check is that comment made executable. # # GIVEN the repository as authored, # WHEN every Flux Kustomization under clusters/ that declares postBuild.substituteFrom # is resolved to the exact file set its `spec.path` renders (by following # kustomization.yaml `resources:` transitively — a path, not a glob), # THEN (1) every `${...}` in those files is DEFINED in cluster-vars, or ESCAPED `$${...}`; # (2) no UNESCAPED expansion carries a default/length/substitution operator on an # undefined name (the silent-empty class strict mode does not catch); # (3) no UNESCAPED expansion is unparseable by envsubst (`${...}`, `${ARR[0]}`) — # that fails the entire Kustomization, so it is fatal wherever it appears; # (4) every Kustomization that renders a cluster-vars token actually declares # `substituteFrom` a ConfigMap named cluster-vars (else the literal token # ships into the cluster); # (5) every knob an overlay patches (`/data/KEY`) exists in the BASE ConfigMap — # the "add the knob with a base default BEFORE referencing it" rule that the # incident's interim discipline depends on; # AND (6) this check is LOAD-BEARING: ci/l0.sh runs it on every gate run and # ci/gate.sh sets the merge block when the l0 stage fails. # Assertion (1)-(3) are applied to COMMENTS as well as content: `kustomize build` # preserves body comments into the stream envsubst reads (verified: a `$`-carrying # comment survives into the built apps/wave-1 output), so a braced expression in a # comment is a live hazard, not prose. # # WHAT THIS DOES *NOT* PROVE (honest gaps — this is a STATIC check): # - It does NOT render the manifests. It scans the SOURCE files a path includes, # which is a deliberate SUPERSET of the post-`kustomize build` stream (kustomize # drops some comments). A finding in a comment may not reach envsubst TODAY; it is # still reported, because whether a given comment survives the build is a kustomize # implementation detail that changes with the pinned version, and the failure it # causes is total (assertion 3). The end-to-end proof — `kustomize build | # flux envsubst --strict` for every substituted path — needs kustomize + flux, # which the L0 CI toolchain does not install (.gitea/workflows/scripts/install-tools.sh # l0), so it cannot run here without permanently SKIPPING. It was run by hand and # recorded in DECISIONS-LOG; this check is the reproducible static half. # - It does NOT prove the cluster's live state matches git, nor that the # StrictPostBuildSubstitutions feature gate is actually enabled on # kustomize-controller. Both are cluster facts (L2), untested here. Tagging # R-GITOPS-01 claims ONLY the static slice: what git says a Kustomization renders # is fully determined, with no variable silently emptied on the way in. # - It does NOT validate cluster-vars VALUES (that GATEWAY_VIP is inside the LB pool, # that an overlay's LAN block is self-consistent). Presence, not correctness. # - A cluster-vars key nothing references is reported as an ADVISORY, never a failure: # several are deliberately reference-only and say so in cluster-vars.yaml # (PIA_REGION "retained, not substituted"; CLUSTER_POD_CIDR "Documents ..."), so # failing on them would be a false block on documented intent. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root PYTHON="${PYTHON:-python3}" # Honest SKIP (never green) if the thing under test is not present at all. if [ ! -d "${ROOT}/clusters" ] || [ ! -f "${ROOT}/clusters/base/cluster-vars.yaml" ]; then echo "check SKIPPED: clusters/ or clusters/base/cluster-vars.yaml absent — nothing to audit" >&2 exit 3 fi cd "${ROOT}" "${PYTHON}" - "${ROOT}" <<'PY' """Static audit of Flux postBuild substitution safety. Pure standard library. No PyYAML: the file shapes parsed here are line-constrained and the parse contracts are documented at each parser, matching lib/coverage.py + lib/status.py. The kustomization `resources:` parser is IMPORTED from lib/status.py rather than forked, so there is one definition of "what a Flux path actually renders" in the harness. """ from __future__ import annotations import os import re import sys from pathlib import Path ROOT = Path(sys.argv[1]) sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree sys.path.insert(0, str(ROOT / "conformance" / "lib")) import status as st # noqa: E402 — reuse the walk, don't fork it FAILURES: list[str] = [] ADVISORIES: list[str] = [] CHECKS = 0 def check(cond, msg): global CHECKS CHECKS += 1 if not cond: FAILURES.append(msg) # ------------------------------------------------------------------ cluster-vars # Parse contract: `data:` block of a single ConfigMap, one ` KEY: "value"` per line # (every entry in clusters/base/cluster-vars.yaml is a quoted scalar; comments and # continuation-comment lines are indented further and never match). _DATA_KEY_RE = re.compile(r'^ ([A-Za-z_][A-Za-z0-9_]*):\s*(?:"|\')') def cluster_vars_keys(path: Path) -> set[str]: keys: set[str] = set() in_data = False for raw in path.read_text().splitlines(): if re.match(r"^data:\s*$", raw): in_data = True continue if not in_data: continue if st._TOP_KEY_RE.match(raw): # a new top-level key ends the data block break m = _DATA_KEY_RE.match(raw) if m: keys.add(m.group(1)) return keys VARS_FILE = ROOT / "clusters" / "base" / "cluster-vars.yaml" KEYS = cluster_vars_keys(VARS_FILE) check(len(KEYS) > 0, f"no data keys parsed from {VARS_FILE} — the parser or the file shape changed") # ------------------------------------------- Flux Kustomizations with substituteFrom # Parse contract mirrors lib/status.py's flux_paths(): docs split on a lone `---`, a doc # is a Flux entrypoint only if it carries the kustomize.toolkit apiVersion AND # `kind: Kustomization` AND a spec-level `path:`. Inline `patch: |-` bodies in the # overlays carry Flux Kustomizations WITHOUT a path and are correctly skipped. _SUBFROM_RE = re.compile(r"^(\s*)substituteFrom:\s*$", re.M) _SUBITEM_KIND_RE = re.compile(r"^\s*-\s*kind:\s*(\S+)") _SUBITEM_NAME_RE = re.compile(r"^\s*(?:-\s*)?name:\s*([^\s#]+)") def substitute_sources(doc: str) -> list[tuple[str, str]]: """[(kind, name)] declared under postBuild.substituteFrom in this doc.""" m = _SUBFROM_RE.search(doc) if not m: return [] indent = len(m.group(1)) out: list[tuple[str, str]] = [] kind = "" for raw in doc[m.end():].splitlines(): if raw.strip() == "" or raw.lstrip().startswith("#"): continue if len(raw) - len(raw.lstrip()) <= indent and not raw.lstrip().startswith("-"): break # dedented out of the list km = _SUBITEM_KIND_RE.match(raw) if km: kind = km.group(1) continue nm = _SUBITEM_NAME_RE.match(raw) if nm and kind: out.append((kind, nm.group(1))) kind = "" return out kustomizations: list[dict] = [] for yfile in sorted((ROOT / "clusters").rglob("*.yaml")): text = yfile.read_text() if st.FLUX_API not in text: continue for doc in st._DOC_SPLIT_RE.split(text): if st.FLUX_API not in doc or not re.search(r"^kind:\s*Kustomization\s*$", doc, re.M): continue pm = st._PATH_RE.search(doc) if not pm: continue nm = st._NAME_RE.search(doc) srcs = substitute_sources(doc) if not srcs: continue # no substitution => nothing to audit kustomizations.append({ "file": str(yfile.relative_to(ROOT)), "name": nm.group(1) if nm else "?", "path": pm.group(1), "sources": srcs, }) check(len(kustomizations) > 0, "no Flux Kustomization with postBuild.substituteFrom found under clusters/ — " "the parser or the repo layout changed; this check would silently audit nothing") # ------------------------------------------------------------------ token scanning # One `$`-run followed by a braced expression. `[^{}]*` is exhaustive for this tree: # a repo-wide sweep found no nested/unclosed brace expression in any manifest (only # Gitea Actions `${{ ... }}` in .gitea/, which no Flux Kustomization renders). TOKEN_RE = re.compile(r"(\$+)\{([^{}]*)\}") NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") # An expansion whose name is followed by a POSIX parameter operator. envsubst EVALUATES # these, so an undefined name yields the default/empty AT RECONCILE TIME. OP_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)(:[-=?+].*|:[0-9].*|/.*|#.*|%.*|\^.*|,.*)$") LEN_RE = re.compile(r"^#([A-Za-z_][A-Za-z0-9_]*)$") def classify(dollars: int, inner: str) -> tuple[str, str] | None: """Return (severity-tag, remediation) for a hazardous expansion, else None.""" if dollars >= 2: return None # escaped: emitted as a literal if NAME_RE.match(inner): if inner in KEYS: return None # defined knob — the happy path return ("UNDEFINED", f"strict mode fails the Kustomization; today it renders EMPTY. Either add " f"'{inner}' to clusters/base/cluster-vars.yaml with a base default, or " f"escape it as $${{{inner}}} if it is meant for a shell/cloud-init consumer") m = OP_RE.match(inner) or LEN_RE.match(inner) if m: base = m.group(1) if base in KEYS: return None # defined knob with an operator return ("SILENT-EMPTY", f"envsubst resolves this AT RECONCILE TIME because '{base}' is not a " f"cluster-var, so the rendered manifest carries the default, not the " f"expression. StrictPostBuildSubstitutions does NOT catch this form. " f"Escape it as $${{{inner}}} if the CONTAINER's shell is meant to expand it") return ("FATAL-PARSE", "envsubst cannot parse this and fails the ENTIRE Kustomization (every resource " "in the path). Escape it as $${...} or rewrite the prose without braces") def rendered_files(path_spec: str) -> tuple[list[str], bool]: """Repo-relative files the Flux `spec.path` renders, resolving `resources:` transitively.""" rel = os.path.normpath(path_spec.lstrip("./")) if path_spec not in (".", "./") else "." if not (ROOT / rel).is_dir(): return [], False applied, _ = st.reconciled_files(ROOT, rel) return sorted(applied), True findings: list[str] = [] referenced: dict[str, set[str]] = {} inventory: list[str] = [] for k in kustomizations: files, exists = rendered_files(k["path"]) check(exists, f"{k['file']}: Kustomization '{k['name']}' renders path '{k['path']}' " f"which is not a directory in this repo") has_cluster_vars = ("ConfigMap", "cluster-vars") in k["sources"] n_def = n_esc = n_bad = 0 for rel in files: f = ROOT / rel try: text = f.read_text() except (OSError, UnicodeDecodeError): continue for lineno, line in enumerate(text.splitlines(), 1): for m in TOKEN_RE.finditer(line): dollars, inner = len(m.group(1)), m.group(2) verdict = classify(dollars, inner) if verdict is None: if dollars >= 2: n_esc += 1 else: n_def += 1 referenced.setdefault(re.split(r"[:/#%^,]", inner)[0] or inner, set()).add(k["name"]) # (4) a DEFINED knob is only substituted if this Kustomization # actually pulls cluster-vars in; otherwise the literal ships. check(has_cluster_vars, f"{rel}:{lineno}: renders cluster-var ${{{inner}}} but " f"Kustomization '{k['name']}' ({k['file']}) does not " f"substituteFrom ConfigMap/cluster-vars — the literal token " f"would ship into the cluster") continue n_bad += 1 tag, fix = verdict findings.append(f" [{tag}] {rel}:{lineno} {m.group(0)}\n" f" rendered by Flux Kustomization '{k['name']}' " f"({k['file']}, path {k['path']})\n" f" {fix}\n" f" | {line.strip()[:120]}") inventory.append(f" {k['name']:<16} path={k['path']:<22} files={len(files):<3} " f"defined={n_def:<3} escaped={n_esc:<3} hazards={n_bad}") check(not findings, f"{len(findings)} unescaped, undefined or unparseable expansion(s) in substituted paths") # --------------------------------------------- (5) overlay knobs exist in the base # Parse contract: overlays patch cluster-vars with JSON6902 ops (`- op: replace` / # `path: /data/KEY`) inside an inline `patch: |-` block. The repo rule (cluster-vars.yaml # header) is that a knob is ADDED TO THE BASE with a default BEFORE any overlay or # manifest references it — that ordering is what closes the define-then-use race. _OP_PATH_RE = re.compile(r"^\s*(?:-\s*)?path:\s*/data/([A-Za-z_][A-Za-z0-9_]*)\s*$", re.M) for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")): for key in set(_OP_PATH_RE.findall(overlay.read_text())): check(key in KEYS, f"{overlay.relative_to(ROOT)}: patches /data/{key} but '{key}' has no BASE " f"default in clusters/base/cluster-vars.yaml — a second overlay would render " f"${{{key}}} empty (R-ENV-02: author once, supply per env)") # ------------------------------------------------------- (6) the check is load-bearing L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else "" GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else "" check("flux-postbuild-substitution/check.sh" in L0, "ci/l0.sh does not run flux-postbuild-substitution/check.sh — the requirement-scoped " "conformance selection can filter this check out of a gate run, so l0.sh must invoke " "it directly and unconditionally or the gate is not actually enforcing it") check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE), "ci/gate.sh does not set the merge block when the l0 stage fails") # ------------------------------------------------------------- advisory: dead knobs for key in sorted(KEYS - set(referenced)): ADVISORIES.append(f"cluster-vars knob '{key}' is referenced by NO reconciled manifest " f"(dead knob, or documentation-only — advisory, not a failure)") # ------------------------------------------------------------------------- report print("Flux postBuild substitution inventory (per Kustomization):") for line in inventory: print(line) print(f"\ncluster-vars knobs defined: {len(KEYS)} referenced: {len(referenced)}") if ADVISORIES: print("\nADVISORIES (not failures):") for a in ADVISORIES: print(f" - {a}") if findings: print("\nHAZARDS — unescaped expansions that are not cluster-vars knobs:", file=sys.stderr) for f in findings: print(f, file=sys.stderr) print() if not FAILURES: print(f"check OK: flux-postbuild-substitution — {CHECKS} assertions " f"(R-ENV-02, R-GITOPS-01)") raise SystemExit(0) print(f"CHECK FAIL (flux-postbuild-substitution): {len(FAILURES)} failed, " f"{CHECKS - len(FAILURES)} passed", file=sys.stderr) for f in FAILURES: print(f" FAIL {f}", file=sys.stderr) raise SystemExit(1) PY