home-network-k3s-impl/conformance/tests/l0-static/flux-postbuild-substitution/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — flux-postbuild-substitution
# Serves: R-ENV-02 ("Same GitOps config promotes dev/test -> prod (authored once)."
#                   — spec docs/02-environments-and-provisioning.md)
#         R-GITOPS-01 ("All desired state in git, reconciled by Flux."
#                   — spec docs/05-gitops-and-delivery.md)
#
# WHY THIS EXISTS — Flux post-build substitution FAILS OPEN
# --------------------------------------------------------
# R-ENV-02 is implemented by exactly one mechanism: `clusters/base/cluster-vars.yaml`
# is injected into every `${TOKEN}` of the manifests each Flux Kustomization renders,
# via `postBuild.substituteFrom`. That mechanism is silently lossy by default: an
# UNDEFINED variable becomes the EMPTY STRING and the Kustomization still reports
# Ready. On 2026-09-01 that emptied `k8sServiceHost` in the Cilium HelmRelease and put
# cilium into Init:Error on 2 of 3 dev nodes (DECISIONS-LOG "substitution race that
# fails OPEN"). "Authored once, promoted" is therefore only true while every token a
# substituted path renders is either DEFINED or deliberately ESCAPED — and nothing in
# the repo asserted that. This check does.
#
# ENVSUBST BEHAVIOUR ASSERTED HERE (empirically verified against the pinned toolchain,
# `flux envsubst`, flux 2.4.0 — this is measured behaviour, not assumption):
#   ${DEFINED}        -> substituted with the cluster-vars value.
#   ${UNDEFINED}      -> EMPTY STRING, silently. `--strict` / the cluster-wide
#                        StrictPostBuildSubstitutions feature gate turns this into a
#                        loud Kustomization failure ("variable not set (strict mode)").
#   ${UNDEFINED:-d}   -> the DEFAULT is applied AT RECONCILE TIME. Strict mode does
#                        NOT catch this (it exempts default-operator forms by design),
#                        so a shell guard written `[ -z "${X:-}" ]` inside a rendered
#                        manifest is pre-resolved to `[ -z "" ]` and its else-branch
#                        becomes unreachable in the container. Silent, and permanent.
#   $${VAR}, $${V:-d}, $${ARR[0]}  -> emitted as the LITERAL ${...} for a downstream
#                        shell/cloud-init consumer. Escaping is total: it neutralises
#                        even the indexed form.
#   $VAR, $(cmd)      -> left alone by envsubst; they need no escaping.
#   ${...}, ${a b}, ${ARR[0]}  (UNESCAPED) -> HARD PARSE ERROR ("unable to parse
#                        variable name" / "missing closing brace") which fails the
#                        WHOLE Kustomization — every resource in the path, not just
#                        the offending one.
# The escape contract this enforces is the one already documented in
# `clusters/base/apps-vms.yaml`; this check is that comment made executable.
#
# GIVEN the repository as authored,
# WHEN  every Flux Kustomization under clusters/ that declares postBuild.substituteFrom
#       is resolved to the exact file set its `spec.path` renders (by following
#       kustomization.yaml `resources:` transitively — a path, not a glob),
# THEN  (1) every `${...}` in those files is DEFINED in cluster-vars, or ESCAPED `$${...}`;
#       (2) no UNESCAPED expansion carries a default/length/substitution operator on an
#           undefined name (the silent-empty class strict mode does not catch);
#       (3) no UNESCAPED expansion is unparseable by envsubst (`${...}`, `${ARR[0]}`) —
#           that fails the entire Kustomization, so it is fatal wherever it appears;
#       (4) every Kustomization that renders a cluster-vars token actually declares
#           `substituteFrom` a ConfigMap named cluster-vars (else the literal token
#           ships into the cluster);
#       (5) every knob an overlay patches (`/data/KEY`) exists in the BASE ConfigMap —
#           the "add the knob with a base default BEFORE referencing it" rule that the
#           incident's interim discipline depends on;
#   AND (6) this check is LOAD-BEARING: it DECLARES `conformance.home.k3s/scope: whole-repo`,
#           so ci/l0.sh's `--include-scope whole-repo` runs it on every gate run, and
#           ci/gate.sh sets the merge block when the l0 stage fails.
# Assertion (1)-(3) are applied to COMMENTS as well as content: `kustomize build`
# preserves body comments into the stream envsubst reads (verified: a `$`-carrying
# comment survives into the built apps/wave-1 output), so a braced expression in a
# comment is a live hazard, not prose.
#
# WHAT THIS DOES *NOT* PROVE (honest gaps — this is a STATIC check):
#   - It does NOT render the manifests. It scans the SOURCE files a path includes,
#     which is a deliberate SUPERSET of the post-`kustomize build` stream (kustomize
#     drops some comments). A finding in a comment may not reach envsubst TODAY; it is
#     still reported, because whether a given comment survives the build is a kustomize
#     implementation detail that changes with the pinned version, and the failure it
#     causes is total (assertion 3). The end-to-end proof — `kustomize build <path> |
#     flux envsubst --strict` for every substituted path — needs kustomize + flux,
#     which the L0 CI toolchain does not install (.gitea/workflows/scripts/install-tools.sh
#     l0), so it cannot run here without permanently SKIPPING. It was run by hand and
#     recorded in DECISIONS-LOG; this check is the reproducible static half.
#   - It does NOT prove the cluster's live state matches git, nor that the
#     StrictPostBuildSubstitutions feature gate is actually enabled on
#     kustomize-controller. Both are cluster facts (L2), untested here. Tagging
#     R-GITOPS-01 claims ONLY the static slice: what git says a Kustomization renders
#     is fully determined, with no variable silently emptied on the way in.
#   - It does NOT validate cluster-vars VALUES (that GATEWAY_VIP is inside the LB pool,
#     that an overlay's LAN block is self-consistent). Presence, not correctness.
#   - A cluster-vars key nothing references is reported as an ADVISORY, never a failure:
#     several are deliberately reference-only and say so in cluster-vars.yaml
#     (PIA_REGION "retained, not substituted"; CLUSTER_POD_CIDR "Documents ..."), so
#     failing on them would be a false block on documented intent.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"

# Honest SKIP (never green) if the thing under test is not present at all.
if [ ! -d "${ROOT}/clusters" ] || [ ! -f "${ROOT}/clusters/base/cluster-vars.yaml" ]; then
  echo "check SKIPPED: clusters/ or clusters/base/cluster-vars.yaml absent — nothing to audit" >&2
  exit 3
fi

cd "${ROOT}"
"${PYTHON}" - "${ROOT}" <<'PY'
"""Static audit of Flux postBuild substitution safety. Pure standard library.

No PyYAML: the file shapes parsed here are line-constrained and the parse contracts are
documented at each parser, matching lib/coverage.py + lib/status.py. The kustomization
`resources:` parser is IMPORTED from lib/status.py rather than forked, so there is one
definition of "what a Flux path actually renders" in the harness.
"""
from __future__ import annotations

import os
import re
import sys
from pathlib import Path

ROOT = Path(sys.argv[1])
sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree
sys.path.insert(0, str(ROOT / "conformance" / "lib"))
import status as st                     # noqa: E402 — reuse the walk, don't fork it

FAILURES: list[str] = []
ADVISORIES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(msg)


# ------------------------------------------------------------------ cluster-vars
# Parse contract: `data:` block of a single ConfigMap, one `  KEY: "value"` per line
# (every entry in clusters/base/cluster-vars.yaml is a quoted scalar; comments and
# continuation-comment lines are indented further and never match).
_DATA_KEY_RE = re.compile(r'^  ([A-Za-z_][A-Za-z0-9_]*):\s*(?:"|\')')


def cluster_vars_keys(path: Path) -> set[str]:
    keys: set[str] = set()
    in_data = False
    for raw in path.read_text().splitlines():
        if re.match(r"^data:\s*$", raw):
            in_data = True
            continue
        if not in_data:
            continue
        if st._TOP_KEY_RE.match(raw):   # a new top-level key ends the data block
            break
        m = _DATA_KEY_RE.match(raw)
        if m:
            keys.add(m.group(1))
    return keys


VARS_FILE = ROOT / "clusters" / "base" / "cluster-vars.yaml"
KEYS = cluster_vars_keys(VARS_FILE)
check(len(KEYS) > 0, f"no data keys parsed from {VARS_FILE} — the parser or the file shape changed")

# ------------------------------------------- Flux Kustomizations with substituteFrom
# Parse contract mirrors lib/status.py's flux_paths(): docs split on a lone `---`, a doc
# is a Flux entrypoint only if it carries the kustomize.toolkit apiVersion AND
# `kind: Kustomization` AND a spec-level `path:`. Inline `patch: |-` bodies in the
# overlays carry Flux Kustomizations WITHOUT a path and are correctly skipped.
_SUBFROM_RE = re.compile(r"^(\s*)substituteFrom:\s*$", re.M)
_SUBITEM_KIND_RE = re.compile(r"^\s*-\s*kind:\s*(\S+)")
_SUBITEM_NAME_RE = re.compile(r"^\s*(?:-\s*)?name:\s*([^\s#]+)")


def substitute_sources(doc: str) -> list[tuple[str, str]]:
    """[(kind, name)] declared under postBuild.substituteFrom in this doc."""
    m = _SUBFROM_RE.search(doc)
    if not m:
        return []
    indent = len(m.group(1))
    out: list[tuple[str, str]] = []
    kind = ""
    for raw in doc[m.end():].splitlines():
        if raw.strip() == "" or raw.lstrip().startswith("#"):
            continue
        if len(raw) - len(raw.lstrip()) <= indent and not raw.lstrip().startswith("-"):
            break                                    # dedented out of the list
        km = _SUBITEM_KIND_RE.match(raw)
        if km:
            kind = km.group(1)
            continue
        nm = _SUBITEM_NAME_RE.match(raw)
        if nm and kind:
            out.append((kind, nm.group(1)))
            kind = ""
    return out


kustomizations: list[dict] = []
for yfile in sorted((ROOT / "clusters").rglob("*.yaml")):
    text = yfile.read_text()
    if st.FLUX_API not in text:
        continue
    for doc in st._DOC_SPLIT_RE.split(text):
        if st.FLUX_API not in doc or not re.search(r"^kind:\s*Kustomization\s*$", doc, re.M):
            continue
        pm = st._PATH_RE.search(doc)
        if not pm:
            continue
        nm = st._NAME_RE.search(doc)
        srcs = substitute_sources(doc)
        if not srcs:
            continue                                  # no substitution => nothing to audit
        kustomizations.append({
            "file": str(yfile.relative_to(ROOT)),
            "name": nm.group(1) if nm else "?",
            "path": pm.group(1),
            "sources": srcs,
        })

check(len(kustomizations) > 0,
      "no Flux Kustomization with postBuild.substituteFrom found under clusters/ — "
      "the parser or the repo layout changed; this check would silently audit nothing")

# ------------------------------------------------------------------ token scanning
# One `$`-run followed by a braced expression. `[^{}]*` is exhaustive for this tree:
# a repo-wide sweep found no nested/unclosed brace expression in any manifest (only
# Gitea Actions `${{ ... }}` in .gitea/, which no Flux Kustomization renders).
TOKEN_RE = re.compile(r"(\$+)\{([^{}]*)\}")
NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
# An expansion whose name is followed by a POSIX parameter operator. envsubst EVALUATES
# these, so an undefined name yields the default/empty AT RECONCILE TIME.
OP_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)(:[-=?+].*|:[0-9].*|/.*|#.*|%.*|\^.*|,.*)$")
LEN_RE = re.compile(r"^#([A-Za-z_][A-Za-z0-9_]*)$")


def classify(dollars: int, inner: str) -> tuple[str, str] | None:
    """Return (severity-tag, remediation) for a hazardous expansion, else None."""
    if dollars >= 2:
        return None                                   # escaped: emitted as a literal
    if NAME_RE.match(inner):
        if inner in KEYS:
            return None                               # defined knob — the happy path
        return ("UNDEFINED",
                f"strict mode fails the Kustomization; today it renders EMPTY. Either add "
                f"'{inner}' to clusters/base/cluster-vars.yaml with a base default, or "
                f"escape it as $${{{inner}}} if it is meant for a shell/cloud-init consumer")
    m = OP_RE.match(inner) or LEN_RE.match(inner)
    if m:
        base = m.group(1)
        if base in KEYS:
            return None                               # defined knob with an operator
        return ("SILENT-EMPTY",
                f"envsubst resolves this AT RECONCILE TIME because '{base}' is not a "
                f"cluster-var, so the rendered manifest carries the default, not the "
                f"expression. StrictPostBuildSubstitutions does NOT catch this form. "
                f"Escape it as $${{{inner}}} if the CONTAINER's shell is meant to expand it")
    return ("FATAL-PARSE",
            "envsubst cannot parse this and fails the ENTIRE Kustomization (every resource "
            "in the path). Escape it as $${...} or rewrite the prose without braces")


def rendered_files(path_spec: str) -> tuple[list[str], bool]:
    """Repo-relative files the Flux `spec.path` renders, resolving `resources:` transitively."""
    rel = os.path.normpath(path_spec.lstrip("./")) if path_spec not in (".", "./") else "."
    if not (ROOT / rel).is_dir():
        return [], False
    applied, _ = st.reconciled_files(ROOT, rel)
    return sorted(applied), True


findings: list[str] = []
referenced: dict[str, set[str]] = {}
inventory: list[str] = []

for k in kustomizations:
    files, exists = rendered_files(k["path"])
    check(exists, f"{k['file']}: Kustomization '{k['name']}' renders path '{k['path']}' "
                  f"which is not a directory in this repo")
    has_cluster_vars = ("ConfigMap", "cluster-vars") in k["sources"]
    n_def = n_esc = n_bad = 0
    for rel in files:
        f = ROOT / rel
        try:
            text = f.read_text()
        except (OSError, UnicodeDecodeError):
            continue
        for lineno, line in enumerate(text.splitlines(), 1):
            for m in TOKEN_RE.finditer(line):
                dollars, inner = len(m.group(1)), m.group(2)
                verdict = classify(dollars, inner)
                if verdict is None:
                    if dollars >= 2:
                        n_esc += 1
                    else:
                        n_def += 1
                        referenced.setdefault(re.split(r"[:/#%^,]", inner)[0] or inner,
                                              set()).add(k["name"])
                        # (4) a DEFINED knob is only substituted if this Kustomization
                        #     actually pulls cluster-vars in; otherwise the literal ships.
                        check(has_cluster_vars,
                              f"{rel}:{lineno}: renders cluster-var ${{{inner}}} but "
                              f"Kustomization '{k['name']}' ({k['file']}) does not "
                              f"substituteFrom ConfigMap/cluster-vars — the literal token "
                              f"would ship into the cluster")
                    continue
                n_bad += 1
                tag, fix = verdict
                findings.append(f"  [{tag}] {rel}:{lineno}  {m.group(0)}\n"
                                f"        rendered by Flux Kustomization '{k['name']}' "
                                f"({k['file']}, path {k['path']})\n"
                                f"        {fix}\n"
                                f"        | {line.strip()[:120]}")
    inventory.append(f"  {k['name']:<16} path={k['path']:<22} files={len(files):<3} "
                     f"defined={n_def:<3} escaped={n_esc:<3} hazards={n_bad}")

check(not findings,
      f"{len(findings)} unescaped, undefined or unparseable expansion(s) in substituted paths")

# --------------------------------------------- (5) overlay knobs exist in the base
# Parse contract: overlays patch cluster-vars with JSON6902 ops (`- op: replace` /
# `path: /data/KEY`) inside an inline `patch: |-` block. The repo rule (cluster-vars.yaml
# header) is that a knob is ADDED TO THE BASE with a default BEFORE any overlay or
# manifest references it — that ordering is what closes the define-then-use race.
_OP_PATH_RE = re.compile(r"^\s*(?:-\s*)?path:\s*/data/([A-Za-z_][A-Za-z0-9_]*)\s*$", re.M)
for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
    for key in set(_OP_PATH_RE.findall(overlay.read_text())):
        check(key in KEYS,
              f"{overlay.relative_to(ROOT)}: patches /data/{key} but '{key}' has no BASE "
              f"default in clusters/base/cluster-vars.yaml — a second overlay would render "
              f"${{{key}}} empty (R-ENV-02: author once, supply per env)")

# ------------------------------------------------------- (6) the check is load-bearing
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism.
# It used to assert `"flux-postbuild-substitution/check.sh" in L0`: ci/l0.sh carried a hand-written
# "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone
# (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED
# to write it). The same property is now asserted two ways: this test DECLARES
# `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into
# the run via `--include-scope`. Neither half alone is enough, so both are checked here; the
# mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than
# passing) is proven end-to-end and falsifiably by
# conformance/tests/l0-static/whole-repo-invariants-wired.
SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "flux-postbuild-substitution" / "chainsaw-test.yaml"
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None,
      "flux-postbuild-substitution/chainsaw-test.yaml does not declare "
      'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only '
      "when a change happens to touch one of the requirements it is labelled with. "
      "The path map keys only clusters/* to R-ENV-02, so an edit to a SUBSTITUTED manifest under apps/ or platform/ attributes nothing and this check would be filtered out of exactly the change that empties a variable (the 2026-09-01 Cilium k8sServiceHost CNI outage)")
check("--include-scope" in L0,
      "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is "
      "a purely requirement-scoped run again and the scope annotation is inert decoration — "
      "this check would then run only by coincidence")
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
      "ci/gate.sh does not set the merge block when the l0 stage fails")

# ------------------------------------------------------------- advisory: dead knobs
for key in sorted(KEYS - set(referenced)):
    ADVISORIES.append(f"cluster-vars knob '{key}' is referenced by NO reconciled manifest "
                      f"(dead knob, or documentation-only — advisory, not a failure)")

# ------------------------------------------------------------------------- report
print("Flux postBuild substitution inventory (per Kustomization):")
for line in inventory:
    print(line)
print(f"\ncluster-vars knobs defined: {len(KEYS)}   referenced: {len(referenced)}")

if ADVISORIES:
    print("\nADVISORIES (not failures):")
    for a in ADVISORIES:
        print(f"  - {a}")

if findings:
    print("\nHAZARDS — unescaped expansions that are not cluster-vars knobs:", file=sys.stderr)
    for f in findings:
        print(f, file=sys.stderr)

print()
if not FAILURES:
    print(f"check OK: flux-postbuild-substitution — {CHECKS} assertions "
          f"(R-ENV-02, R-GITOPS-01)")
    raise SystemExit(0)
print(f"CHECK FAIL (flux-postbuild-substitution): {len(FAILURES)} failed, "
      f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
for f in FAILURES:
    print(f"  FAIL {f}", file=sys.stderr)
raise SystemExit(1)
PY