#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — dev-nodes-provisioned-as-code # Serves: R-ENV-01 ("dev/test MUST be provisioned as local KVMs via IaC: created, # baselined, cluster-joined, and managed as code." — spec # docs/00-vision-and-scope.md; the four stages are enumerated in # docs/02-environments-and-provisioning.md §"Node lifecycle stages # (R-ENV-01 — 'created, baselined, cluster-joined, managed as code')": # 1. Create — OpenTofu instantiates the VM # 2. Baseline — OS baseline and prerequisites # 3. Cluster-join — Ansible installs/joins k3s with Cilium as CNI; disables # k3s bundled flannel/traefik/servicelb/kube-proxy # 4. Manage — GitOps-managed by Flux; node config changes are made via IaC # and re-applied, never hand-edited on the box.) # # WHY L0 IS THE RIGHT LAYER, AND WHAT IT MEANS TO TEST THIS BEHAVIOURALLY # ---------------------------------------------------------------------- # R-ENV-01 is a requirement about HOW the dev nodes come to exist. The live cluster # cannot answer it: a Kubernetes node object looks identical whether it was built by # OpenTofu or by hand in a hypervisor UI (there is no API field for "was I created by # IaC"). What CAN be proven, with no cluster, is that a COMPLETE, EXECUTABLE # provisioning path exists for all four stages and covers every dev node — and, for the # Ansible half, that is proven by RUNNING the real toolchain in a no-connection mode # (`--syntax-check`, `--list-hosts`) rather than by reading the files, so a chain that # does not actually resolve fails here. The IaC half is asserted by parsing the module: # `tofu init/validate` needs the provider registry, and L0 is offline by contract. # # GIVEN the repository as git tracks it, # WHEN the provisioning chain is parsed and (where offline execution is possible) # actually driven, # THEN CREATE: the IaC module targets a LOCAL KVM hypervisor (every declared provider # is a KVM-family hypervisor provider — no cloud provider may appear), # instantiates its nodes from a for_each map (never a count index, so one # node can be replaced without disturbing its siblings), and gives each # node CPU, RAM, an OS disk, a DEDICATED Longhorn data disk, a NIC and a # cloud-init seed carrying the login user + SSH key; # AND the node set it creates is EXACTLY the dev node set Ansible manages — # names derived from the module's own prefix/count, so no node can be # hand-added to the inventory or hand-created in the hypervisor; # BASELINE: a baseline stage exists as declarative Ansible (package floor, kernel # modules persisted AND loaded, sysctls, the Longhorn data disk formatted # and mounted, swap off, iscsid enabled), driven by variables with # defaults rather than per-host special cases; # JOIN: one role installs k3s for both roles of every node — `--cluster-init` # on the primary, `--server` join on the secondaries — with flannel, # kube-proxy, traefik and servicelb disabled AT INSTALL, and Cilium # installed as the CNI afterwards; # MANAGE: site.yml chains baseline -> k3s -> Cilium -> verify -> Flux bootstrap; # `ansible-playbook --syntax-check` on that chain SUCCEEDS; and # `--list-hosts` shows every one of those stages targeting EVERY dev # node, so no node is outside the code path. # # ENVIRONMENT-AGNOSTIC: no hostname, address, prefix or node count is written down here. # The dev group names come from the playbooks' own default host patterns, the node names # from the IaC module's prefix/count variables, and the two sets are compared to each # other. This check therefore keeps working when the estate is renamed or resized. # # WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them): # - That the RUNNING dev VMs were in fact created by this module. Nothing offline can: # it would need the OpenTofu state (a remote R2 backend + credentials) or hypervisor # access. `tofu plan` against the live estate is the operator's own check. # - That the module APPLIES cleanly. `tofu validate` needs `tofu init`, which downloads # the provider from the registry; L0 is offline by contract, so the module is parsed, # not evaluated. ci/lint.sh does not cover .tf either — a gap worth closing in CI # with a network-allowed stage, not here. # - "Never hand-edited on the box" (stage 4's second half) — a drift check against the # live nodes, which is an operator/Ansible-check-mode exercise, not a repo property. # - REBUILD/re-converge ("destroy a dev node VM and rebuild it", docs/02's own proof # line) — that is exercised at L3 by tests/l3-fault-injection/node-loss (R-STATE-02). # - That the baseline actually ran on a node (cluster/host state), and that the # hypervisor is on the operator's own LAN rather than remote — "local" is asserted as # "a KVM hypervisor provider driving a self-hosted endpoint variable", not as a # network topology fact. # - DEVIATION FROM docs/02 PROSE, ASSERTED DELIBERATELY: docs/02 describes stage 2 as # cloud-init applying the OS baseline. The implementation moved the rich baseline into # the Ansible `node_baseline` role (cloud-init keeps hostname/user/SSH/network), # because Proxmox provisioning is API-only and a custom cloud-init snippet needs # SSH/SFTP to the hypervisor. R-ENV-01's own wording requires the node to be # "baselined ... as code", not that cloud-init be the tool, so this check asserts the # STAGE, wherever it lives, and records the divergence rather than hiding it. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root PYTHON="${PYTHON:-python3}" cd "${ROOT}" "${PYTHON}" - "${ROOT}" <<'PY' import re import shutil import subprocess import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) FAILURES: list[str] = [] NOTES: list[str] = [] CHECKS = 0 def check(cond, msg): global CHECKS CHECKS += 1 if not cond: FAILURES.append(msg) def note(msg): NOTES.append(msg) # A LOCAL DEPENDENCY the repo deliberately does not carry — the vault passphrase (kept # outside the repo, R-SEC-01) or the Galaxy collections (gitignored; installed from # collections/requirements.yml). Their absence is an ENVIRONMENT gap, not a repo defect, # and is the ONLY tolerated reason the driven sub-assertions may not run. Deliberately # narrow: every other non-zero exit is a real failure. The parse assertions always run. ENV_ONLY_FAILURE = re.compile( r"vault password|vault-id|Attempting to decrypt" r"|No module named 'ansible_collections|couldn't resolve module/action", re.I) TOFU = ROOT / "infra" / "tofu" ANS = ROOT / "infra" / "ansible" check(TOFU.is_dir(), "infra/tofu/ is missing — the CREATE stage has no IaC at all (R-ENV-01)") check(ANS.is_dir(), "infra/ansible/ is missing — the BASELINE/JOIN/MANAGE stages have no code") if not TOFU.is_dir() or not ANS.is_dir(): print(f"CHECK FAIL (dev-nodes-provisioned-as-code): {len(FAILURES)} of {CHECKS} failed", file=sys.stderr) for f in FAILURES: print(f" - {f}", file=sys.stderr) raise SystemExit(1) TF = "\n".join((p.read_text(errors="replace") for p in sorted(TOFU.glob("*.tf")))) check(len(TF) > 500, "infra/tofu/*.tf is empty or a stub — nothing creates the dev nodes") # ============================================================ STAGE 1 — CREATE (IaC) # A LOCAL KVM hypervisor, by allowlist: any provider outside this set fails, so moving # dev onto a cloud (or anything that is not a KVM hypervisor the operator runs) is caught # here rather than being noticed later. Each entry is justified, not just listed: # bpg/proxmox Proxmox VE — a KVM/QEMU hypervisor the operator hosts. # dmacvicar/libvirt libvirt/KVM directly (the pre-Proxmox substrate). KVM_PROVIDERS = {"bpg/proxmox", "dmacvicar/libvirt"} providers = dict(re.findall(r'source\s*=\s*"([^"]+)"', TF) and [(m.group(1), m.group(2)) for m in re.finditer(r'(\w[\w-]*)\s*=\s*\{[^}]*?source\s*=\s*"([^"]+)"', TF, re.S)]) check(len(providers) > 0, "infra/tofu declares no required_providers — the substrate is unpinned, so 'via IaC' " "cannot be checked at all") for alias, source in sorted(providers.items()): check(source.lower() in KVM_PROVIDERS, f"infra/tofu declares provider {alias}={source!r}, which is not a local KVM hypervisor " f"provider (allowed: {sorted(KVM_PROVIDERS)}) — R-ENV-01 requires dev/test to be LOCAL KVMs") # The VM resource: for_each-keyed, and carrying the hardware + seed docs/02 enumerates. vm_blocks = re.findall(r'resource\s+"([\w]*(?:_vm|_domain))"\s+"(\w+)"\s*\{(.*?)\n\}', TF, re.S) check(len(vm_blocks) >= 1, "no VM/domain resource found in infra/tofu — nothing in IaC creates a dev node") for rtype, rname, body in vm_blocks: check("for_each" in body, f'resource "{rtype}" "{rname}" is not for_each-keyed. A count index makes single-node ' "replace unsafe (a -target'ed destroy cascaded to sibling nodes — DECISIONS-LOG F1), so " "'created ... as code' would not survive a rebuild") check(not re.search(r"^\s*count\s*=", body, re.M), f'resource "{rtype}" "{rname}" uses a positional count index') for want, why in ( (r"\bcpu\b", "CPU"), (r"\bmemory\b", "RAM"), (r"network_device|network_interface", "a NIC"), (r"initialization|cloud_init|cloudinit", "a cloud-init seed"), ): check(re.search(want, body) is not None, f'resource "{rtype}" "{rname}" declares no {why} — docs/02 stage 1 requires CPU, RAM, ' "disks, NICs and a cloud-init seed to come from IaC") disks = len(re.findall(r"^\s*disk\s*\{", body, re.M)) check(disks >= 2, f'resource "{rtype}" "{rname}" declares {disks} disk block(s); docs/02 requires the OS ' "disk PLUS a dedicated data disk for Longhorn") check(re.search(r"user_account|ssh_authorized_key|ssh_keys", body) is not None, f'resource "{rtype}" "{rname}" seeds no login user / SSH key — the node could not be ' "reached by the BASELINE stage without a hand step") # Node identity comes from variables, and those variables define the whole node set. def tf_default(var: str): m = re.search(r'variable\s+"' + re.escape(var) + r'"\s*\{(.*?)\n\}', TF, re.S) if not m: return None d = re.search(r'default\s*=\s*(.+)', m.group(1)) return d.group(1).strip() if d else None prefix_raw = tf_default("node_name_prefix") count_raw = tf_default("node_count") ips_raw = tf_default("node_ips") check(prefix_raw is not None and count_raw is not None, "infra/tofu has no node_name_prefix/node_count variables — the node set is not derived " "from code, so the created nodes cannot be compared with the managed ones") prefix = (prefix_raw or "").strip('"') try: node_count = int((count_raw or "0").strip()) except ValueError: node_count = 0 check(node_count >= 1, f"node_count default is {count_raw!r} — no node would be created") if ips_raw is not None: n_ips = len(re.findall(r'"[^"]+"', ips_raw)) check(n_ips >= node_count, f"node_ips default has {n_ips} address(es) for node_count={node_count} — the module " "cannot address every node it creates") check((TOFU / "outputs.tf").is_file() and re.search(r'output\s+"(ansible_inventory|nodes)"', (TOFU / "outputs.tf").read_text()) is not None, "infra/tofu/outputs.tf surfaces no node inventory output — the JOIN stage would have to be " "told the node facts by hand instead of consuming them from IaC") # ------------------------------------------------- the dev node set, derived twice over # Group names are taken from the PLAYBOOKS' own default host patterns, so nothing here # restates the estate's naming. PLAYBOOKS = ANS / "playbooks" def default_patterns(playbook: str) -> list[str]: p = PLAYBOOKS / playbook if not p.is_file(): return [] return re.findall(r"default\('([^']+)'\)", p.read_text()) dev_groups = set(default_patterns("node-baseline.yml") + default_patterns("k3s.yml")) check(len(dev_groups) > 0, "no playbook declares a default host pattern — the dev target is not expressed in code") INV = ANS / "inventory" / "hosts.ini" check(INV.is_file(), "infra/ansible/inventory/hosts.ini is missing — no node is managed as code") ini_groups: dict[str, list[str]] = {} children: dict[str, list[str]] = {} cur = None for line in (INV.read_text().splitlines() if INV.is_file() else []): s = line.strip() if not s or s.startswith("#"): continue m = re.match(r"^\[([^\]]+)\]$", s) if m: cur = m.group(1) (children if cur.endswith(":children") else ini_groups).setdefault( cur.split(":")[0] if cur.endswith(":children") else cur, []) continue if cur is None or cur.endswith(":vars"): continue host = s.split()[0] if cur.endswith(":children"): children.setdefault(cur.split(":")[0], []).append(host) else: ini_groups.setdefault(cur, []).append(host) def hosts_in(group: str, seen=None) -> set[str]: seen = seen or set() if group in seen: return set() seen.add(group) out = set(ini_groups.get(group, [])) for child in children.get(group, []): out |= hosts_in(child, seen) return out dev_hosts: set[str] = set() for g in sorted(dev_groups): dev_hosts |= hosts_in(g) check(len(dev_hosts) > 0, f"the dev groups {sorted(dev_groups)} resolve to NO hosts in hosts.ini — nothing is managed") expected = {f"{prefix}{i + 1}" for i in range(node_count)} check(dev_hosts == expected, f"the dev nodes Ansible manages ({sorted(dev_hosts)}) are not exactly the nodes the IaC " f"module creates ({sorted(expected)}) — a node outside IaC is a hand-built node (R-ENV-01)") # ============================================================ STAGE 2 — BASELINE (code) BASE_ROLE = ANS / "roles" / "node_baseline" check(BASE_ROLE.is_dir(), "no node_baseline role — the BASELINE stage has no code (docs/02 stage 2)") base_tasks = "\n".join(p.read_text(errors="replace") for p in sorted((BASE_ROLE / "tasks").glob("*.yml"))) if BASE_ROLE.is_dir() else "" base_defaults = "\n".join(p.read_text(errors="replace") for p in sorted((BASE_ROLE / "defaults").glob("*.yml"))) if BASE_ROLE.is_dir() else "" # Each item docs/02 names for stage 2, matched on the ANSIBLE MODULE that would do it — # a declarative, idempotent task, not a shell one-liner. for pattern, what in ( (r"ansible\.builtin\.apt:", "a package baseline (apt module)"), (r"modprobe:", "kernel modules loaded (modprobe module)"), (r"modules-load|/etc/modules", "kernel modules PERSISTED across reboot"), (r"sysctl", "sysctl tuning"), (r"filesystem:|mkfs", "a filesystem on the Longhorn data disk"), (r"mount:|ansible\.posix\.mount", "the Longhorn data disk MOUNTED"), (r"swap", "swap disabled"), (r"iscsid", "iscsid enabled (Longhorn prerequisite)"), ): check(re.search(pattern, base_tasks) is not None, f"node_baseline does not declare {what} — docs/02 stage 2 requires the OS baseline and " "k3s/Longhorn/KubeVirt prerequisites to be applied as code") check(re.search(r"^node_baseline_packages:", base_defaults, re.M) is not None and re.search(r"^node_baseline_kernel_modules:", base_defaults, re.M) is not None, "node_baseline's package/module sets are not variables with defaults — a per-host special " "case is not 'managed as code' (and could not be scoped per environment, R-ENV-02)") # ========================================================== STAGE 3 — CLUSTER-JOIN (k3s) K3S_ROLE = ANS / "roles" / "k3s_server" k3s_text = "\n".join(p.read_text(errors="replace") for p in sorted((K3S_ROLE).rglob("*.yml"))) if K3S_ROLE.is_dir() else "" check(K3S_ROLE.is_dir() and len(k3s_text) > 200, "no k3s_server role — nothing joins the nodes into a cluster (docs/02 stage 3)") check("--cluster-init" in k3s_text, "the k3s role never passes --cluster-init — the HA embedded-etcd cluster is never founded") check(re.search(r"--server\s+https", k3s_text) is not None, "the k3s role never joins secondaries with --server — nodes would not form one cluster") check(re.search(r"K3S_TOKEN|--token", k3s_text) is not None, "the k3s role never supplies a join token — the join could only be completed by hand") # The bundled components docs/02 requires to be disabled AT INSTALL, each checked by the # flag that actually disables it. for pattern, what in ( (r"--flannel-backend=none", "flannel (Cilium is the CNI)"), (r"--disable-kube-proxy", "kube-proxy (Cilium kube-proxy replacement)"), (r"traefik", "traefik (Gateway API via Cilium)"), (r"servicelb", "servicelb"), ): check(re.search(pattern, k3s_text) is not None, f"the k3s install does not disable {what} — docs/02 stage 3 requires it disabled at " "install, or the bundled component fights the platform's own") check((ANS / "roles" / "cilium").is_dir(), "no cilium role — stage 3 requires Cilium to be installed as the CNI at bootstrap") check((ANS / "roles" / "flux_bootstrap").is_dir(), "no flux_bootstrap role — stage 4's hand-off to GitOps has no code") # ============================================== STAGE 4 — MANAGE + the chain is REAL SITE = PLAYBOOKS / "site.yml" check(SITE.is_file(), "no playbooks/site.yml — the provisioning chain is not expressed as one path") site_txt = SITE.read_text() if SITE.is_file() else "" order = [m for m in re.findall(r"import_playbook:\s*(\S+)", site_txt)] check(order[:1] == ["node-baseline.yml"], f"site.yml imports {order} — the BASELINE stage must come first (docs/02 sequencing)") for want in ("k3s.yml", "cilium.yml", "flux-bootstrap.yml"): check(want in order, f"site.yml does not import {want} — a lifecycle stage is missing from the chain") if "k3s.yml" in order and "cilium.yml" in order: check(order.index("k3s.yml") < order.index("cilium.yml"), "site.yml installs Cilium before k3s — the CNI cannot precede the cluster") # --- DRIVE THE REAL TOOLCHAIN, OFFLINE ------------------------------------------------ # --syntax-check and --list-hosts open NO connection to any node: they resolve the # playbook, roles, collections and inventory only. Running them is the difference between # "the chain is written down" and "the chain resolves". if shutil.which("ansible-playbook") is None: note("ansible-playbook is not installed here (e.g. the in-cluster chainsaw image), so the " "chain was PARSED but not DRIVEN; the merge gate's container has it and does drive it") else: # cwd=infra/ansible so the tracked ansible.cfg (roles_path, collections_path, inventory) # is the one in force — the same resolution a real run gets. def ansible(*args) -> subprocess.CompletedProcess: return subprocess.run(["ansible-playbook", *args], cwd=ANS, capture_output=True, text=True) syn = ansible("--syntax-check", "playbooks/site.yml") syn_out = (syn.stderr or "") + (syn.stdout or "") lh = None if syn.returncode != 0 and ENV_ONLY_FAILURE.search(syn_out): # NARROW, SELF-POLICING SKIP — see ENV_ONLY_FAILURE above. note("ansible-playbook lacks a LOCAL dependency kept outside the repo by design (the vault " "passphrase and/or the gitignored Galaxy collections), so --syntax-check/--list-hosts " "were not driven; every parse assertion above still ran") else: check(syn.returncode == 0, "`ansible-playbook --syntax-check playbooks/site.yml` FAILED — the provisioning chain " f"does not even load: {syn_out.strip()[:400]}") lh = ansible("--list-hosts", "playbooks/site.yml") check(lh.returncode == 0, "`ansible-playbook --list-hosts playbooks/site.yml` FAILED — the chain does not " f"resolve against the tracked inventory: " f"{((lh.stderr or '') + (lh.stdout or '')).strip()[:400]}") if lh is not None and lh.returncode == 0: # Parse plays -> hosts. NEVER `ansible-inventory --list`: it prints DECRYPTED vault values. plays: list[tuple[str, set[str]]] = [] name, hosts, in_hosts = None, set(), False for line in lh.stdout.splitlines(): m = re.match(r"\s*play #\d+ \([^)]*\):\s*(.*?)\s*TAGS:", line) if m: if name is not None: plays.append((name, hosts)) name, hosts, in_hosts = m.group(1), set(), False continue if re.match(r"\s*hosts \(\d+\):", line): in_hosts = True continue if in_hosts and line.strip() and not line.strip().startswith("pattern:"): hosts.add(line.strip()) if name is not None: plays.append((name, hosts)) check(len(plays) >= 4, f"--list-hosts resolved {len(plays)} play(s) — the four-stage chain did not expand") # Every lifecycle stage must cover EVERY dev node. A stage that reaches two of three # nodes means the third was finished by hand. STAGES = { "baseline": r"baseline", "cluster-join": r"^k3s (primary|secondar)", "cni": r"cilium", "manage (flux hand-off)": r"flux", } for stage, rx in STAGES.items(): covered: set[str] = set() matched = 0 for pname, phosts in plays: if re.search(rx, pname, re.I): matched += 1 covered |= phosts check(matched >= 1, f"no play in site.yml implements the {stage} stage") check(covered == dev_hosts, f"the {stage} stage targets {sorted(covered)}, not every dev node " f"{sorted(dev_hosts)} — a node outside the code path is a hand-provisioned " "node (R-ENV-01)") # --------------------------------------------------------------------------- verdict # NO-OP GUARD: a check that quietly asserts nothing is worse than no check. check(CHECKS >= 30, f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass " "would be meaningless") if FAILURES: print(f"CHECK FAIL (dev-nodes-provisioned-as-code): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr) for f in FAILURES: print(f" - {f}", file=sys.stderr) raise SystemExit(1) for n in NOTES: print(f" note: {n}") print(f"check OK: dev-nodes-provisioned-as-code — {CHECKS} assertions; " f"{len(dev_hosts)} dev node(s) {sorted(dev_hosts)} created by IaC ({sorted(providers.values())}), " f"baselined, cluster-joined and Flux-managed by the same chain. R-ENV-01's authored-path " "clause only — the live VMs' provenance, `tofu validate`, and rebuild/re-converge are NOT " "proven here (see the header's gap list; rebuild is L3 node-loss).") PY