home-network-k3s-impl/conformance/tests/l0-static/dev-nodes-provisioned-as-code/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — dev-nodes-provisioned-as-code
# Serves: R-ENV-01 ("dev/test MUST be provisioned as local KVMs via IaC: created,
#         baselined, cluster-joined, and managed as code." — spec
#         docs/00-vision-and-scope.md; the four stages are enumerated in
#         docs/02-environments-and-provisioning.md §"Node lifecycle stages
#         (R-ENV-01 — 'created, baselined, cluster-joined, managed as code')":
#           1. Create   — OpenTofu instantiates the VM
#           2. Baseline — OS baseline and prerequisites
#           3. Cluster-join — Ansible installs/joins k3s with Cilium as CNI; disables
#              k3s bundled flannel/traefik/servicelb/kube-proxy
#           4. Manage   — GitOps-managed by Flux; node config changes are made via IaC
#              and re-applied, never hand-edited on the box.)
#
# WHY L0 IS THE RIGHT LAYER, AND WHAT IT MEANS TO TEST THIS BEHAVIOURALLY
# ----------------------------------------------------------------------
# R-ENV-01 is a requirement about HOW the dev nodes come to exist. The live cluster
# cannot answer it: a Kubernetes node object looks identical whether it was built by
# OpenTofu or by hand in a hypervisor UI (there is no API field for "was I created by
# IaC"). What CAN be proven, with no cluster, is that a COMPLETE, EXECUTABLE
# provisioning path exists for all four stages and covers every dev node — and, for the
# Ansible half, that is proven by RUNNING the real toolchain in a no-connection mode
# (`--syntax-check`, `--list-hosts`) rather than by reading the files, so a chain that
# does not actually resolve fails here. The IaC half is asserted by parsing the module:
# `tofu init/validate` needs the provider registry, and L0 is offline by contract.
#
# GIVEN the repository as git tracks it,
# WHEN  the provisioning chain is parsed and (where offline execution is possible)
#       actually driven,
# THEN  CREATE:  the IaC module targets a LOCAL KVM hypervisor (every declared provider
#                is a KVM-family hypervisor provider — no cloud provider may appear),
#                instantiates its nodes from a for_each map (never a count index, so one
#                node can be replaced without disturbing its siblings), and gives each
#                node CPU, RAM, an OS disk, a DEDICATED Longhorn data disk, a NIC and a
#                cloud-init seed carrying the login user + SSH key;
#       AND      the node set it creates is EXACTLY the dev node set Ansible manages —
#                names derived from the module's own prefix/count, so no node can be
#                hand-added to the inventory or hand-created in the hypervisor;
#       BASELINE: a baseline stage exists as declarative Ansible (package floor, kernel
#                modules persisted AND loaded, sysctls, the Longhorn data disk formatted
#                and mounted, swap off, iscsid enabled), driven by variables with
#                defaults rather than per-host special cases;
#       JOIN:    one role installs k3s for both roles of every node — `--cluster-init`
#                on the primary, `--server` join on the secondaries — with flannel,
#                kube-proxy, traefik and servicelb disabled AT INSTALL, and Cilium
#                installed as the CNI afterwards;
#       MANAGE:  site.yml chains baseline -> k3s -> Cilium -> verify -> Flux bootstrap;
#                `ansible-playbook --syntax-check` on that chain SUCCEEDS; and
#                `--list-hosts` shows every one of those stages targeting EVERY dev
#                node, so no node is outside the code path.
#
# ENVIRONMENT-AGNOSTIC: no hostname, address, prefix or node count is written down here.
# The dev group names come from the playbooks' own default host patterns, the node names
# from the IaC module's prefix/count variables, and the two sets are compared to each
# other. This check therefore keeps working when the estate is renamed or resized.
#
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
#   - That the RUNNING dev VMs were in fact created by this module. Nothing offline can:
#     it would need the OpenTofu state (a remote R2 backend + credentials) or hypervisor
#     access. `tofu plan` against the live estate is the operator's own check.
#   - That the module APPLIES cleanly. `tofu validate` needs `tofu init`, which downloads
#     the provider from the registry; L0 is offline by contract, so the module is parsed,
#     not evaluated. ci/lint.sh does not cover .tf either — a gap worth closing in CI
#     with a network-allowed stage, not here.
#   - "Never hand-edited on the box" (stage 4's second half) — a drift check against the
#     live nodes, which is an operator/Ansible-check-mode exercise, not a repo property.
#   - REBUILD/re-converge ("destroy a dev node VM and rebuild it", docs/02's own proof
#     line) — that is exercised at L3 by tests/l3-fault-injection/node-loss (R-STATE-02).
#   - That the baseline actually ran on a node (cluster/host state), and that the
#     hypervisor is on the operator's own LAN rather than remote — "local" is asserted as
#     "a KVM hypervisor provider driving a self-hosted endpoint variable", not as a
#     network topology fact.
#   - DEVIATION FROM docs/02 PROSE, ASSERTED DELIBERATELY: docs/02 describes stage 2 as
#     cloud-init applying the OS baseline. The implementation moved the rich baseline into
#     the Ansible `node_baseline` role (cloud-init keeps hostname/user/SSH/network),
#     because Proxmox provisioning is API-only and a custom cloud-init snippet needs
#     SSH/SFTP to the hypervisor. R-ENV-01's own wording requires the node to be
#     "baselined ... as code", not that cloud-init be the tool, so this check asserts the
#     STAGE, wherever it lives, and records the divergence rather than hiding it.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"

cd "${ROOT}"
"${PYTHON}" - "${ROOT}" <<'PY'
import re
import shutil
import subprocess
import sys
from pathlib import Path

try:
    import yaml
except ImportError:                                  # pragma: no cover
    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
          file=sys.stderr)
    raise SystemExit(1)

ROOT = Path(sys.argv[1])
FAILURES: list[str] = []
NOTES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(msg)


def note(msg):
    NOTES.append(msg)


# A LOCAL DEPENDENCY the repo deliberately does not carry — the vault passphrase (kept
# outside the repo, R-SEC-01) or the Galaxy collections (gitignored; installed from
# collections/requirements.yml). Their absence is an ENVIRONMENT gap, not a repo defect,
# and is the ONLY tolerated reason the driven sub-assertions may not run. Deliberately
# narrow: every other non-zero exit is a real failure. The parse assertions always run.
ENV_ONLY_FAILURE = re.compile(
    r"vault password|vault-id|Attempting to decrypt"
    r"|No module named 'ansible_collections|couldn't resolve module/action", re.I)

TOFU = ROOT / "infra" / "tofu"
ANS = ROOT / "infra" / "ansible"
check(TOFU.is_dir(), "infra/tofu/ is missing — the CREATE stage has no IaC at all (R-ENV-01)")
check(ANS.is_dir(), "infra/ansible/ is missing — the BASELINE/JOIN/MANAGE stages have no code")
if not TOFU.is_dir() or not ANS.is_dir():
    print(f"CHECK FAIL (dev-nodes-provisioned-as-code): {len(FAILURES)} of {CHECKS} failed",
          file=sys.stderr)
    for f in FAILURES:
        print(f"  - {f}", file=sys.stderr)
    raise SystemExit(1)

TF = "\n".join((p.read_text(errors="replace") for p in sorted(TOFU.glob("*.tf"))))
check(len(TF) > 500, "infra/tofu/*.tf is empty or a stub — nothing creates the dev nodes")

# ============================================================ STAGE 1 — CREATE (IaC)
# A LOCAL KVM hypervisor, by allowlist: any provider outside this set fails, so moving
# dev onto a cloud (or anything that is not a KVM hypervisor the operator runs) is caught
# here rather than being noticed later. Each entry is justified, not just listed:
#   bpg/proxmox        Proxmox VE — a KVM/QEMU hypervisor the operator hosts.
#   dmacvicar/libvirt  libvirt/KVM directly (the pre-Proxmox substrate).
KVM_PROVIDERS = {"bpg/proxmox", "dmacvicar/libvirt"}
providers = dict(re.findall(r'source\s*=\s*"([^"]+)"', TF) and
                 [(m.group(1), m.group(2)) for m in
                  re.finditer(r'(\w[\w-]*)\s*=\s*\{[^}]*?source\s*=\s*"([^"]+)"', TF, re.S)])
check(len(providers) > 0,
      "infra/tofu declares no required_providers — the substrate is unpinned, so 'via IaC' "
      "cannot be checked at all")
for alias, source in sorted(providers.items()):
    check(source.lower() in KVM_PROVIDERS,
          f"infra/tofu declares provider {alias}={source!r}, which is not a local KVM hypervisor "
          f"provider (allowed: {sorted(KVM_PROVIDERS)}) — R-ENV-01 requires dev/test to be LOCAL KVMs")

# The VM resource: for_each-keyed, and carrying the hardware + seed docs/02 enumerates.
vm_blocks = re.findall(r'resource\s+"([\w]*(?:_vm|_domain))"\s+"(\w+)"\s*\{(.*?)\n\}',
                       TF, re.S)
check(len(vm_blocks) >= 1,
      "no VM/domain resource found in infra/tofu — nothing in IaC creates a dev node")
for rtype, rname, body in vm_blocks:
    check("for_each" in body,
          f'resource "{rtype}" "{rname}" is not for_each-keyed. A count index makes single-node '
          "replace unsafe (a -target'ed destroy cascaded to sibling nodes — DECISIONS-LOG F1), so "
          "'created ... as code' would not survive a rebuild")
    check(not re.search(r"^\s*count\s*=", body, re.M),
          f'resource "{rtype}" "{rname}" uses a positional count index')
    for want, why in (
        (r"\bcpu\b", "CPU"),
        (r"\bmemory\b", "RAM"),
        (r"network_device|network_interface", "a NIC"),
        (r"initialization|cloud_init|cloudinit", "a cloud-init seed"),
    ):
        check(re.search(want, body) is not None,
              f'resource "{rtype}" "{rname}" declares no {why} — docs/02 stage 1 requires CPU, RAM, '
              "disks, NICs and a cloud-init seed to come from IaC")
    disks = len(re.findall(r"^\s*disk\s*\{", body, re.M))
    check(disks >= 2,
          f'resource "{rtype}" "{rname}" declares {disks} disk block(s); docs/02 requires the OS '
          "disk PLUS a dedicated data disk for Longhorn")
    check(re.search(r"user_account|ssh_authorized_key|ssh_keys", body) is not None,
          f'resource "{rtype}" "{rname}" seeds no login user / SSH key — the node could not be '
          "reached by the BASELINE stage without a hand step")

# Node identity comes from variables, and those variables define the whole node set.
def tf_default(var: str):
    m = re.search(r'variable\s+"' + re.escape(var) + r'"\s*\{(.*?)\n\}', TF, re.S)
    if not m:
        return None
    d = re.search(r'default\s*=\s*(.+)', m.group(1))
    return d.group(1).strip() if d else None


prefix_raw = tf_default("node_name_prefix")
count_raw = tf_default("node_count")
ips_raw = tf_default("node_ips")
check(prefix_raw is not None and count_raw is not None,
      "infra/tofu has no node_name_prefix/node_count variables — the node set is not derived "
      "from code, so the created nodes cannot be compared with the managed ones")
prefix = (prefix_raw or "").strip('"')
try:
    node_count = int((count_raw or "0").strip())
except ValueError:
    node_count = 0
check(node_count >= 1, f"node_count default is {count_raw!r} — no node would be created")
if ips_raw is not None:
    n_ips = len(re.findall(r'"[^"]+"', ips_raw))
    check(n_ips >= node_count,
          f"node_ips default has {n_ips} address(es) for node_count={node_count} — the module "
          "cannot address every node it creates")
check((TOFU / "outputs.tf").is_file() and
      re.search(r'output\s+"(ansible_inventory|nodes)"', (TOFU / "outputs.tf").read_text()) is not None,
      "infra/tofu/outputs.tf surfaces no node inventory output — the JOIN stage would have to be "
      "told the node facts by hand instead of consuming them from IaC")

# ------------------------------------------------- the dev node set, derived twice over
# Group names are taken from the PLAYBOOKS' own default host patterns, so nothing here
# restates the estate's naming.
PLAYBOOKS = ANS / "playbooks"


def default_patterns(playbook: str) -> list[str]:
    p = PLAYBOOKS / playbook
    if not p.is_file():
        return []
    return re.findall(r"default\('([^']+)'\)", p.read_text())


dev_groups = set(default_patterns("node-baseline.yml") + default_patterns("k3s.yml"))
check(len(dev_groups) > 0,
      "no playbook declares a default host pattern — the dev target is not expressed in code")

INV = ANS / "inventory" / "hosts.ini"
check(INV.is_file(), "infra/ansible/inventory/hosts.ini is missing — no node is managed as code")
ini_groups: dict[str, list[str]] = {}
children: dict[str, list[str]] = {}
cur = None
for line in (INV.read_text().splitlines() if INV.is_file() else []):
    s = line.strip()
    if not s or s.startswith("#"):
        continue
    m = re.match(r"^\[([^\]]+)\]$", s)
    if m:
        cur = m.group(1)
        (children if cur.endswith(":children") else ini_groups).setdefault(
            cur.split(":")[0] if cur.endswith(":children") else cur, [])
        continue
    if cur is None or cur.endswith(":vars"):
        continue
    host = s.split()[0]
    if cur.endswith(":children"):
        children.setdefault(cur.split(":")[0], []).append(host)
    else:
        ini_groups.setdefault(cur, []).append(host)


def hosts_in(group: str, seen=None) -> set[str]:
    seen = seen or set()
    if group in seen:
        return set()
    seen.add(group)
    out = set(ini_groups.get(group, []))
    for child in children.get(group, []):
        out |= hosts_in(child, seen)
    return out


dev_hosts: set[str] = set()
for g in sorted(dev_groups):
    dev_hosts |= hosts_in(g)
check(len(dev_hosts) > 0,
      f"the dev groups {sorted(dev_groups)} resolve to NO hosts in hosts.ini — nothing is managed")
expected = {f"{prefix}{i + 1}" for i in range(node_count)}
check(dev_hosts == expected,
      f"the dev nodes Ansible manages ({sorted(dev_hosts)}) are not exactly the nodes the IaC "
      f"module creates ({sorted(expected)}) — a node outside IaC is a hand-built node (R-ENV-01)")

# ============================================================ STAGE 2 — BASELINE (code)
BASE_ROLE = ANS / "roles" / "node_baseline"
check(BASE_ROLE.is_dir(),
      "no node_baseline role — the BASELINE stage has no code (docs/02 stage 2)")
base_tasks = "\n".join(p.read_text(errors="replace")
                       for p in sorted((BASE_ROLE / "tasks").glob("*.yml"))) if BASE_ROLE.is_dir() else ""
base_defaults = "\n".join(p.read_text(errors="replace")
                          for p in sorted((BASE_ROLE / "defaults").glob("*.yml"))) if BASE_ROLE.is_dir() else ""
# Each item docs/02 names for stage 2, matched on the ANSIBLE MODULE that would do it —
# a declarative, idempotent task, not a shell one-liner.
for pattern, what in (
    (r"ansible\.builtin\.apt:", "a package baseline (apt module)"),
    (r"modprobe:", "kernel modules loaded (modprobe module)"),
    (r"modules-load|/etc/modules", "kernel modules PERSISTED across reboot"),
    (r"sysctl", "sysctl tuning"),
    (r"filesystem:|mkfs", "a filesystem on the Longhorn data disk"),
    (r"mount:|ansible\.posix\.mount", "the Longhorn data disk MOUNTED"),
    (r"swap", "swap disabled"),
    (r"iscsid", "iscsid enabled (Longhorn prerequisite)"),
):
    check(re.search(pattern, base_tasks) is not None,
          f"node_baseline does not declare {what} — docs/02 stage 2 requires the OS baseline and "
          "k3s/Longhorn/KubeVirt prerequisites to be applied as code")
check(re.search(r"^node_baseline_packages:", base_defaults, re.M) is not None and
      re.search(r"^node_baseline_kernel_modules:", base_defaults, re.M) is not None,
      "node_baseline's package/module sets are not variables with defaults — a per-host special "
      "case is not 'managed as code' (and could not be scoped per environment, R-ENV-02)")

# ========================================================== STAGE 3 — CLUSTER-JOIN (k3s)
K3S_ROLE = ANS / "roles" / "k3s_server"
k3s_text = "\n".join(p.read_text(errors="replace")
                     for p in sorted((K3S_ROLE).rglob("*.yml"))) if K3S_ROLE.is_dir() else ""
check(K3S_ROLE.is_dir() and len(k3s_text) > 200,
      "no k3s_server role — nothing joins the nodes into a cluster (docs/02 stage 3)")
check("--cluster-init" in k3s_text,
      "the k3s role never passes --cluster-init — the HA embedded-etcd cluster is never founded")
check(re.search(r"--server\s+https", k3s_text) is not None,
      "the k3s role never joins secondaries with --server — nodes would not form one cluster")
check(re.search(r"K3S_TOKEN|--token", k3s_text) is not None,
      "the k3s role never supplies a join token — the join could only be completed by hand")
# The bundled components docs/02 requires to be disabled AT INSTALL, each checked by the
# flag that actually disables it.
for pattern, what in (
    (r"--flannel-backend=none", "flannel (Cilium is the CNI)"),
    (r"--disable-kube-proxy", "kube-proxy (Cilium kube-proxy replacement)"),
    (r"traefik", "traefik (Gateway API via Cilium)"),
    (r"servicelb", "servicelb"),
):
    check(re.search(pattern, k3s_text) is not None,
          f"the k3s install does not disable {what} — docs/02 stage 3 requires it disabled at "
          "install, or the bundled component fights the platform's own")
check((ANS / "roles" / "cilium").is_dir(),
      "no cilium role — stage 3 requires Cilium to be installed as the CNI at bootstrap")
check((ANS / "roles" / "flux_bootstrap").is_dir(),
      "no flux_bootstrap role — stage 4's hand-off to GitOps has no code")

# ============================================== STAGE 4 — MANAGE + the chain is REAL
SITE = PLAYBOOKS / "site.yml"
check(SITE.is_file(), "no playbooks/site.yml — the provisioning chain is not expressed as one path")
site_txt = SITE.read_text() if SITE.is_file() else ""
order = [m for m in re.findall(r"import_playbook:\s*(\S+)", site_txt)]
check(order[:1] == ["node-baseline.yml"],
      f"site.yml imports {order} — the BASELINE stage must come first (docs/02 sequencing)")
for want in ("k3s.yml", "cilium.yml", "flux-bootstrap.yml"):
    check(want in order, f"site.yml does not import {want} — a lifecycle stage is missing from the chain")
if "k3s.yml" in order and "cilium.yml" in order:
    check(order.index("k3s.yml") < order.index("cilium.yml"),
          "site.yml installs Cilium before k3s — the CNI cannot precede the cluster")

# --- DRIVE THE REAL TOOLCHAIN, OFFLINE ------------------------------------------------
# --syntax-check and --list-hosts open NO connection to any node: they resolve the
# playbook, roles, collections and inventory only. Running them is the difference between
# "the chain is written down" and "the chain resolves".
if shutil.which("ansible-playbook") is None:
    note("ansible-playbook is not installed here (e.g. the in-cluster chainsaw image), so the "
         "chain was PARSED but not DRIVEN; the merge gate's container has it and does drive it")
else:
    # cwd=infra/ansible so the tracked ansible.cfg (roles_path, collections_path, inventory)
    # is the one in force — the same resolution a real run gets.
    def ansible(*args) -> subprocess.CompletedProcess:
        return subprocess.run(["ansible-playbook", *args], cwd=ANS,
                              capture_output=True, text=True)

    syn = ansible("--syntax-check", "playbooks/site.yml")
    syn_out = (syn.stderr or "") + (syn.stdout or "")
    lh = None
    if syn.returncode != 0 and ENV_ONLY_FAILURE.search(syn_out):
        # NARROW, SELF-POLICING SKIP — see ENV_ONLY_FAILURE above.
        note("ansible-playbook lacks a LOCAL dependency kept outside the repo by design (the vault "
             "passphrase and/or the gitignored Galaxy collections), so --syntax-check/--list-hosts "
             "were not driven; every parse assertion above still ran")
    else:
        check(syn.returncode == 0,
              "`ansible-playbook --syntax-check playbooks/site.yml` FAILED — the provisioning chain "
              f"does not even load: {syn_out.strip()[:400]}")
        lh = ansible("--list-hosts", "playbooks/site.yml")
        check(lh.returncode == 0,
              "`ansible-playbook --list-hosts playbooks/site.yml` FAILED — the chain does not "
              f"resolve against the tracked inventory: "
              f"{((lh.stderr or '') + (lh.stdout or '')).strip()[:400]}")
    if lh is not None and lh.returncode == 0:
        # Parse plays -> hosts. NEVER `ansible-inventory --list`: it prints DECRYPTED vault values.
        plays: list[tuple[str, set[str]]] = []
        name, hosts, in_hosts = None, set(), False
        for line in lh.stdout.splitlines():
            m = re.match(r"\s*play #\d+ \([^)]*\):\s*(.*?)\s*TAGS:", line)
            if m:
                if name is not None:
                    plays.append((name, hosts))
                name, hosts, in_hosts = m.group(1), set(), False
                continue
            if re.match(r"\s*hosts \(\d+\):", line):
                in_hosts = True
                continue
            if in_hosts and line.strip() and not line.strip().startswith("pattern:"):
                hosts.add(line.strip())
        if name is not None:
            plays.append((name, hosts))
        check(len(plays) >= 4,
              f"--list-hosts resolved {len(plays)} play(s) — the four-stage chain did not expand")
        # Every lifecycle stage must cover EVERY dev node. A stage that reaches two of three
        # nodes means the third was finished by hand.
        STAGES = {
            "baseline": r"baseline",
            "cluster-join": r"^k3s (primary|secondar)",
            "cni": r"cilium",
            "manage (flux hand-off)": r"flux",
        }
        for stage, rx in STAGES.items():
            covered: set[str] = set()
            matched = 0
            for pname, phosts in plays:
                if re.search(rx, pname, re.I):
                    matched += 1
                    covered |= phosts
            check(matched >= 1, f"no play in site.yml implements the {stage} stage")
            check(covered == dev_hosts,
                  f"the {stage} stage targets {sorted(covered)}, not every dev node "
                  f"{sorted(dev_hosts)} — a node outside the code path is a hand-provisioned "
                  "node (R-ENV-01)")

# --------------------------------------------------------------------------- verdict
# NO-OP GUARD: a check that quietly asserts nothing is worse than no check.
check(CHECKS >= 30,
      f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
      "would be meaningless")

if FAILURES:
    print(f"CHECK FAIL (dev-nodes-provisioned-as-code): {len(FAILURES)} of {CHECKS} assertions failed",
          file=sys.stderr)
    for f in FAILURES:
        print(f"  - {f}", file=sys.stderr)
    raise SystemExit(1)
for n in NOTES:
    print(f"  note: {n}")
print(f"check OK: dev-nodes-provisioned-as-code — {CHECKS} assertions; "
      f"{len(dev_hosts)} dev node(s) {sorted(dev_hosts)} created by IaC ({sorted(providers.values())}), "
      f"baselined, cluster-joined and Flux-managed by the same chain. R-ENV-01's authored-path "
      "clause only — the live VMs' provenance, `tofu validate`, and rebuild/re-converge are NOT "
      "proven here (see the header's gap list; rebuild is L3 node-loss).")
PY