Wire L1 render conformance tests to the renderer (R-NET-01, R-NET-05/06/07)
Implements the two L1 rendering tests that were failing scaffolds: - egress-class-render (R-NET-05/06/07): renders vpn + internet fixtures, diffs each against a committed golden, asserts the vpn steering label is present on vpn and absent on internet (fail-closed), and that flipping the class changes ONLY the egress path (R-NET-07). This fixes the --requirements R-NET-05 merge gate. - default-deny-baseline-render (R-NET-01): asserts every render carries the namespaced default-deny baseline, shaped deny-all + both directions. Each test carries a pure-offline render-check.sh (renderer + golden diff, no cluster). run.sh runs L1 render checks offline and emits a REAL pass/fail JUnit (docs/14: L1 rendering assertions run in CI without a cluster) instead of skipping L1 with the cluster-gated layers; the chainsaw-test.yaml wrappers invoke the same render-check.sh so an in-cluster run executes the identical assertion. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
1 parent
6b2c04ed5c
commit
4a017c590e
12 files changed
+851
-26
No files matched your search
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the L1 *rendering* conformance checks OFFLINE (no cluster) and emit JUnit.
|
||||
|
||||
docs/14: "L1 rendering assertions run in CI without a cluster." L1 verifies the
|
||||
ServiceDefinition -> rendered-resources mapping against golden expectations; unlike L2/L3
|
||||
it needs no live cluster. Each l1-rendering test dir carries a `render-check.sh` (renderer
|
||||
+ golden diff + structural asserts). run.sh invokes this for `--layer l1` so those tests
|
||||
report a REAL pass/fail instead of being skipped with the cluster-gated layers.
|
||||
|
||||
Pure standard library (matches the harness's no-pip-dependency posture); the render checks
|
||||
themselves shell out to catalog/render.py (the renderer's PyYAML dependency lives there).
|
||||
|
||||
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] <test-dir> [<test-dir> ...]
|
||||
Exit 0 iff every selected render check passes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from xml.dom import minidom
|
||||
|
||||
LABEL_LINE_RE_TMPL = r"^\s*{prefix}/(R-[A-Z0-9]+-[0-9]+)\s*:\s*[\"']?true"
|
||||
|
||||
|
||||
def collect_meta(test_yaml: Path, prefix: str):
|
||||
text = test_yaml.read_text() if test_yaml.exists() else ""
|
||||
name_m = re.search(r"^\s*name:\s*(\S+)", text, re.M)
|
||||
name = name_m.group(1) if name_m else test_yaml.parent.name
|
||||
rx = re.compile(LABEL_LINE_RE_TMPL.format(prefix=re.escape(prefix)), re.I)
|
||||
reqs = sorted({m.group(1).upper() for m in (rx.match(l) for l in text.splitlines()) if m})
|
||||
return name, reqs
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--junit", type=Path, required=True)
|
||||
ap.add_argument("--label-prefix", default="conformance.home.k3s")
|
||||
ap.add_argument("dirs", nargs="+", type=Path)
|
||||
args = ap.parse_args()
|
||||
|
||||
cases = [] # (name, reqs, dir, rc, output)
|
||||
for d in args.dirs:
|
||||
check = d / "render-check.sh"
|
||||
name, reqs = collect_meta(d / "chainsaw-test.yaml", args.label_prefix)
|
||||
if not check.exists():
|
||||
cases.append((name, reqs, d, 3, f"no render-check.sh in {d}"))
|
||||
continue
|
||||
proc = subprocess.run(
|
||||
["bash", str(check)], capture_output=True, text=True
|
||||
)
|
||||
out = (proc.stdout + proc.stderr).strip()
|
||||
cases.append((name, reqs, d, proc.returncode, out))
|
||||
|
||||
failures = sum(1 for c in cases if c[3] not in (0, 3))
|
||||
skipped = sum(1 for c in cases if c[3] == 3)
|
||||
|
||||
suite = ET.Element(
|
||||
"testsuite",
|
||||
{
|
||||
"name": "home-k3s-conformance L1 (offline render)",
|
||||
"tests": str(len(cases)),
|
||||
"failures": str(failures),
|
||||
"errors": "0",
|
||||
"skipped": str(skipped),
|
||||
},
|
||||
)
|
||||
for name, reqs, d, rc, out in cases:
|
||||
tc = ET.SubElement(
|
||||
suite,
|
||||
"testcase",
|
||||
{"classname": f"L1:{','.join(reqs) if reqs else 'UNTAGGED'}", "name": name},
|
||||
)
|
||||
props = ET.SubElement(tc, "properties")
|
||||
ET.SubElement(props, "property", {"name": "requirements", "value": ",".join(reqs)})
|
||||
ET.SubElement(props, "property", {"name": "layer", "value": "L1"})
|
||||
ET.SubElement(props, "property", {"name": "path", "value": str(d)})
|
||||
if rc == 3:
|
||||
ET.SubElement(tc, "skipped", {"message": out or "no render-check.sh"})
|
||||
elif rc != 0:
|
||||
fail = ET.SubElement(tc, "failure", {"message": f"render-check exit {rc}"})
|
||||
fail.text = out
|
||||
status = "PASS" if rc == 0 else ("SKIP" if rc == 3 else "FAIL")
|
||||
print(f"[l1-offline] {status:4s} {name} ({','.join(reqs) or 'untagged'})",
|
||||
file=sys.stderr)
|
||||
if rc not in (0, 3):
|
||||
print(out, file=sys.stderr)
|
||||
|
||||
root = ET.Element(
|
||||
"testsuites",
|
||||
{
|
||||
"name": "home-k3s-conformance",
|
||||
"tests": str(len(cases)),
|
||||
"failures": str(failures),
|
||||
"errors": "0",
|
||||
"skipped": str(skipped),
|
||||
},
|
||||
)
|
||||
root.append(suite)
|
||||
args.junit.parent.mkdir(parents=True, exist_ok=True)
|
||||
xml = minidom.parseString(ET.tostring(root, "utf-8")).toprettyxml(indent=" ")
|
||||
args.junit.write_text(xml)
|
||||
print(f"[l1-offline] wrote JUnit ({len(cases)} case(s), {failures} failure(s)) -> "
|
||||
f"{args.junit}", file=sys.stderr)
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user