Home-Network/home-network-k3s-impl
2
0 Fork 0
Code Issues 3 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

T-0.4 Chainsaw conformance harness + coverage + behavior scaffolds (R-TEST-01..06)

Browse Source 
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed 2026-08-16 00:08:32 -05:00
1 parent a778d923b6
commit 9d8e03892f
31 files changed
+2445 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

conformance/lib/common.sh
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# Shared helpers for the conformance harness. Sourced by run.sh; not executed directly.
# shellcheck shell=bash
# --- Requirement-tagging convention (single source of truth) --------------------------
# A Chainsaw Test declares the requirement(s) it verifies with a per-requirement label:
#
# metadata:
# labels:
# conformance.home.k3s/R-NET-01: "true" # one label per verified R-* ID
# annotations:
# conformance.home.k3s/requirements: "R-NET-01, R-NET-02" # human summary
# conformance.home.k3s/layer: "L2"
# conformance.home.k3s/safety-critical: "false"
#
# The LABEL is authoritative and does double duty:
# * `chainsaw test --selector conformance.home.k3s/R-NET-01=true` runs the subset.
# * lib/coverage.py reads the same labels to map requirement -> verifying test(s).
# The annotation is a convenience for humans/reports and MUST agree with the labels
# (coverage.py flags a mismatch). This is the convention referenced in README.md.
export CONF_LABEL_PREFIX="conformance.home.k3s"
# Absolute path to the conformance root (dir containing run.sh). Set by run.sh.
: "${CONF_ROOT:?CONF_ROOT must be set before sourcing common.sh}"
# --- Test discovery -------------------------------------------------------------------
# Every leaf test is a directory containing a `chainsaw-test.yaml`.
conf_all_test_files() {
find "${CONF_ROOT}/tests" -type f -name 'chainsaw-test.yaml' 2>/dev/null | sort
}
# Directories of tests that carry ANY of the given requirement labels (space-separated
# R-IDs). Used to translate `--requirements R-NET-01,R-NET-06` into a set of test dirs
# so we run each matching test exactly once (union, deduplicated).
conf_dirs_for_requirements() {
local reqs="$1" f dir rid matched
while IFS= read -r f; do
[ -n "$f" ] || continue
matched=0
for rid in $reqs; do
# Match a label line: conformance.home.k3s/R-NET-01: "true"
if grep -Eq "^[[:space:]]*${CONF_LABEL_PREFIX}/${rid}:[[:space:]]*[\"']?true" "$f"; then
matched=1; break
fi
done
[ "$matched" = 1 ] && dirname "$f"
done < <(conf_all_test_files) | sort -u
}
# Directories for a given layer subtree: l1 | l2 | l3 | meta | all
conf_dirs_for_layer() {
local layer="$1" sub
case "$layer" in
all) find "${CONF_ROOT}/tests" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; | sort -u ;;
l1) sub="l1-rendering" ;;
l2) sub="l2-behavioral" ;;
l3) sub="l3-fault-injection" ;;
meta) sub="meta" ;;
*) echo "unknown layer: $layer" >&2; return 2 ;;
esac
if [ "$layer" != all ]; then
find "${CONF_ROOT}/tests/${sub}" -type f -name 'chainsaw-test.yaml' -exec dirname {} \; 2>/dev/null | sort -u
fi
}
# Normalize a comma/space separated requirement list to space-separated, uppercased.
conf_normalize_reqs() {
echo "$1" | tr ',' ' ' | tr '[:lower:]' '[:upper:]' | xargs
}
# Is a live cluster reachable? (Preflight for L1/L2/L3 execution.)
conf_cluster_reachable() {
command -v kubectl >/dev/null 2>&1 || return 1
kubectl cluster-info >/dev/null 2>&1
}
conformance/lib/coverage.py
Executable
+319
View File
@@ -0,0 +1,319 @@
#!/usr/bin/env python3
"""Requirement-coverage report generator (R-TEST-04).
Maps EVERY R-* requirement in the authoritative registry to the Chainsaw test(s) that
verify it, and flags requirements with no verifying test as visible, blocking gaps.
Single source of truth for coverage is the per-requirement label on each Chainsaw Test:
metadata:
labels:
conformance.home.k3s/R-NET-01: "true"
Reads:
* requirements/registry.yaml -- the mirror of every spec R-* ID (see that file).
* tests/**/chainsaw-test.yaml -- discovers labels + the requirements annotation.
Writes (under --out, default _artifacts/coverage/):
* coverage.json -- machine-readable {requirement: [tests...]}, gaps, safety gaps.
* coverage.md -- human-readable table for PRs / CI logs.
Exit status / gating (R-TEST-03 gates are *scoped*, not "all 51 always"):
0 always in plain report mode.
--strict HYGIENE gate (always-on, WS-TEST owns it): fails on tag/annotation
drift or tags absent from the registry. Does NOT fail merely because
a future-phase requirement has no test yet — those are visible gaps,
not harness defects.
--require IDs SCOPED completeness gate (merge gate): fails if any listed requirement
is uncovered, or a listed safety-critical requirement lacks an L3 test.
CI passes the requirement IDs the change touches.
--require-all FULL completeness gate (phase-exit / final): fails if ANY registry
requirement is uncovered or any safety-critical one lacks an L3 test.
--allow-gap IDs exempt specific IDs from the completeness gates (documented, out-of-
scope gaps such as R-STATE-02 node-loss owned by another workstream).
Pure standard library (no PyYAML) so the harness has no pip dependency: the registry and
the test label/annotation lines we care about are parsed with line-oriented regexes, and
the file formats are constrained (documented in registry.yaml / common.sh) to keep that
parse deterministic.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from pathlib import Path
LABEL_PREFIX = "conformance.home.k3s"
RID_RE = re.compile(r"R-[A-Z0-9]+-[0-9]+")
LABEL_LINE_RE = re.compile(
r"^\s*" + re.escape(LABEL_PREFIX) + r"/(R-[A-Z0-9]+-[0-9]+)\s*:\s*[\"']?true", re.I
)
# --------------------------------------------------------------------------- registry
def parse_registry(path: Path) -> list[dict]:
"""Parse the constrained registry.yaml (`- id:` blocks, single-line scalars)."""
reqs: list[dict] = []
cur: dict | None = None
for raw in path.read_text().splitlines():
line = raw.split("#", 1)[0] if raw.lstrip().startswith("#") else raw
m = re.match(r"^\s*-\s*id:\s*(\S+)", line)
if m:
if cur:
reqs.append(cur)
cur = {"id": m.group(1).strip(), "doc": "", "safety_critical": False,
"summary": ""}
continue
if cur is None:
continue
m = re.match(r"^\s+doc:\s*(.+?)\s*$", line)
if m:
cur["doc"] = m.group(1).strip().strip('"')
continue
m = re.match(r"^\s+safety_critical:\s*(true|false)\s*$", line)
if m:
cur["safety_critical"] = (m.group(1) == "true")
continue
m = re.match(r"^\s+summary:\s*(.+?)\s*$", line)
if m:
cur["summary"] = m.group(1).strip().strip('"')
if cur:
reqs.append(cur)
return reqs
# ------------------------------------------------------------------------------ tests
def parse_test(path: Path) -> dict:
"""Extract name, verified R-IDs (from labels), annotation list, and layer."""
text = path.read_text()
labels: set[str] = set()
for line in text.splitlines():
m = LABEL_LINE_RE.match(line)
if m:
labels.add(m.group(1).upper())
ann_reqs: set[str] = set()
m = re.search(r"^\s*" + re.escape(LABEL_PREFIX) + r"/requirements:\s*(.+)$",
text, re.M)
if m:
ann_reqs = {r.upper() for r in RID_RE.findall(m.group(1))}
layer = ""
m = re.search(r"^\s*" + re.escape(LABEL_PREFIX) + r"/layer:\s*[\"']?([LlAaMm0-9]+)",
text, re.M)
if m:
layer = m.group(1).upper()
name = ""
m = re.search(r"^\s*name:\s*(\S+)", text, re.M)
if m:
name = m.group(1)
# Layer inferred from path as a fallback / cross-check.
rel = str(path)
if not layer:
if "l1-rendering" in rel:
layer = "L1"
elif "l2-behavioral" in rel:
layer = "L2"
elif "l3-fault-injection" in rel:
layer = "L3"
elif "/meta/" in rel:
layer = "META"
return {
"name": name,
"path": rel,
"labels": sorted(labels),
"annotation_requirements": sorted(ann_reqs),
"layer": layer,
}
def discover_tests(tests_dir: Path) -> list[dict]:
return [parse_test(p) for p in sorted(tests_dir.rglob("chainsaw-test.yaml"))]
# ----------------------------------------------------------------------------- report
def build(registry: list[dict], tests: list[dict]) -> dict:
by_req: dict[str, list[dict]] = {r["id"]: [] for r in registry}
unknown: list[tuple[str, str]] = [] # (rid, test path) not in registry
mismatches: list[dict] = [] # label set != annotation set
for t in tests:
lbl = set(t["labels"])
ann = set(t["annotation_requirements"])
if lbl != ann:
mismatches.append({"test": t["path"], "labels": sorted(lbl),
"annotation": sorted(ann)})
for rid in lbl:
if rid in by_req:
by_req[rid].append(t)
else:
unknown.append((rid, t["path"]))
reg_index = {r["id"]: r for r in registry}
covered, uncovered, safety_gaps = [], [], []
for rid, req in reg_index.items():
verifiers = by_req[rid]
if verifiers:
covered.append(rid)
else:
uncovered.append(rid)
if req["safety_critical"]:
has_l3 = any(v["layer"] == "L3" for v in verifiers)
if not has_l3:
safety_gaps.append(rid)
return {
"summary": {
"total_requirements": len(registry),
"covered": len(covered),
"uncovered": len(uncovered),
"coverage_pct": round(100.0 * len(covered) / max(1, len(registry)), 1),
"safety_critical_gaps": len(safety_gaps),
"annotation_label_mismatches": len(mismatches),
"unknown_requirement_tags": len(unknown),
"total_tests": len(tests),
},
"by_requirement": {
r["id"]: {
"doc": r["doc"],
"safety_critical": r["safety_critical"],
"summary": r["summary"],
"verified_by": [
{"name": v["name"], "path": v["path"], "layer": v["layer"]}
for v in by_req[r["id"]]
],
}
for r in registry
},
"uncovered": sorted(uncovered),
"safety_critical_gaps": sorted(safety_gaps),
"annotation_label_mismatches": mismatches,
"unknown_requirement_tags": [{"requirement": u[0], "test": u[1]} for u in unknown],
}
def render_md(rep: dict) -> str:
s = rep["summary"]
out: list[str] = []
out.append("# Requirement coverage report\n")
out.append(
f"- Requirements: **{s['total_requirements']}** | "
f"Covered: **{s['covered']}** ({s['coverage_pct']}%) | "
f"Uncovered: **{s['uncovered']}**\n"
f"- Safety-critical requirements missing an L3 fault-injection test: "
f"**{s['safety_critical_gaps']}**\n"
f"- Scaffolded tests: **{s['total_tests']}** | "
f"Annotation/label mismatches: **{s['annotation_label_mismatches']}** | "
f"Unknown tags: **{s['unknown_requirement_tags']}**\n"
)
out.append("\n> Phase 0: tests are scaffolds. A requirement is **satisfied** only "
"when its tagged test passes green against a cluster (R-TEST-03) — this "
"report tracks *scaffold* coverage, i.e. that a verifying test exists.\n")
out.append("\n## Coverage by requirement\n")
out.append("| Requirement | Safety | Verified by (layer) | Status |")
out.append("|---|---|---|---|")
for rid, info in rep["by_requirement"].items():
vs = info["verified_by"]
if vs:
cell = "<br>".join(f"`{v['name']}` ({v['layer']})" for v in vs)
status = "covered"
else:
cell = "—"
status = "**GAP**"
sc = "yes" if info["safety_critical"] else ""
out.append(f"| {rid} | {sc} | {cell} | {status} |")
if rep["uncovered"]:
out.append("\n## Uncovered requirements (gaps)\n")
for rid in rep["uncovered"]:
out.append(f"- {rid} — {rep['by_requirement'][rid]['summary']}")
if rep["safety_critical_gaps"]:
out.append("\n## Safety-critical requirements without an L3 fault-injection test "
"(R-TEST-05)\n")
for rid in rep["safety_critical_gaps"]:
out.append(f"- {rid}")
if rep["annotation_label_mismatches"]:
out.append("\n## Tests whose labels disagree with their `requirements` annotation\n")
for m in rep["annotation_label_mismatches"]:
out.append(f"- `{m['test']}` labels={m['labels']} annotation={m['annotation']}")
if rep["unknown_requirement_tags"]:
out.append("\n## Tags not present in the registry (typo or spec drift)\n")
for u in rep["unknown_requirement_tags"]:
out.append(f"- {u['requirement']} in `{u['test']}`")
return "\n".join(out) + "\n"
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
here = Path(__file__).resolve().parent.parent # conformance/
ap.add_argument("--registry", type=Path,
default=here / "requirements" / "registry.yaml")
ap.add_argument("--tests", type=Path, default=here / "tests")
ap.add_argument("--out", type=Path, default=here / "_artifacts" / "coverage")
ap.add_argument("--strict", action="store_true",
help="hygiene gate: fail on tag/annotation drift or unknown tags")
ap.add_argument("--require", default="",
help="comma/space list of R-IDs that MUST be covered (merge gate)")
ap.add_argument("--require-all", action="store_true",
help="every registry requirement MUST be covered (phase-exit gate)")
ap.add_argument("--allow-gap", default="",
help="comma/space list of R-IDs exempt from completeness gates")
args = ap.parse_args()
registry = parse_registry(args.registry)
tests = discover_tests(args.tests)
rep = build(registry, tests)
args.out.mkdir(parents=True, exist_ok=True)
(args.out / "coverage.json").write_text(json.dumps(rep, indent=2) + "\n")
(args.out / "coverage.md").write_text(render_md(rep))
s = rep["summary"]
print(f"[coverage] {s['covered']}/{s['total_requirements']} requirements have a "
f"scaffolded test ({s['coverage_pct']}%); {s['uncovered']} gap(s), "
f"{s['safety_critical_gaps']} safety-critical gap(s).")
print(f"[coverage] wrote {args.out/'coverage.md'} and {args.out/'coverage.json'}")
def split(v):
return {t.upper() for t in re.split(r"[,\s]+", v) if t}
exempt = split(args.allow_gap)
uncovered = set(rep["uncovered"])
safety_gaps = set(rep["safety_critical_gaps"])
rc = 0
# HYGIENE (always-on): things WS-TEST controls and must keep green.
if args.strict:
if rep["annotation_label_mismatches"] or rep["unknown_requirement_tags"]:
print("[coverage] STRICT(hygiene): FAILED — label/annotation drift or "
"tags not in the registry.", file=sys.stderr)
rc = 1
# SCOPED completeness (merge gate): only the requirements the change touches.
required = split(args.require)
if args.require_all:
required |= {r["id"] for r in registry}
required -= exempt
if required:
miss = sorted(required & uncovered)
sgap = sorted(required & safety_gaps)
if miss:
print(f"[coverage] REQUIRE: FAILED — required requirement(s) uncovered: "
f"{miss}", file=sys.stderr)
rc = 1
if sgap:
print(f"[coverage] REQUIRE: FAILED — required safety-critical requirement(s) "
f"lack an L3 fault-injection test: {sgap}", file=sys.stderr)
rc = 1
return rc
if __name__ == "__main__":
raise SystemExit(main())
conformance/lib/emit_skipped_junit.py
Executable
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""Emit a JUnit report marking every discovered Chainsaw test as *skipped*.
Used in Phase 0 (and any run where no cluster is reachable): the behavior tests are
valid scaffolds but cannot execute without a cluster, so instead of failing the suite we
emit a JUnit file whose testcases are `<skipped>` with the reason. This keeps the
merge/phase gate green while the coverage report still shows which requirements have a
verifying test. As later phases deliver a dev/test cluster, run.sh hands execution to
Chainsaw, which emits the real pass/fail JUnit and this shim is no longer used.
Each testcase carries its verified R-* IDs in the `classname` + a property so the JUnit
remains requirement-tagged (R-TEST-04) even in skipped form.
"""
from __future__ import annotations
import argparse
import re
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
from xml.dom import minidom
LABEL_PREFIX = "conformance.home.k3s"
LABEL_LINE_RE = re.compile(
r"^\s*" + re.escape(LABEL_PREFIX) + r"/(R-[A-Z0-9]+-[0-9]+)\s*:\s*[\"']?true", re.I
)
def collect(path: Path) -> tuple[str, list[str], str]:
text = path.read_text()
name_m = re.search(r"^\s*name:\s*(\S+)", text, re.M)
name = name_m.group(1) if name_m else path.parent.name
reqs = sorted({m.group(1).upper() for m in
(LABEL_LINE_RE.match(l) for l in text.splitlines()) if m})
rel = str(path)
layer = ("L1" if "l1-rendering" in rel else "L2" if "l2-behavioral" in rel
else "L3" if "l3-fault-injection" in rel else "META")
return name, reqs, layer
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
here = Path(__file__).resolve().parent.parent
ap.add_argument("--tests", type=Path, default=here / "tests")
ap.add_argument("--out", type=Path,
default=here / "_artifacts" / "report.junit.xml")
ap.add_argument("--reason", default="no cluster reachable (Phase 0 scaffold): "
"behavior test not executed")
args = ap.parse_args()
files = sorted(args.tests.rglob("chainsaw-test.yaml"))
suite = ET.Element("testsuite", {
"name": "home-k3s-conformance (scaffold/skipped)",
"tests": str(len(files)),
"failures": "0",
"errors": "0",
"skipped": str(len(files)),
})
for f in files:
name, reqs, layer = collect(f)
tc = ET.SubElement(suite, "testcase", {
# classname carries layer + requirement tags so the JUnit stays traceable.
"classname": f"{layer}:{','.join(reqs) if reqs else 'UNTAGGED'}",
"name": name,
})
props = ET.SubElement(tc, "properties")
ET.SubElement(props, "property",
{"name": "requirements", "value": ",".join(reqs)})
ET.SubElement(props, "property", {"name": "layer", "value": layer})
ET.SubElement(props, "property", {"name": "path", "value": str(f)})
ET.SubElement(tc, "skipped", {"message": args.reason})
root = ET.Element("testsuites", {
"name": "home-k3s-conformance",
"tests": str(len(files)),
"skipped": str(len(files)),
"failures": "0",
"errors": "0",
})
root.append(suite)
args.out.parent.mkdir(parents=True, exist_ok=True)
xml = minidom.parseString(ET.tostring(root, "utf-8")).toprettyxml(indent=" ")
args.out.write_text(xml)
print(f"[junit] wrote skipped JUnit for {len(files)} scaffolded test(s) -> {args.out}",
file=sys.stderr)
return 0
if __name__ == "__main__":
raise SystemExit(main())