jakman85andClaude Opus 4.8
9d8e03892f
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
320 lines
13 KiB
Python
Executable File
320 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Requirement-coverage report generator (R-TEST-04).
|
|
|
|
Maps EVERY R-* requirement in the authoritative registry to the Chainsaw test(s) that
|
|
verify it, and flags requirements with no verifying test as visible, blocking gaps.
|
|
|
|
Single source of truth for coverage is the per-requirement label on each Chainsaw Test:
|
|
|
|
metadata:
|
|
labels:
|
|
conformance.home.k3s/R-NET-01: "true"
|
|
|
|
Reads:
|
|
* requirements/registry.yaml -- the mirror of every spec R-* ID (see that file).
|
|
* tests/**/chainsaw-test.yaml -- discovers labels + the requirements annotation.
|
|
|
|
Writes (under --out, default _artifacts/coverage/):
|
|
* coverage.json -- machine-readable {requirement: [tests...]}, gaps, safety gaps.
|
|
* coverage.md -- human-readable table for PRs / CI logs.
|
|
|
|
Exit status / gating (R-TEST-03 gates are *scoped*, not "all 51 always"):
|
|
0 always in plain report mode.
|
|
--strict HYGIENE gate (always-on, WS-TEST owns it): fails on tag/annotation
|
|
drift or tags absent from the registry. Does NOT fail merely because
|
|
a future-phase requirement has no test yet — those are visible gaps,
|
|
not harness defects.
|
|
--require IDs SCOPED completeness gate (merge gate): fails if any listed requirement
|
|
is uncovered, or a listed safety-critical requirement lacks an L3 test.
|
|
CI passes the requirement IDs the change touches.
|
|
--require-all FULL completeness gate (phase-exit / final): fails if ANY registry
|
|
requirement is uncovered or any safety-critical one lacks an L3 test.
|
|
--allow-gap IDs exempt specific IDs from the completeness gates (documented, out-of-
|
|
scope gaps such as R-STATE-02 node-loss owned by another workstream).
|
|
|
|
Pure standard library (no PyYAML) so the harness has no pip dependency: the registry and
|
|
the test label/annotation lines we care about are parsed with line-oriented regexes, and
|
|
the file formats are constrained (documented in registry.yaml / common.sh) to keep that
|
|
parse deterministic.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
LABEL_PREFIX = "conformance.home.k3s"
|
|
RID_RE = re.compile(r"R-[A-Z0-9]+-[0-9]+")
|
|
LABEL_LINE_RE = re.compile(
|
|
r"^\s*" + re.escape(LABEL_PREFIX) + r"/(R-[A-Z0-9]+-[0-9]+)\s*:\s*[\"']?true", re.I
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- registry
|
|
def parse_registry(path: Path) -> list[dict]:
|
|
"""Parse the constrained registry.yaml (`- id:` blocks, single-line scalars)."""
|
|
reqs: list[dict] = []
|
|
cur: dict | None = None
|
|
for raw in path.read_text().splitlines():
|
|
line = raw.split("#", 1)[0] if raw.lstrip().startswith("#") else raw
|
|
m = re.match(r"^\s*-\s*id:\s*(\S+)", line)
|
|
if m:
|
|
if cur:
|
|
reqs.append(cur)
|
|
cur = {"id": m.group(1).strip(), "doc": "", "safety_critical": False,
|
|
"summary": ""}
|
|
continue
|
|
if cur is None:
|
|
continue
|
|
m = re.match(r"^\s+doc:\s*(.+?)\s*$", line)
|
|
if m:
|
|
cur["doc"] = m.group(1).strip().strip('"')
|
|
continue
|
|
m = re.match(r"^\s+safety_critical:\s*(true|false)\s*$", line)
|
|
if m:
|
|
cur["safety_critical"] = (m.group(1) == "true")
|
|
continue
|
|
m = re.match(r"^\s+summary:\s*(.+?)\s*$", line)
|
|
if m:
|
|
cur["summary"] = m.group(1).strip().strip('"')
|
|
if cur:
|
|
reqs.append(cur)
|
|
return reqs
|
|
|
|
|
|
# ------------------------------------------------------------------------------ tests
|
|
def parse_test(path: Path) -> dict:
|
|
"""Extract name, verified R-IDs (from labels), annotation list, and layer."""
|
|
text = path.read_text()
|
|
labels: set[str] = set()
|
|
for line in text.splitlines():
|
|
m = LABEL_LINE_RE.match(line)
|
|
if m:
|
|
labels.add(m.group(1).upper())
|
|
|
|
ann_reqs: set[str] = set()
|
|
m = re.search(r"^\s*" + re.escape(LABEL_PREFIX) + r"/requirements:\s*(.+)$",
|
|
text, re.M)
|
|
if m:
|
|
ann_reqs = {r.upper() for r in RID_RE.findall(m.group(1))}
|
|
|
|
layer = ""
|
|
m = re.search(r"^\s*" + re.escape(LABEL_PREFIX) + r"/layer:\s*[\"']?([LlAaMm0-9]+)",
|
|
text, re.M)
|
|
if m:
|
|
layer = m.group(1).upper()
|
|
|
|
name = ""
|
|
m = re.search(r"^\s*name:\s*(\S+)", text, re.M)
|
|
if m:
|
|
name = m.group(1)
|
|
|
|
# Layer inferred from path as a fallback / cross-check.
|
|
rel = str(path)
|
|
if not layer:
|
|
if "l1-rendering" in rel:
|
|
layer = "L1"
|
|
elif "l2-behavioral" in rel:
|
|
layer = "L2"
|
|
elif "l3-fault-injection" in rel:
|
|
layer = "L3"
|
|
elif "/meta/" in rel:
|
|
layer = "META"
|
|
|
|
return {
|
|
"name": name,
|
|
"path": rel,
|
|
"labels": sorted(labels),
|
|
"annotation_requirements": sorted(ann_reqs),
|
|
"layer": layer,
|
|
}
|
|
|
|
|
|
def discover_tests(tests_dir: Path) -> list[dict]:
|
|
return [parse_test(p) for p in sorted(tests_dir.rglob("chainsaw-test.yaml"))]
|
|
|
|
|
|
# ----------------------------------------------------------------------------- report
|
|
def build(registry: list[dict], tests: list[dict]) -> dict:
|
|
by_req: dict[str, list[dict]] = {r["id"]: [] for r in registry}
|
|
unknown: list[tuple[str, str]] = [] # (rid, test path) not in registry
|
|
mismatches: list[dict] = [] # label set != annotation set
|
|
|
|
for t in tests:
|
|
lbl = set(t["labels"])
|
|
ann = set(t["annotation_requirements"])
|
|
if lbl != ann:
|
|
mismatches.append({"test": t["path"], "labels": sorted(lbl),
|
|
"annotation": sorted(ann)})
|
|
for rid in lbl:
|
|
if rid in by_req:
|
|
by_req[rid].append(t)
|
|
else:
|
|
unknown.append((rid, t["path"]))
|
|
|
|
reg_index = {r["id"]: r for r in registry}
|
|
covered, uncovered, safety_gaps = [], [], []
|
|
for rid, req in reg_index.items():
|
|
verifiers = by_req[rid]
|
|
if verifiers:
|
|
covered.append(rid)
|
|
else:
|
|
uncovered.append(rid)
|
|
if req["safety_critical"]:
|
|
has_l3 = any(v["layer"] == "L3" for v in verifiers)
|
|
if not has_l3:
|
|
safety_gaps.append(rid)
|
|
|
|
return {
|
|
"summary": {
|
|
"total_requirements": len(registry),
|
|
"covered": len(covered),
|
|
"uncovered": len(uncovered),
|
|
"coverage_pct": round(100.0 * len(covered) / max(1, len(registry)), 1),
|
|
"safety_critical_gaps": len(safety_gaps),
|
|
"annotation_label_mismatches": len(mismatches),
|
|
"unknown_requirement_tags": len(unknown),
|
|
"total_tests": len(tests),
|
|
},
|
|
"by_requirement": {
|
|
r["id"]: {
|
|
"doc": r["doc"],
|
|
"safety_critical": r["safety_critical"],
|
|
"summary": r["summary"],
|
|
"verified_by": [
|
|
{"name": v["name"], "path": v["path"], "layer": v["layer"]}
|
|
for v in by_req[r["id"]]
|
|
],
|
|
}
|
|
for r in registry
|
|
},
|
|
"uncovered": sorted(uncovered),
|
|
"safety_critical_gaps": sorted(safety_gaps),
|
|
"annotation_label_mismatches": mismatches,
|
|
"unknown_requirement_tags": [{"requirement": u[0], "test": u[1]} for u in unknown],
|
|
}
|
|
|
|
|
|
def render_md(rep: dict) -> str:
|
|
s = rep["summary"]
|
|
out: list[str] = []
|
|
out.append("# Requirement coverage report\n")
|
|
out.append(
|
|
f"- Requirements: **{s['total_requirements']}** | "
|
|
f"Covered: **{s['covered']}** ({s['coverage_pct']}%) | "
|
|
f"Uncovered: **{s['uncovered']}**\n"
|
|
f"- Safety-critical requirements missing an L3 fault-injection test: "
|
|
f"**{s['safety_critical_gaps']}**\n"
|
|
f"- Scaffolded tests: **{s['total_tests']}** | "
|
|
f"Annotation/label mismatches: **{s['annotation_label_mismatches']}** | "
|
|
f"Unknown tags: **{s['unknown_requirement_tags']}**\n"
|
|
)
|
|
out.append("\n> Phase 0: tests are scaffolds. A requirement is **satisfied** only "
|
|
"when its tagged test passes green against a cluster (R-TEST-03) — this "
|
|
"report tracks *scaffold* coverage, i.e. that a verifying test exists.\n")
|
|
|
|
out.append("\n## Coverage by requirement\n")
|
|
out.append("| Requirement | Safety | Verified by (layer) | Status |")
|
|
out.append("|---|---|---|---|")
|
|
for rid, info in rep["by_requirement"].items():
|
|
vs = info["verified_by"]
|
|
if vs:
|
|
cell = "<br>".join(f"`{v['name']}` ({v['layer']})" for v in vs)
|
|
status = "covered"
|
|
else:
|
|
cell = "—"
|
|
status = "**GAP**"
|
|
sc = "yes" if info["safety_critical"] else ""
|
|
out.append(f"| {rid} | {sc} | {cell} | {status} |")
|
|
|
|
if rep["uncovered"]:
|
|
out.append("\n## Uncovered requirements (gaps)\n")
|
|
for rid in rep["uncovered"]:
|
|
out.append(f"- {rid} — {rep['by_requirement'][rid]['summary']}")
|
|
if rep["safety_critical_gaps"]:
|
|
out.append("\n## Safety-critical requirements without an L3 fault-injection test "
|
|
"(R-TEST-05)\n")
|
|
for rid in rep["safety_critical_gaps"]:
|
|
out.append(f"- {rid}")
|
|
if rep["annotation_label_mismatches"]:
|
|
out.append("\n## Tests whose labels disagree with their `requirements` annotation\n")
|
|
for m in rep["annotation_label_mismatches"]:
|
|
out.append(f"- `{m['test']}` labels={m['labels']} annotation={m['annotation']}")
|
|
if rep["unknown_requirement_tags"]:
|
|
out.append("\n## Tags not present in the registry (typo or spec drift)\n")
|
|
for u in rep["unknown_requirement_tags"]:
|
|
out.append(f"- {u['requirement']} in `{u['test']}`")
|
|
return "\n".join(out) + "\n"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
here = Path(__file__).resolve().parent.parent # conformance/
|
|
ap.add_argument("--registry", type=Path,
|
|
default=here / "requirements" / "registry.yaml")
|
|
ap.add_argument("--tests", type=Path, default=here / "tests")
|
|
ap.add_argument("--out", type=Path, default=here / "_artifacts" / "coverage")
|
|
ap.add_argument("--strict", action="store_true",
|
|
help="hygiene gate: fail on tag/annotation drift or unknown tags")
|
|
ap.add_argument("--require", default="",
|
|
help="comma/space list of R-IDs that MUST be covered (merge gate)")
|
|
ap.add_argument("--require-all", action="store_true",
|
|
help="every registry requirement MUST be covered (phase-exit gate)")
|
|
ap.add_argument("--allow-gap", default="",
|
|
help="comma/space list of R-IDs exempt from completeness gates")
|
|
args = ap.parse_args()
|
|
|
|
registry = parse_registry(args.registry)
|
|
tests = discover_tests(args.tests)
|
|
rep = build(registry, tests)
|
|
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
(args.out / "coverage.json").write_text(json.dumps(rep, indent=2) + "\n")
|
|
(args.out / "coverage.md").write_text(render_md(rep))
|
|
|
|
s = rep["summary"]
|
|
print(f"[coverage] {s['covered']}/{s['total_requirements']} requirements have a "
|
|
f"scaffolded test ({s['coverage_pct']}%); {s['uncovered']} gap(s), "
|
|
f"{s['safety_critical_gaps']} safety-critical gap(s).")
|
|
print(f"[coverage] wrote {args.out/'coverage.md'} and {args.out/'coverage.json'}")
|
|
|
|
def split(v):
|
|
return {t.upper() for t in re.split(r"[,\s]+", v) if t}
|
|
|
|
exempt = split(args.allow_gap)
|
|
uncovered = set(rep["uncovered"])
|
|
safety_gaps = set(rep["safety_critical_gaps"])
|
|
rc = 0
|
|
|
|
# HYGIENE (always-on): things WS-TEST controls and must keep green.
|
|
if args.strict:
|
|
if rep["annotation_label_mismatches"] or rep["unknown_requirement_tags"]:
|
|
print("[coverage] STRICT(hygiene): FAILED — label/annotation drift or "
|
|
"tags not in the registry.", file=sys.stderr)
|
|
rc = 1
|
|
|
|
# SCOPED completeness (merge gate): only the requirements the change touches.
|
|
required = split(args.require)
|
|
if args.require_all:
|
|
required |= {r["id"] for r in registry}
|
|
required -= exempt
|
|
if required:
|
|
miss = sorted(required & uncovered)
|
|
sgap = sorted(required & safety_gaps)
|
|
if miss:
|
|
print(f"[coverage] REQUIRE: FAILED — required requirement(s) uncovered: "
|
|
f"{miss}", file=sys.stderr)
|
|
rc = 1
|
|
if sgap:
|
|
print(f"[coverage] REQUIRE: FAILED — required safety-critical requirement(s) "
|
|
f"lack an L3 fault-injection test: {sgap}", file=sys.stderr)
|
|
rc = 1
|
|
return rc
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|