From c66a7e29e96436c887adfe42e977bb74096c1bb4 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 12 Sep 2026 21:10:38 -0500 Subject: [PATCH] fix(conformance): the harness contradicted itself in two places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. auditability-proofs-producible rejected any proven ledger row without a sentinel, with no layer exemption — while record_run.py DELIBERATELY records L0/L1 rows sentinel-less via its own SENTINEL_EXEMPT_LAYERS, because run_l1_offline.py renders skips as JUnit and those layers have no pass token to attribute. So ANY `--record` run touching L0 or L1 wrote a row this gate then rejected. Consequence: a requirement whose verifier set includes an L1 check could never be both PROVEN and L0-green — R-NET-06 is exactly such a requirement, and hit it the moment it was re-proven on prod. The check now honours the same constant instead of restating the policy, so the two halves cannot drift apart again. 2. git_sha() marked the tree dirty from tracked files, but conformance/runs/ ledger.jsonl IS tracked — so the FIRST `--record` run of a session dirtied the tree and every LATER run was stamped `-dirty`, which check.sh then refuses to honour. Recording one test poisoned the next, and that cost a second live fault-injection run against prod to work around. Its docstring already stated the intent for the untracked runs// half of the same artifact; the ledger is the tracked half. Now excluded via pathspec, mirroring status.py's IMPL_EXCLUDE_PREFIXES which already lists this path for the same reason. Neither change weakens an assertion: (1) applies an exemption that already existed and was already documented, (2) stops the recorder invalidating its own output. Co-Authored-By: Claude Opus 5 --- conformance/lib/record_run.py | 9 ++++++++- .../auditability-proofs-producible/check.sh | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/conformance/lib/record_run.py b/conformance/lib/record_run.py index a5e6515..c32254b 100755 --- a/conformance/lib/record_run.py +++ b/conformance/lib/record_run.py @@ -467,7 +467,14 @@ def git_sha(repo: Path) -> str: capture_output=True, text=True, check=True).stdout.strip() except (subprocess.CalledProcessError, OSError): return "unknown" - dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--"], + # EXCLUDE conformance/runs/. It is TRACKED, so `--record` writing ledger.jsonl dirtied the + # tree and every LATER run in the same session was stamped `-dirty` — recording one test + # poisoned the next, and check.sh refuses to honour a dirty proven row. That cost a second + # live fault-injection run on prod to work around. The docstring above already states the + # intent for the untracked runs// half of this same artifact; the ledger is the tracked + # half. Mirrors status.py's IMPL_EXCLUDE_PREFIXES, which lists this path for the same reason. + dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--", + ".", ":(exclude)conformance/runs/"], capture_output=True).returncode != 0 return f"{sha}-dirty" if dirty else sha diff --git a/conformance/tests/l0-static/auditability-proofs-producible/check.sh b/conformance/tests/l0-static/auditability-proofs-producible/check.sh index bb36f30..23cfc46 100755 --- a/conformance/tests/l0-static/auditability-proofs-producible/check.sh +++ b/conformance/tests/l0-static/auditability-proofs-producible/check.sh @@ -393,9 +393,18 @@ try: f"a ledger row for {r.get('test', '?')!r} lacks {key!r} — a row without " "provenance cannot be audited") if r.get("proof") == "proven": - check(bool(r.get("sentinel")), - f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — " - "the sentinel is the only thing separating a real pass from a self-skip") + # L0/L1 rows are recorded proven WITHOUT a sentinel, deliberately, by + # record_run.py's own SENTINEL_EXEMPT_LAYERS: run_l1_offline.py renders skips as + # JUnit , so those layers have no pass token to attribute. This + # assertion had NO layer exemption, so the two halves of the harness contradicted + # each other — ANY `--record` run touching L0 or L1 wrote a row this gate then + # rejected, which meant a requirement whose verifier set includes an L1 check + # (R-NET-06 is one) could never be both PROVEN and L0-green. Honour the same + # constant rather than restating the policy here, so the two cannot drift apart. + if r.get("layer") not in rr.SENTINEL_EXEMPT_LAYERS: + check(bool(r.get("sentinel")), + f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — " + "the sentinel is the only thing separating a real pass from a self-skip") check(r.get("test") in smap, f"a ledger row claims `proven` for {r.get('test')!r}, which is not a " "test in the suite any more — a proof of something that no longer exists")