From 24b8111d0c39e9417e9f4d782f333f683540cfa7 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sun, 13 Sep 2026 21:13:51 -0500 Subject: [PATCH] fix(conformance): the ledger stamped every proof env=dev regardless of cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit record_run.py defaulted --env to the literal 'dev' (os.environ.get("CONF_ENV", "dev")) and run.sh hardcoded the same fallback at both call sites, while CONF_ENV is set nowhere in this repo. So every recorded proof claimed dev no matter which cluster it ran against. Harmless while dev was the only cluster. Dev was decommissioned 2026-09-12, and today a genuine DR-restore run against k3s-prod was recorded as env=dev — caught before it was committed, and the record discarded rather than hand-edited. A proof that lies about WHERE it was proven is worse than no proof, because locating the demonstration is the ledger's whole job; it would also have looked like a sixth dev-only row while actually being the prod proof Phase 7 is waiting on. env is now derived from k3s-prod (k3s-prod -> prod), which cannot drift from the cluster the run actually talked to. An undeterminable context records 'unknown', never a cluster name, and an unexpected context is recorded verbatim so it is visible rather than coerced. CONF_ENV still overrides. Recorder selftest passes. Co-Authored-By: Claude Opus 5 --- conformance/lib/record_run.py | 34 +++++++++++++++++++++++++++++++++- conformance/run.sh | 4 ++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/conformance/lib/record_run.py b/conformance/lib/record_run.py index c32254b..77d463f 100755 --- a/conformance/lib/record_run.py +++ b/conformance/lib/record_run.py @@ -489,6 +489,36 @@ def pinned_chainsaw_version(conf_root: Path) -> str: return "unknown" +def context_env() -> str: + """The env a run TARGETED, derived from the live kube context — never guessed. + + WHY THIS IS NOT A DEFAULT STRING. `--env` used to fall back to the literal "dev" + (`os.environ.get("CONF_ENV", "dev")`), and CONF_ENV is set nowhere in this repo, so EVERY + recorded proof was stamped `env: dev` regardless of the cluster it ran against. That was + harmless while dev was the only cluster. Dev was decommissioned 2026-09-12 and prod is now + the only one, so on 2026-09-13 a genuine prod DR-restore proof was recorded as `env: dev` — + a proof that lies about where it was proven is worse than no proof, because the ledger's + whole purpose is to say what was demonstrated WHERE. Caught before it was committed. + + Derived from `kubectl config current-context` (`k3s-prod` -> `prod`), which cannot drift + from the cluster the run actually talked to. Undeterminable is "unknown", NEVER a cluster + name: an unreadable context must not silently attribute a proof to an environment. + """ + try: + p = subprocess.run(["kubectl", "config", "current-context"], + capture_output=True, text=True, timeout=15) + except (OSError, subprocess.TimeoutExpired): + return "unknown" + if p.returncode != 0: + return "unknown" + ctx = (p.stdout or "").strip() + if not ctx: + return "unknown" + # contexts in this repo are named k3s-; anything else is recorded verbatim so an + # unexpected context is visible in the ledger rather than being coerced into a known name. + return ctx[len("k3s-"):] if ctx.startswith("k3s-") else ctx + + def server_kube_version() -> str: """Server gitVersion via `kubectl version` (read-only), else "unknown".""" for args in (["kubectl", "version", "-o", "json"], @@ -883,7 +913,9 @@ def main() -> int: help=f"append-only ledger (default {DEFAULT_LEDGER})") ap.add_argument("--tests", type=Path, default=CONF_ROOT / "tests") ap.add_argument("--repo", type=Path, default=REPO_ROOT) - ap.add_argument("--env", default=os.environ.get("CONF_ENV", "dev")) + ap.add_argument("--env", default=os.environ.get("CONF_ENV") or context_env(), + help="env the run targeted; derived from the kube context when " + "unset (never defaulted to a cluster name)") ap.add_argument("--chainsaw-version", default=None, help="override the versions.env pin (recorded verbatim)") ap.add_argument("--kube-version", default=os.environ.get("CONF_KUBE_VERSION"), diff --git a/conformance/run.sh b/conformance/run.sh index b0b0559..eeb440e 100755 --- a/conformance/run.sh +++ b/conformance/run.sh @@ -201,7 +201,7 @@ if [ "${RECORD}" = 1 ] && [ -z "${CONF_RECORD_CHILD:-}" ]; then # CONF_LEDGER exists so the harness's own tests can exercise this path without writing # to the real ledger; operators should never set it. "${PYTHON}" "${CONF_ROOT}/lib/record_run.py" \ - --run-dir "${RUN_DIR}" --env "${CONF_ENV:-dev}" \ + --run-dir "${RUN_DIR}" ${CONF_ENV:+--env "$CONF_ENV"} \ --ledger "${CONF_LEDGER:-${CONF_ROOT}/runs/ledger.jsonl}" || true log "RECORD: run rc=${RUN_RC}. Commit conformance/runs/${RUN_ID}/ WITH its run.log and" log "RECORD: conformance/runs/ledger.jsonl — an uncommitted ledger proves nothing." @@ -239,7 +239,7 @@ gen_status() { --registry "${CONF_ROOT}/requirements/registry.yaml" \ --tests "${CONF_ROOT}/tests" \ --out "${STATUS_OUT}" \ - --env "${CONF_ENV:-dev}" \ + ${CONF_ENV:+--env "$CONF_ENV"} \ "$@" }