Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9a4b669d4 | ||
|
|
aa0a50a143 | ||
|
|
22174186ff | ||
|
|
23aa2854f1 | ||
|
|
63f98aab26 | ||
|
|
62968c9ee9 |
No files matched your search
@@ -0,0 +1,60 @@
|
||||
---
|
||||
# Flux Kustomization: CILIUM (the CNI) — reconciles ./platform/cilium.
|
||||
# Serves R-PLAT-01 (platform components reconciled by GitOps) and R-GITOPS-01/03.
|
||||
#
|
||||
# WHY CILIUM HAS ITS OWN KUSTOMIZATION (split out of `platform`, 2026-09-01)
|
||||
#
|
||||
# This is the MIRROR IMAGE of the oauth2-proxy / headlamp splits, not a repeat of them. Those are
|
||||
# leaves that must NEVER gate a tier, so they left `platform` and run `wait: false` below it.
|
||||
# Cilium is the one component that legitimately SHOULD gate everything — but inside the shared
|
||||
# `platform` Kustomization its health was indistinguishable from eleven unrelated components:
|
||||
#
|
||||
# * `platform`'s Ready condition could not tell you "the CNI is down" from "longhorn is slow".
|
||||
# * The only emergency valve was `flux suspend kustomization platform` — freezing the ENTIRE
|
||||
# tier (storage, TLS, DNS, ingress, monitoring, Keycloak, KubeVirt) to stop touching the CNI.
|
||||
# `flux suspend kustomization cilium` now freezes exactly the CNI and nothing else, which is
|
||||
# the valve the 2026-09-01 substitution incident wanted and did not have.
|
||||
# * `platform`'s 10m interval + 5m timeout are wrong for a CNI: a 3-node DaemonSet rollout with
|
||||
# `wait: true` can exceed 5m, and re-rendering the CNI every 10m buys nothing.
|
||||
#
|
||||
# GATING IS PRESERVED, NOT WEAKENED: `wait: true` here plus `platform.dependsOn: [cilium]` means
|
||||
# nothing in the graph reconciles until the CNI is healthy — the same guarantee as before, now
|
||||
# expressed as an explicit edge instead of an accident of co-tenancy.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cilium
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# A CNI does not need a 10m re-render cadence. A git push still reconciles immediately —
|
||||
# kustomize-controller is source-triggered; `interval` is only the drift-correction floor.
|
||||
interval: 1h
|
||||
retryInterval: 2m
|
||||
# 10m (not the 5m `platform` uses): a 3-node Cilium DaemonSet rollout under `wait: true`,
|
||||
# rolled ONE NODE AT A TIME (maxUnavailable: 1, P2), legitimately exceeds 5m.
|
||||
timeout: 10m
|
||||
prune: true
|
||||
# TRUE, deliberately: this is the ONE component whose health should gate the whole platform.
|
||||
wait: true
|
||||
# Same revision-gating reason `platform` has it (2026-09-01 incident): `cluster-vars` is APPLIED
|
||||
# by the root `flux-system` Kustomization, and this Kustomization SUBSTITUTES from it. `dependsOn`
|
||||
# is revision-gated (kustomize-controller v1.4.0 `checkDependencies`), so this cannot render
|
||||
# revision N of helmrelease.yaml against the revision N-1 `cluster-vars`. That race is exactly how
|
||||
# `k8sServiceHost: ""` once reached the Cilium HelmRelease and took down 2/3 agents.
|
||||
dependsOn:
|
||||
- name: flux-system # the ROOT Kustomization (sync.yaml) that applies cluster-vars
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/cilium
|
||||
# MANDATORY, not decorative: platform/cilium/helmrelease.yaml references ${K3S_API_HOST}.
|
||||
# kustomize-controller now runs --feature-gates=StrictPostBuildSubstitutions=true, so without
|
||||
# this stanza an undefined ${K3S_API_HOST} FAILS this Kustomization outright (it no longer
|
||||
# renders empty). That is the intended fail-closed behaviour — but it means the stanza is
|
||||
# load-bearing for the CNI coming up at all. Do not remove it.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# No `decryption` stanza: platform/cilium carries no SOPS-encrypted resource (unlike
|
||||
# oauth2-proxy/headlamp/crafty/velero, whose overlays patch one in by name).
|
||||
@@ -5,6 +5,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- flux-system.yaml # GitRepository (reconciliation source)
|
||||
# Cilium (the CNI) — its OWN Kustomization, split OUT of `platform` (2026-09-01). The mirror image
|
||||
# of the oauth2-proxy/headlamp splits: those are leaves that must never gate a tier (wait:false);
|
||||
# Cilium is the one component that legitimately SHOULD gate everything, so it keeps wait:true and
|
||||
# `platform` dependsOn it. Splitting it out buys a per-CNI Ready condition, CNI-appropriate
|
||||
# interval/timeout, and `flux suspend kustomization cilium` as an emergency valve that does not
|
||||
# freeze the other eleven platform components. See clusters/base/cilium.yaml.
|
||||
- cilium.yaml # Flux Kustomization -> ./platform/cilium (dependsOn flux-system, wait:true)
|
||||
- platform.yaml # Flux Kustomization -> ./platform (CONTROLLERS: HelmReleases / CRDs)
|
||||
- platform-config.yaml # Flux Kustomization -> ./platform-config (CONFIGS: CRs; dependsOn platform)
|
||||
# App tier, split into dependency-ordered WAVES (was the single ./apps `apps` Kustomization).
|
||||
|
||||
@@ -11,7 +11,18 @@ spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true # drift correction: delete resources removed from git (R-GITOPS-03)
|
||||
# Cilium was moved OUT of this tier into its own Kustomization (clusters/base/cilium.yaml)
|
||||
# on 2026-09-01. `prune: true` is restored here now that the ownership transfer is VERIFIED
|
||||
# live: Namespace/kube-system, the Cilium HelmRelease and the HelmRepository all carry
|
||||
# `kustomize.toolkit.fluxcd.io/name=cilium`, this Kustomization's inventory no longer lists
|
||||
# any of the three, and the `cilium` Kustomization's inventory holds exactly those three.
|
||||
#
|
||||
# LESSON WORTH KEEPING: `platform/cilium/namespace.yaml` was the ONLY declaration of
|
||||
# Namespace/kube-system under ./platform. Moving that path out of a prune:true Kustomization
|
||||
# made the stale set `delete namespace kube-system` — coredns, metrics-server, local-path AND
|
||||
# Cilium. Before moving ANY platform/* component in future, check whether it is the sole
|
||||
# declarer of a shared Namespace; that turns a component-scoped prune into a cluster-scoped one.
|
||||
prune: true
|
||||
wait: true # block `apps` until platform is healthy
|
||||
# ORDERING FIX (2026-09-01 incident: a substitution race that failed OPEN broke Cilium).
|
||||
#
|
||||
@@ -40,6 +51,13 @@ spec:
|
||||
# health-gates its own children (see clusters/dev/sync.yaml).
|
||||
dependsOn:
|
||||
- name: flux-system # the ROOT Kustomization (sync.yaml) that applies cluster-vars
|
||||
# THE CNI GATES THE TIER. Cilium was split out of ./platform into its own Kustomization
|
||||
# (clusters/base/cilium.yaml, wait:true) so its health has its own Ready condition and its own
|
||||
# suspend valve. This edge preserves the ordering that co-tenancy used to provide implicitly:
|
||||
# nothing in `platform` — and therefore nothing anywhere downstream — reconciles until the CNI
|
||||
# is healthy. It is also revision-gated, which is what makes the prune-safety argument above
|
||||
# deterministic rather than a race.
|
||||
- name: cilium
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
|
||||
@@ -8,3 +8,22 @@ _artifacts/
|
||||
|
||||
# Python bytecode from lib/ (status.py imports coverage.py); never committed.
|
||||
__pycache__/
|
||||
|
||||
# --- THE ONE EXCEPTION: the committed run ledger (runs/) -------------------------------
|
||||
# `runs/<UTC>-<short-sha>/{report.junit.xml,run.log}` + `runs/ledger.jsonl` are the
|
||||
# machine-readable proof-of-pass record that lib/status.py's docstring asks for. They are
|
||||
# SOURCE, not regenerated artifacts: a run's evidence cannot be reconstructed after the
|
||||
# fact, so it must survive in git.
|
||||
#
|
||||
# The negations are needed because the ROOT .gitignore ignores `*.junit.xml` globally
|
||||
# (and `coverage/`). A deeper .gitignore overrides a shallower one, and nothing ignores
|
||||
# the `conformance/runs/` directory itself, so re-including the files here works (git
|
||||
# cannot re-include a file whose PARENT DIRECTORY is excluded — that is why this must not
|
||||
# live under _artifacts/).
|
||||
#
|
||||
# The run LOG is the load-bearing half: Chainsaw's JUnit carries no stdout, and the L2/L3
|
||||
# tests self-skip GREEN, so the sentinel token in run.log is the only thing separating
|
||||
# "proven" from "skipped". Never commit a report.junit.xml without its run.log.
|
||||
!runs/
|
||||
!runs/**
|
||||
!runs/**/*.junit.xml
|
||||
@@ -22,6 +22,7 @@ behavioral / L3 fault-injection** (this harness).
|
||||
./run.sh --lint # structural-validate test defs (no cluster)
|
||||
./run.sh --strict-coverage # hygiene gate: fail on tag/annotation drift
|
||||
./run.sh --require-all # phase-exit gate: every requirement must be covered
|
||||
./run.sh --record # run AND capture the evidence (see "Proof of pass")
|
||||
```
|
||||
|
||||
**Gating is scoped, not "all 51 always"** (R-TEST-03). Three distinct checks:
|
||||
@@ -105,13 +106,89 @@ lib/
|
||||
common.sh tagging convention + test discovery/selection helpers
|
||||
coverage.py requirement-coverage generator (stdlib only)
|
||||
status.py requirement-STATUS generator (stdlib only; --selftest)
|
||||
record_run.py run-ledger writer: PROOF-of-pass (stdlib only; --selftest)
|
||||
emit_skipped_junit.py Phase-0 skipped-JUnit shim (no cluster)
|
||||
runs/
|
||||
ledger.jsonl TRACKED append-only proof-of-pass ledger (one row per
|
||||
testcase per run) — the only source status.py trusts
|
||||
<UTC>-<short-sha>/ TRACKED per-run evidence: run.log + report.junit.xml
|
||||
tests/
|
||||
l1-rendering/ l2-behavioral/ l3-fault-injection/ meta/ (see tests/README.md)
|
||||
incluster/
|
||||
chainsaw-job.yaml rbac.yaml in-cluster L2/L3 Job + scoped test SA (ADR-0011)
|
||||
```
|
||||
|
||||
## Proof of pass — the run ledger (`runs/`)
|
||||
|
||||
`--status` used to report **PROVEN: 0** for all 51 requirements, and said so honestly:
|
||||
proof-of-pass was not recoverable from repo state. `_artifacts/` is gitignored and the
|
||||
root `.gitignore` kills `*.junit.xml` globally, so no run result ever reached git; the
|
||||
only record of a green run was DECISIONS-LOG.md prose. `runs/` is the fix `lib/status.py`
|
||||
asked for — **a committed, machine-readable run ledger**.
|
||||
|
||||
```bash
|
||||
./run.sh --record # runs normally, and captures the evidence
|
||||
# -> conformance/runs/<UTC>-<short-sha>/run.log (tee'd, carries the sentinels)
|
||||
# -> conformance/runs/<UTC>-<short-sha>/report.junit.xml (that run's JUnit)
|
||||
# -> conformance/runs/ledger.jsonl (one appended row per testcase)
|
||||
python3 lib/record_run.py --run-dir runs/<id> [--dry-run] # or record an existing run dir
|
||||
python3 lib/record_run.py --selftest # deterministic, no cluster
|
||||
```
|
||||
|
||||
**Recording is explicit, never implicit.** Plain `./run.sh` writes no ledger row: a run
|
||||
that silently records itself is how a bad run becomes a permanent false proof. **Commit
|
||||
the run dir and the ledger** — an uncommitted ledger proves nothing.
|
||||
|
||||
### Why a committed JUnit alone would be WORSE than nothing
|
||||
|
||||
The L2/L3 tests are runnable and **self-skipping**: when a precondition is missing (no
|
||||
`longhorn` StorageClass, no Velero CRDs, fewer than 3 Ready nodes) they echo a skip token
|
||||
and `exit 0`, and their `check:` accepts either outcome —
|
||||
|
||||
```yaml
|
||||
(contains($stdout, 'DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK')
|
||||
|| contains($stdout, 'DR_RESTORE_SKIPPED_OK')): true
|
||||
```
|
||||
|
||||
So **a self-skip exits 0 and Chainsaw records `passed` exactly as a real run does**, and
|
||||
Chainsaw's JUnit carries no stdout — its `<testcase>` elements hold only `<properties>`.
|
||||
A committed JUnit on its own would therefore record `DR_RESTORE_SKIPPED_OK` as *proof of
|
||||
R-STATE-03*: fake green, and worse than no ledger because it would look authoritative.
|
||||
|
||||
**The sentinel token in `run.log` is the only thing that separates proven from skipped.**
|
||||
`record_run.py` derives each test's tokens from its own `chainsaw-test.yaml` (every
|
||||
`contains($stdout, 'TOKEN')`; `*_SKIP*_OK` is a skip token, the last non-skip token is the
|
||||
verdict token), so the table below is never hand-maintained:
|
||||
|
||||
| Test | pass token (proof) | skip token (never proof) |
|
||||
|---|---|---|
|
||||
| `l3/node-loss` | `NODE_LOSS_VOLUME_AVAILABLE_OK` | `NODE_LOSS_SKIPPED_OK` |
|
||||
| `l3/dr-restore` | `DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK` (+ `OFF_CLUSTER_TARGET_OK`, `BACKUPS_ARE_SCHEDULED_OK`) | `DR_RESTORE_SKIPPED_OK`, `DR_SCHEDULE_SKIPPED_OK` |
|
||||
| `l3/data-survival` | `DATA_SURVIVED_OK` | `DATA_SURVIVAL_SKIPPED_OK` |
|
||||
| `l3/vm-data-survival` | `VM_DATA_SURVIVED_OK` | `VM_DATA_SURVIVAL_SKIPPED_OK` |
|
||||
| `l3/hands-off-update` | `HANDS_OFF_UPDATE_OK` | `HANDS_OFF_UPDATE_SKIPPED_OK` |
|
||||
| `l2/vm-lifecycle` | `VM_LIFECYCLE_OK` | `VM_LIFECYCLE_SKIPPED_OK` |
|
||||
| `l2/vm-default-deny` | `VM_DEFAULT_DENY_OK` | `VM_DEFAULT_DENY_SKIPPED_OK` |
|
||||
|
||||
A row reaches `proof: "proven"` on **one** path: JUnit `passed`, **every** declared pass
|
||||
token present in that test's log lines, and **no** skip token anywhere in them. A skip
|
||||
token beats a pass token (worst wins). Anything else is `skipped`, `failed`,
|
||||
`junit-skipped`, `unknown-no-sentinel-declared` or `unknown-sentinel-absent` — all
|
||||
non-green. L0/L1 need no sentinel because `lib/run_l1_offline.py` already renders their
|
||||
skips as JUnit `<skipped/>`, never as a green pass.
|
||||
|
||||
### Staleness — a proof is a claim about a tree, not a badge
|
||||
|
||||
`--status` honours a `proven` row only while the run's `git_sha` is a **descendant of the
|
||||
last commit touching that requirement's implementing manifests or its verifying test**.
|
||||
Otherwise the row degrades to **`stale-proof`**, which is not green. A run captured from a
|
||||
dirty working tree is recorded as `<sha>-dirty`, which no ancestry check can satisfy — so
|
||||
it never counts. Re-run with `--record` to refresh.
|
||||
|
||||
A requirement reads `proven` only when **every** verifying test does; one skipped, failed
|
||||
or unrecorded test drags the whole requirement down. With an empty ledger `--status`
|
||||
behaves exactly as before: every requirement `not-derivable`, headline **PROVEN 0**.
|
||||
|
||||
## How CI invokes it
|
||||
CI (Gitea Actions, wired in T-0.2 / migrated in T-6.1b) calls **this same `run.sh`**:
|
||||
|
||||
|
||||
Executable
+711
@@ -0,0 +1,711 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Append a conformance RUN LEDGER row per testcase — the committed proof-of-pass record.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
`lib/status.py` reports PROOF as `not-derivable` for every requirement, and names its own
|
||||
fix in its docstring: *"a committed, machine-readable run ledger"*. There was none, because
|
||||
`conformance/_artifacts/` is git-ignored and the root `.gitignore` kills `*.junit.xml` and
|
||||
`coverage/` globally, so nothing about a green run ever reached repo state. This tool is
|
||||
that ledger's writer: it reads one run's JUnit + run log out of `conformance/runs/<run-id>/`
|
||||
and appends one JSON object per testcase to `conformance/runs/ledger.jsonl`, which IS
|
||||
tracked (see conformance/.gitignore for the negations that let it survive).
|
||||
|
||||
THE TRAP THIS TOOL EXISTS TO AVOID — READ BEFORE CHANGING ANYTHING
|
||||
------------------------------------------------------------------
|
||||
The L2/L3 tests are RUNNABLE / SELF-SKIPPING: when a precondition is absent (no `longhorn`
|
||||
StorageClass, no Velero CRDs, <3 Ready nodes) they echo a skip token and `exit 0`, and their
|
||||
`check:` block is deliberately written to accept either outcome, e.g. tests/l3-fault-
|
||||
injection/dr-restore:
|
||||
|
||||
(contains($stdout, 'DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK')
|
||||
|| contains($stdout, 'DR_RESTORE_SKIPPED_OK')): true
|
||||
|
||||
So a SELF-SKIP exits 0 and Chainsaw records `passed` EXACTLY as a real run does. And
|
||||
Chainsaw's JUnit carries no stdout — its `<testcase>` elements hold only `<properties>`
|
||||
(verified against the committed reports). Therefore:
|
||||
|
||||
*** A COMMITTED JUnit ALONE WOULD RECORD `DR_RESTORE_SKIPPED_OK` AS PROOF OF R-STATE-03. ***
|
||||
|
||||
That is the fake-green this project refuses, and it would be worse than no ledger at all
|
||||
because it would look authoritative. THE SENTINEL TOKEN IS THE ONLY THING THAT SEPARATES
|
||||
PROVEN FROM SKIPPED, AND IT EXISTS ONLY IN THE RUN LOG. Hence: this tool requires BOTH
|
||||
files, and for the Chainsaw layers it will NEVER emit `proof: "proven"` on a JUnit `passed`
|
||||
unless the log carries that test's own pass token and carries NO skip token.
|
||||
|
||||
HOW THE TOKENS ARE OBTAINED (derived, never hand-listed)
|
||||
--------------------------------------------------------
|
||||
Hard-coding a token table would rot the moment a test is edited. Instead every token is
|
||||
parsed out of the test's own `chainsaw-test.yaml`: each `contains($stdout, 'TOKEN')` in a
|
||||
`check:` block is a declared sentinel. Classification is by the project's naming convention
|
||||
(`*_SKIP*_OK` is a skip token, anything else is a pass token), and the TERMINAL pass token
|
||||
— the last non-skip token in file order — is the test's verdict token. Edit a test's tokens
|
||||
and this tool follows automatically; delete them and the test simply becomes unprovable
|
||||
rather than silently proven.
|
||||
|
||||
node-loss pass NODE_LOSS_VOLUME_AVAILABLE_OK skip NODE_LOSS_SKIPPED_OK
|
||||
dr-restore pass DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK skip DR_RESTORE_SKIPPED_OK
|
||||
(+ OFF_CLUSTER_TARGET_OK, BACKUPS_ARE_SCHEDULED_OK / DR_SCHEDULE_SKIPPED_OK)
|
||||
|
||||
WHY L0/L1 NEED NO SENTINEL (and why that is not a loophole)
|
||||
-----------------------------------------------------------
|
||||
The offline layers are not run by Chainsaw. `lib/run_l1_offline.py` shells out to each
|
||||
dir's `check.sh` / `render-check.sh` and maps exit 3 to a JUnit `<skipped/>` element and
|
||||
any other non-zero to `<failure>`. A self-skip there is ALREADY a JUnit skip, never a green
|
||||
pass, so the JUnit is a truthful record on its own. Only the Chainsaw layers (L2/L3/META)
|
||||
can express "skipped" as "passed", and only they require a sentinel — see
|
||||
SENTINEL_REQUIRED_LAYERS.
|
||||
|
||||
LEDGER ROW (conformance/runs/ledger.jsonl — append-only, one object per testcase per run)
|
||||
ts_utc run-dir timestamp, else the JUnit's mtime (when)
|
||||
git_sha `git rev-parse HEAD`, suffixed `-dirty` when tracked files were
|
||||
modified — a proof captured from a dirty tree is NOT honoured by
|
||||
status.py, because what ran is not what is committed (what)
|
||||
env --env / CONF_ENV (cluster overlay the run targeted)
|
||||
chainsaw_version CHAINSAW_VERSION from versions.env (the pin IS the truth)
|
||||
kube_version `kubectl version` server gitVersion, else "unknown"
|
||||
layer L0|L1|L2|L3|META, from the test's `layer` annotation / JUnit property
|
||||
test testcase name (== the test dir basename, asserted by convention)
|
||||
requirements[] the `conformance.home.k3s/R-*` labels on the test — the SAME
|
||||
authoritative source coverage.py uses, not the JUnit classname
|
||||
junit_result passed | failed | error | skipped
|
||||
sentinel the DECIDING token grepped from run.log, or null
|
||||
proof proven | skipped | failed | junit-skipped
|
||||
| unknown-no-sentinel-declared | unknown-sentinel-absent
|
||||
run_id the runs/<dir> name (provenance: which log backs this row)
|
||||
note one line saying WHY this row landed on that proof state
|
||||
|
||||
`proven` is emitted on exactly one path (see decide()); every other path is non-green.
|
||||
|
||||
RECORDING IS EXPLICIT, NEVER IMPLICIT
|
||||
-------------------------------------
|
||||
Invoke this tool by hand, or via `run.sh --record`. A run that silently records itself is
|
||||
how a bad run becomes a permanent false proof, so `run.sh` without `--record` writes no
|
||||
ledger row at all.
|
||||
|
||||
EXIT CODES (ci/lib.sh convention: 0 pass / 1 fail / 3 skipped-not-green)
|
||||
0 rows appended.
|
||||
1 a real failure: unparsable JUnit, unreadable test tree, unwritable ledger.
|
||||
3 the inputs a ledger row needs are absent (no report.junit.xml, or no run.log).
|
||||
NOTHING is recorded and this is NEVER green — a run we cannot evidence is not a run.
|
||||
|
||||
Pure standard library, matching the harness's no-pip-dependency posture (coverage.py,
|
||||
status.py). PyYAML lives in catalog/render.py; the harness does not depend on it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
_LIB = Path(__file__).resolve().parent
|
||||
if str(_LIB) not in sys.path:
|
||||
sys.path.insert(0, str(_LIB))
|
||||
# Importing a sibling would otherwise litter conformance/lib/__pycache__/ into the working
|
||||
# tree, and the lint stage scans untracked files. Same guard as status.py.
|
||||
sys.dont_write_bytecode = True
|
||||
import coverage as cov # noqa: E402 — reuse the registry/test parsers, don't fork them
|
||||
|
||||
CONF_ROOT = _LIB.parent
|
||||
REPO_ROOT = CONF_ROOT.parent
|
||||
DEFAULT_LEDGER = CONF_ROOT / "runs" / "ledger.jsonl"
|
||||
|
||||
# The ONLY layers exempt from needing a pass sentinel: the offline ones, whose skips
|
||||
# lib/run_l1_offline.py already renders as JUnit <skipped/> rather than as a green pass
|
||||
# (see docstring). Written as an EXEMPT set, not a REQUIRED set, so that an unrecognised
|
||||
# or new layer fails CLOSED — it needs a sentinel — instead of quietly becoming provable
|
||||
# by JUnit alone.
|
||||
SENTINEL_EXEMPT_LAYERS = frozenset({"L0", "L1"})
|
||||
|
||||
# Layer from the test's subtree. Authoritative over the annotation: coverage.py's layer
|
||||
# regex stops at the first non-[L/A/M/0-9] character, so `layer: "META"` parses as "M" —
|
||||
# harmless for its own report, but here a mis-read layer decides whether a sentinel is
|
||||
# required at all, so it is derived from the path exactly as emit_skipped_junit.py does.
|
||||
_LAYER_BY_SUBTREE = (("/l0-static/", "L0"), ("/l1-rendering/", "L1"),
|
||||
("/l2-behavioral/", "L2"), ("/l3-fault-injection/", "L3"),
|
||||
("/meta/", "META"))
|
||||
|
||||
|
||||
def layer_of(path: Path) -> str:
|
||||
rel = str(path)
|
||||
for frag, layer in _LAYER_BY_SUBTREE:
|
||||
if frag in rel:
|
||||
return layer
|
||||
return ""
|
||||
|
||||
# A declared sentinel: `contains($stdout, 'TOKEN')` inside a Chainsaw `check:` block.
|
||||
# Both quote styles are accepted; the repo uses single quotes throughout.
|
||||
_SENTINEL_RE = re.compile(r"contains\(\s*\$stdout\s*,\s*['\"]([A-Za-z0-9_]+)['\"]\s*\)")
|
||||
|
||||
# Naming convention for a SKIP token (tests/l3-fault-injection/*: NODE_LOSS_SKIPPED_OK,
|
||||
# DR_RESTORE_SKIPPED_OK, DR_SCHEDULE_SKIPPED_OK, VM_DATA_SURVIVAL_SKIPPED_OK, ...).
|
||||
# Deliberately broad: mis-classifying a PASS token as a skip only ever loses a proof;
|
||||
# mis-classifying a SKIP token as a pass would manufacture one.
|
||||
_SKIP_TOKEN_RE = re.compile(r"_SKIP")
|
||||
|
||||
# A Chainsaw log line header: `| 20:41:32 | node-loss | step-name | TRY | RUN | ...`
|
||||
# Field 2 is the test name; it is how stdout is attributed back to a testcase.
|
||||
_LOG_HEADER_RE = re.compile(
|
||||
r"^\s*\|\s*\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\s*\|\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*\|")
|
||||
|
||||
_RUN_ID_TS_RE = re.compile(r"^(\d{8}T\d{6}Z)-")
|
||||
|
||||
|
||||
# ========================================================================= sentinels
|
||||
def parse_sentinels(test_yaml: Path) -> dict:
|
||||
"""Declared sentinel tokens of one Chainsaw test, derived from the test file itself.
|
||||
|
||||
Returns {"pass": [...], "skip": [...], "terminal_pass": str|None}. `terminal_pass` is
|
||||
the LAST non-skip token in file order — the test's verdict token, i.e. the one its
|
||||
final `check:` block asserts (dr-restore -> DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK).
|
||||
A test with no `contains($stdout, ...)` assertion declares no sentinel and is reported
|
||||
unprovable rather than proven; see decide().
|
||||
"""
|
||||
try:
|
||||
text = test_yaml.read_text()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return {"pass": [], "skip": [], "terminal_pass": None}
|
||||
toks = _SENTINEL_RE.findall(text)
|
||||
passes, skips = [], []
|
||||
for t in toks:
|
||||
(skips if _SKIP_TOKEN_RE.search(t) else passes).append(t)
|
||||
# De-duplicate, preserving order (dr-restore names DR_RESTORE_SKIPPED_OK twice).
|
||||
def uniq(xs: list[str]) -> list[str]:
|
||||
return list(dict.fromkeys(xs))
|
||||
passes, skips = uniq(passes), uniq(skips)
|
||||
return {"pass": passes, "skip": skips,
|
||||
"terminal_pass": passes[-1] if passes else None}
|
||||
|
||||
|
||||
def sentinel_map(tests_dir: Path) -> dict[str, dict]:
|
||||
"""{test-name: sentinels} for every test in the tree.
|
||||
|
||||
Keyed on the Chainsaw `metadata.name`, which is what lands in the JUnit `<testcase
|
||||
name=...>`. This repo keeps that equal to the test's directory basename; the mapping
|
||||
is by name, so it stays correct if that ever drifts.
|
||||
"""
|
||||
out: dict[str, dict] = {}
|
||||
for f in sorted(tests_dir.rglob("chainsaw-test.yaml")):
|
||||
meta = cov.parse_test(f)
|
||||
name = meta["name"] or f.parent.name
|
||||
s = parse_sentinels(f)
|
||||
s["dir"] = str(f.parent)
|
||||
s["layer"] = layer_of(f) or meta["layer"]
|
||||
s["requirements"] = meta["labels"]
|
||||
out[name] = s
|
||||
return out
|
||||
|
||||
|
||||
# =============================================================================== log
|
||||
def attribute_log(log_text: str, known: set[str]) -> tuple[dict[str, list[str]], bool]:
|
||||
"""Split a run log into per-testcase line buckets.
|
||||
|
||||
Chainsaw prefixes its log lines with `| HH:MM:SS | <test> | <step> | ...` and prints a
|
||||
script's stdout underneath that header, so the test name is carried by a state machine:
|
||||
a header naming a KNOWN test switches the current bucket, every following line lands in
|
||||
it. Requiring the name to be known is what stops run.sh's own `[run.sh]` chatter (and
|
||||
any other pipe-shaped line) from opening a bogus bucket.
|
||||
|
||||
Returns (buckets, attributed) where `attributed` is False when the log contained no
|
||||
recognisable header at all — the signal for the degraded global scan in decide().
|
||||
"""
|
||||
buckets: dict[str, list[str]] = {}
|
||||
cur: str | None = None
|
||||
attributed = False
|
||||
for line in log_text.splitlines():
|
||||
m = _LOG_HEADER_RE.match(line)
|
||||
if m and m.group(1) in known:
|
||||
cur = m.group(1)
|
||||
attributed = True
|
||||
if cur is not None:
|
||||
buckets.setdefault(cur, []).append(line)
|
||||
return buckets, attributed
|
||||
|
||||
|
||||
def unique_tokens(smap: dict[str, dict]) -> set[str]:
|
||||
"""Tokens claimed by EXACTLY ONE test in the tree.
|
||||
|
||||
Only these may be used by the degraded global scan: `DNS_OK` is echoed by two different
|
||||
tests, so in an unattributable log it can never be evidence for either; but
|
||||
`NODE_LOSS_VOLUME_AVAILABLE_OK` names precisely one test and still can.
|
||||
"""
|
||||
owners: dict[str, set[str]] = {}
|
||||
for name, s in smap.items():
|
||||
for t in s["pass"] + s["skip"]:
|
||||
owners.setdefault(t, set()).add(name)
|
||||
return {t for t, o in owners.items() if len(o) == 1}
|
||||
|
||||
|
||||
# ============================================================================ decide
|
||||
def decide(layer: str, junit_result: str, sent: dict, lines: list[str],
|
||||
scope: str) -> tuple[str | None, str, str]:
|
||||
"""The ONE place a testcase becomes `proven` — or, far more often, does not.
|
||||
|
||||
Returns (sentinel, proof, note).
|
||||
|
||||
Ordering is worst-first ON PURPOSE, so that ambiguity can only ever cost a proof:
|
||||
1. JUnit failure/error -> failed (a red run is recorded as red)
|
||||
2. JUnit <skipped/> -> junit-skipped (never green; ci/lib.sh rule)
|
||||
3. ANY declared skip token seen -> skipped (BEATS a green JUnit and beats
|
||||
the pass token: if any gated
|
||||
step self-skipped, the run did
|
||||
not prove the requirement)
|
||||
4. no sentinel declared -> proven only for L0/L1, whose skips are already
|
||||
JUnit skips; unknown-no-sentinel-declared for
|
||||
the Chainsaw layers
|
||||
5. EVERY declared pass token seen -> proven
|
||||
6. anything else -> unknown-sentinel-absent
|
||||
Step 5 requires ALL pass tokens, not just the terminal one: every Chainsaw step here
|
||||
asserts `pass || skip`, so a missing pass token means a step did not reach its proven
|
||||
branch, whatever the JUnit says.
|
||||
"""
|
||||
if junit_result in ("failed", "error"):
|
||||
return (None, "failed", f"JUnit recorded {junit_result}")
|
||||
if junit_result == "skipped":
|
||||
return (None, "junit-skipped",
|
||||
"JUnit <skipped/> — the test did not execute; a skip is never green")
|
||||
|
||||
text = "\n".join(lines)
|
||||
seen_skip = [t for t in sent["skip"] if t in text]
|
||||
if seen_skip:
|
||||
return (seen_skip[0], "skipped",
|
||||
f"run log carries the SELF-SKIP token {seen_skip[0]!r} — the test exited 0 "
|
||||
f"without exercising the behaviour, so this is NOT proof")
|
||||
|
||||
if not sent["pass"] and not sent["skip"]:
|
||||
if layer not in SENTINEL_EXEMPT_LAYERS:
|
||||
return (None, "unknown-no-sentinel-declared",
|
||||
f"{layer or 'unknown-layer'} test declares no contains($stdout, ...) "
|
||||
f"sentinel, so a green JUnit cannot be told apart from a self-skip — "
|
||||
f"refusing to call it proven")
|
||||
return (None, "proven",
|
||||
f"{layer} offline check exited 0; run_l1_offline.py renders its skips as "
|
||||
f"JUnit <skipped/>, so a green JUnit is truthful without a sentinel")
|
||||
|
||||
missing = [t for t in sent["pass"] if t not in text]
|
||||
if not missing:
|
||||
return (sent["terminal_pass"], "proven",
|
||||
f"run log carries every declared pass token "
|
||||
f"({', '.join(sent['pass'])}) and no skip token [{scope}]")
|
||||
return (None, "unknown-sentinel-absent",
|
||||
f"run log lacks pass token(s) {', '.join(missing)} for this test [{scope}] — "
|
||||
f"refusing to infer proof from the JUnit alone")
|
||||
|
||||
|
||||
# ============================================================================= junit
|
||||
def junit_cases(path: Path) -> list[dict]:
|
||||
"""[{name, layer, requirements, result}] for every <testcase> in a JUnit report."""
|
||||
root = ET.parse(path).getroot()
|
||||
cases: list[dict] = []
|
||||
for c in root.iter("testcase"):
|
||||
if c.find("failure") is not None:
|
||||
result = "failed"
|
||||
elif c.find("error") is not None:
|
||||
result = "error"
|
||||
elif c.find("skipped") is not None:
|
||||
result = "skipped"
|
||||
else:
|
||||
result = "passed"
|
||||
props = {p.get("name"): (p.get("value") or "")
|
||||
for p in c.iter("property") if p.get("name")}
|
||||
reqs = sorted(set(cov.RID_RE.findall(props.get("requirements", ""))))
|
||||
layer = (props.get("layer") or "").upper()
|
||||
if not layer:
|
||||
layer = (c.get("classname") or "").split(":", 1)[0].upper()
|
||||
cases.append({"name": c.get("name") or "?", "layer": layer,
|
||||
"requirements": reqs, "result": result})
|
||||
return cases
|
||||
|
||||
|
||||
# =============================================================================== git
|
||||
def git_sha(repo: Path) -> str:
|
||||
"""HEAD, suffixed `-dirty` when TRACKED files differ from it.
|
||||
|
||||
Untracked files are deliberately ignored, matching status.py's `tracked_files()`
|
||||
posture (untracked scratch is not repo state) — and because a `--record` run creates
|
||||
untracked `runs/<id>/` files as it goes, which would otherwise mark every run dirty.
|
||||
A `-dirty` sha is not a valid rev, so status.py's ancestry check cannot resolve it and
|
||||
the proof degrades to `stale-proof`: correct, since what ran is not what is committed.
|
||||
"""
|
||||
try:
|
||||
sha = subprocess.run(["git", "-C", str(repo), "rev-parse", "HEAD"],
|
||||
capture_output=True, text=True, check=True).stdout.strip()
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
return "unknown"
|
||||
dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--"],
|
||||
capture_output=True).returncode != 0
|
||||
return f"{sha}-dirty" if dirty else sha
|
||||
|
||||
|
||||
def pinned_chainsaw_version(conf_root: Path) -> str:
|
||||
"""CHAINSAW_VERSION from versions.env — the pin is the single source of truth."""
|
||||
f = conf_root / "versions.env"
|
||||
if f.is_file():
|
||||
m = re.search(r'^CHAINSAW_VERSION="?([^"\n]+)"?', f.read_text(), re.M)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return "unknown"
|
||||
|
||||
|
||||
def server_kube_version() -> str:
|
||||
"""Server gitVersion via `kubectl version` (read-only), else "unknown"."""
|
||||
for args in (["kubectl", "version", "-o", "json"],
|
||||
["kubectl", "version", "--output=json"]):
|
||||
try:
|
||||
p = subprocess.run(args, capture_output=True, text=True, timeout=15)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
continue
|
||||
if p.returncode != 0 or not p.stdout.strip():
|
||||
continue
|
||||
try:
|
||||
v = json.loads(p.stdout).get("serverVersion") or {}
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if v.get("gitVersion"):
|
||||
return v["gitVersion"]
|
||||
return "unknown"
|
||||
|
||||
|
||||
def run_timestamp(run_dir: Path, junit: Path) -> str:
|
||||
"""UTC stamp: the run-dir name's own `<UTC>-<sha>` prefix, else the JUnit's mtime."""
|
||||
m = _RUN_ID_TS_RE.match(run_dir.name)
|
||||
if m:
|
||||
t = m.group(1)
|
||||
return f"{t[0:4]}-{t[4:6]}-{t[6:8]}T{t[9:11]}:{t[11:13]}:{t[13:15]}Z"
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(junit.stat().st_mtime))
|
||||
|
||||
|
||||
# ============================================================================ record
|
||||
def build_rows(run_dir: Path, tests_dir: Path, env: str, chainsaw_version: str,
|
||||
kube_version: str, sha: str) -> list[dict]:
|
||||
junit = run_dir / "report.junit.xml"
|
||||
log = run_dir / "run.log"
|
||||
smap = sentinel_map(tests_dir)
|
||||
buckets, attributed = attribute_log(log.read_text(errors="replace"), set(smap))
|
||||
uniq = unique_tokens(smap)
|
||||
all_lines = log.read_text(errors="replace").splitlines()
|
||||
ts = run_timestamp(run_dir, junit)
|
||||
|
||||
rows: list[dict] = []
|
||||
for case in junit_cases(junit):
|
||||
name = case["name"]
|
||||
sent = smap.get(name, {"pass": [], "skip": [], "terminal_pass": None,
|
||||
"layer": case["layer"], "requirements": []})
|
||||
layer = sent.get("layer") or case["layer"] or "?"
|
||||
# Requirements come from the test's LABELS (coverage.py's authoritative source);
|
||||
# the JUnit property is the fallback for a testcase with no matching test file.
|
||||
reqs = sent.get("requirements") or case["requirements"]
|
||||
|
||||
if name in buckets:
|
||||
lines, scope = buckets[name], "log scoped to this testcase"
|
||||
elif not attributed:
|
||||
# Degraded mode: the log had no recognisable Chainsaw header (a redirected or
|
||||
# reformatted log). Fall back to a whole-log scan, but ONLY for tokens owned by
|
||||
# exactly one test, so a token another test emitted can never prove this one.
|
||||
lines = all_lines
|
||||
sent = dict(sent,
|
||||
**{"pass": [t for t in sent["pass"] if t in uniq],
|
||||
"skip": [t for t in sent["skip"] if t in uniq]})
|
||||
sent["terminal_pass"] = sent["pass"][-1] if sent["pass"] else None
|
||||
scope = "DEGRADED: unattributable log, globally-unique tokens only"
|
||||
else:
|
||||
lines, scope = [], "no log lines attributed to this testcase"
|
||||
|
||||
sentinel, proof, note = decide(layer, case["result"], sent, lines, scope)
|
||||
rows.append({
|
||||
"ts_utc": ts,
|
||||
"git_sha": sha,
|
||||
"env": env,
|
||||
"chainsaw_version": chainsaw_version,
|
||||
"kube_version": kube_version,
|
||||
"layer": layer,
|
||||
"test": name,
|
||||
"requirements": reqs,
|
||||
"junit_result": case["result"],
|
||||
"sentinel": sentinel,
|
||||
"proof": proof,
|
||||
"run_id": run_dir.name,
|
||||
"note": note,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def append_rows(ledger: Path, rows: list[dict]) -> None:
|
||||
"""Append-only, one compact JSON object per line. Never rewrites existing rows."""
|
||||
ledger.parent.mkdir(parents=True, exist_ok=True)
|
||||
with ledger.open("a", encoding="utf-8") as fh:
|
||||
for r in rows:
|
||||
fh.write(json.dumps(r, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
# ========================================================================== selftest
|
||||
def _selftest() -> int:
|
||||
"""Deterministic, no-cluster self-test of every decision rule (house pattern:
|
||||
`status.py --selftest`, `catalog/render.py --selftest`). Throwaway tree only.
|
||||
|
||||
The load-bearing case is `SKIP sentinel does NOT produce proven`: it is the single
|
||||
assertion standing between this ledger and a permanent false proof of R-STATE-03.
|
||||
"""
|
||||
import tempfile
|
||||
ok = True
|
||||
|
||||
def check(label: str, cond: bool) -> None:
|
||||
nonlocal ok
|
||||
if not cond:
|
||||
sys.stderr.write(f"record_run.py selftest: FAIL — {label}\n")
|
||||
ok = False
|
||||
|
||||
# --- token derivation, against the dr-restore/node-loss shape ---------------------
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root = Path(td)
|
||||
tdir = root / "tests" / "l3-fault-injection" / "dr-restore"
|
||||
tdir.mkdir(parents=True)
|
||||
(tdir / "chainsaw-test.yaml").write_text(
|
||||
"apiVersion: chainsaw.kyverno.io/v1alpha1\nkind: Test\nmetadata:\n"
|
||||
" name: dr-restore\n labels:\n"
|
||||
" conformance.home.k3s/R-STATE-03: \"true\"\n annotations:\n"
|
||||
" conformance.home.k3s/requirements: \"R-STATE-03\"\n"
|
||||
" conformance.home.k3s/layer: \"L3\"\nspec:\n steps:\n"
|
||||
" - script:\n content: 'echo OFF_CLUSTER_TARGET_OK'\n"
|
||||
" check:\n"
|
||||
" (contains($stdout, 'OFF_CLUSTER_TARGET_OK')"
|
||||
" || contains($stdout, 'DR_RESTORE_SKIPPED_OK')): true\n"
|
||||
" - script:\n content: 'echo x'\n check:\n"
|
||||
" (contains($stdout, 'DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK')"
|
||||
" || contains($stdout, 'DR_RESTORE_SKIPPED_OK')): true\n")
|
||||
s = parse_sentinels(tdir / "chainsaw-test.yaml")
|
||||
check("pass tokens derived from the test file",
|
||||
s["pass"] == ["OFF_CLUSTER_TARGET_OK",
|
||||
"DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK"])
|
||||
check("skip token classified by the *_SKIP*_OK convention, de-duplicated",
|
||||
s["skip"] == ["DR_RESTORE_SKIPPED_OK"])
|
||||
check("terminal pass token is the LAST non-skip token",
|
||||
s["terminal_pass"] == "DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK")
|
||||
|
||||
smap = sentinel_map(root / "tests")
|
||||
check("sentinel_map keys on metadata.name and carries layer + labels",
|
||||
smap["dr-restore"]["layer"] == "L3"
|
||||
and smap["dr-restore"]["requirements"] == ["R-STATE-03"])
|
||||
|
||||
# --- decide(): the SKIP-is-never-proven contract ----------------------------------
|
||||
l3 = {"pass": ["DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK"],
|
||||
"skip": ["DR_RESTORE_SKIPPED_OK"],
|
||||
"terminal_pass": "DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK"}
|
||||
|
||||
# *** THE CASE THIS WHOLE FILE EXISTS FOR ***
|
||||
sent, proof, _ = decide("L3", "passed", l3, ["DR_RESTORE_SKIPPED_OK"], "t")
|
||||
check("SKIP sentinel on a GREEN JUnit is 'skipped', NEVER 'proven'",
|
||||
proof == "skipped" and sent == "DR_RESTORE_SKIPPED_OK")
|
||||
|
||||
_, proof, _ = decide("L3", "passed", l3,
|
||||
["DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK"], "t")
|
||||
check("pass sentinel on a green JUnit is 'proven'", proof == "proven")
|
||||
|
||||
_, proof, _ = decide("L3", "passed", l3,
|
||||
["DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK",
|
||||
"DR_RESTORE_SKIPPED_OK"], "t")
|
||||
check("a skip token BEATS a pass token in the same log (worst wins)",
|
||||
proof == "skipped")
|
||||
|
||||
_, proof, _ = decide("L3", "passed", l3, ["nothing relevant here"], "t")
|
||||
check("green JUnit with no sentinel in the log is unknown, not proven",
|
||||
proof == "unknown-sentinel-absent")
|
||||
|
||||
_, proof, _ = decide("L3", "passed", l3, [], "t")
|
||||
check("green JUnit with NO log lines attributed is unknown, not proven",
|
||||
proof == "unknown-sentinel-absent")
|
||||
|
||||
_, proof, _ = decide("L3", "failed", l3,
|
||||
["DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK"], "t")
|
||||
check("a failing JUnit outranks a pass token in the log", proof == "failed")
|
||||
|
||||
_, proof, _ = decide("L3", "skipped", l3, [], "t")
|
||||
check("JUnit <skipped/> records junit-skipped, never green",
|
||||
proof == "junit-skipped")
|
||||
|
||||
multi = {"pass": ["A_OK", "B_OK"], "skip": [], "terminal_pass": "B_OK"}
|
||||
_, proof, _ = decide("L3", "passed", multi, ["B_OK"], "t")
|
||||
check("every declared pass token must be present, not just the terminal one",
|
||||
proof == "unknown-sentinel-absent")
|
||||
_, proof, _ = decide("L3", "passed", multi, ["A_OK", "B_OK"], "t")
|
||||
check("all pass tokens present == proven", proof == "proven")
|
||||
|
||||
none = {"pass": [], "skip": [], "terminal_pass": None}
|
||||
_, proof, _ = decide("L2", "passed", none, ["whatever"], "t")
|
||||
check("a Chainsaw test declaring no sentinel is NOT proven by a green JUnit",
|
||||
proof == "unknown-no-sentinel-declared")
|
||||
_, proof, _ = decide("L1", "passed", none, [], "t")
|
||||
check("L0/L1 offline checks are proven by JUnit (their skips are JUnit skips)",
|
||||
proof == "proven")
|
||||
_, proof, _ = decide("L0", "skipped", none, [], "t")
|
||||
check("an L0 JUnit skip is still never green", proof == "junit-skipped")
|
||||
_, proof, _ = decide("META", "passed", none, [], "t")
|
||||
check("META (a Chainsaw layer) needs a sentinel like L2/L3",
|
||||
proof == "unknown-no-sentinel-declared")
|
||||
_, proof, _ = decide("", "passed", none, [], "t")
|
||||
check("an unknown/blank layer fails CLOSED — sentinel required",
|
||||
proof == "unknown-no-sentinel-declared")
|
||||
|
||||
# --- log attribution --------------------------------------------------------------
|
||||
log = ("[run.sh] Coverage report first\n"
|
||||
" | 20:41:32 | node-loss | step | SCRIPT | RUN |\n"
|
||||
" NODE_LOSS_SKIPPED_OK\n"
|
||||
" | 20:44:01 | dr-restore | step | SCRIPT | RUN |\n"
|
||||
" DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK\n")
|
||||
b, attributed = attribute_log(log, {"node-loss", "dr-restore"})
|
||||
check("log lines are attributed to the testcase whose header preceded them",
|
||||
attributed
|
||||
and any("NODE_LOSS_SKIPPED_OK" in l for l in b["node-loss"])
|
||||
and not any("NODE_LOSS_SKIPPED_OK" in l for l in b["dr-restore"]))
|
||||
check("a pass token does NOT leak across testcase boundaries",
|
||||
not any("DR_RESTORED" in l for l in b["node-loss"]))
|
||||
_, attributed2 = attribute_log("no chainsaw headers at all\n", {"node-loss"})
|
||||
check("an unattributable log is reported as such (degraded mode)", not attributed2)
|
||||
|
||||
smap2 = {"a": {"pass": ["DNS_OK", "A_ONLY_OK"], "skip": []},
|
||||
"b": {"pass": ["DNS_OK"], "skip": []}}
|
||||
check("only tokens owned by exactly one test may be used in degraded mode",
|
||||
unique_tokens(smap2) == {"A_ONLY_OK"})
|
||||
|
||||
# --- end-to-end: a self-skipped L3 run must land in the ledger as `skipped` --------
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root = Path(td)
|
||||
tdir = root / "tests" / "l3-fault-injection" / "node-loss"
|
||||
tdir.mkdir(parents=True)
|
||||
(tdir / "chainsaw-test.yaml").write_text(
|
||||
"kind: Test\nmetadata:\n name: node-loss\n labels:\n"
|
||||
" conformance.home.k3s/R-STATE-02: \"true\"\n annotations:\n"
|
||||
" conformance.home.k3s/requirements: \"R-STATE-02\"\n"
|
||||
" conformance.home.k3s/layer: \"L3\"\nspec:\n steps:\n"
|
||||
" - script:\n content: 'echo hi'\n check:\n"
|
||||
" (contains($stdout, 'NODE_LOSS_VOLUME_AVAILABLE_OK')"
|
||||
" || contains($stdout, 'NODE_LOSS_SKIPPED_OK')): true\n")
|
||||
rd = root / "runs" / "20260901T120000Z-abc1234"
|
||||
rd.mkdir(parents=True)
|
||||
(rd / "report.junit.xml").write_text(
|
||||
'<testsuites><testsuite><testcase classname="L3:R-STATE-02" '
|
||||
'name="node-loss"><properties>'
|
||||
'<property name="requirements" value="R-STATE-02"/>'
|
||||
'<property name="layer" value="L3"/></properties></testcase>'
|
||||
"</testsuite></testsuites>")
|
||||
(rd / "run.log").write_text(
|
||||
" | 12:00:01 | node-loss | s | SCRIPT | RUN |\n"
|
||||
" SKIP: 'longhorn' StorageClass absent\n"
|
||||
" NODE_LOSS_SKIPPED_OK\n")
|
||||
rows = build_rows(rd, root / "tests", "dev", "v0.2.12", "v1.36.4", "abc1234")
|
||||
check("end-to-end: Chainsaw 'passed' + skip token => proof 'skipped'",
|
||||
len(rows) == 1 and rows[0]["junit_result"] == "passed"
|
||||
and rows[0]["proof"] == "skipped"
|
||||
and rows[0]["sentinel"] == "NODE_LOSS_SKIPPED_OK")
|
||||
check("end-to-end: row carries ts from the run-dir name and the R-ID label",
|
||||
rows[0]["ts_utc"] == "2026-09-01T12:00:00Z"
|
||||
and rows[0]["requirements"] == ["R-STATE-02"])
|
||||
|
||||
(rd / "run.log").write_text(
|
||||
" | 12:00:01 | node-loss | s | SCRIPT | RUN |\n"
|
||||
" NODE_LOSS_VOLUME_AVAILABLE_OK\n")
|
||||
rows = build_rows(rd, root / "tests", "dev", "v0.2.12", "v1.36.4", "abc1234")
|
||||
check("end-to-end: Chainsaw 'passed' + pass token => proof 'proven'",
|
||||
rows[0]["proof"] == "proven"
|
||||
and rows[0]["sentinel"] == "NODE_LOSS_VOLUME_AVAILABLE_OK")
|
||||
|
||||
ledger = root / "ledger.jsonl"
|
||||
append_rows(ledger, rows)
|
||||
append_rows(ledger, rows)
|
||||
check("ledger is append-only JSONL (one parsable object per line)",
|
||||
len(ledger.read_text().strip().splitlines()) == 2
|
||||
and json.loads(ledger.read_text().splitlines()[0])["test"] == "node-loss")
|
||||
|
||||
if not ok:
|
||||
return 1
|
||||
sys.stderr.write("record_run.py selftest: PASS\n")
|
||||
return 0
|
||||
|
||||
|
||||
# ============================================================================== main
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--run-dir", type=Path,
|
||||
help="conformance/runs/<UTC>-<sha>/ holding report.junit.xml + run.log")
|
||||
ap.add_argument("--ledger", type=Path, default=DEFAULT_LEDGER,
|
||||
help=f"append-only ledger (default {DEFAULT_LEDGER})")
|
||||
ap.add_argument("--tests", type=Path, default=CONF_ROOT / "tests")
|
||||
ap.add_argument("--repo", type=Path, default=REPO_ROOT)
|
||||
ap.add_argument("--env", default=os.environ.get("CONF_ENV", "dev"))
|
||||
ap.add_argument("--chainsaw-version", default=None,
|
||||
help="override the versions.env pin (recorded verbatim)")
|
||||
ap.add_argument("--kube-version", default=os.environ.get("CONF_KUBE_VERSION"),
|
||||
help="override the `kubectl version` probe (recorded verbatim)")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="print the rows that WOULD be appended; write nothing")
|
||||
ap.add_argument("--selftest", action="store_true", help="run the built-in self-test")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.selftest:
|
||||
return _selftest()
|
||||
if args.run_dir is None:
|
||||
sys.stderr.write("[record] --run-dir is required (or use --selftest)\n")
|
||||
return 1
|
||||
|
||||
run_dir = args.run_dir.resolve()
|
||||
junit, log = run_dir / "report.junit.xml", run_dir / "run.log"
|
||||
# BOTH files or nothing. The JUnit alone cannot tell a self-skip from a real pass, so
|
||||
# recording from it alone is exactly the false proof this tool exists to prevent.
|
||||
missing = [p.name for p in (junit, log) if not p.is_file()]
|
||||
if missing:
|
||||
sys.stderr.write(
|
||||
f"[record] {run_dir} is missing {', '.join(missing)} — nothing recorded. "
|
||||
f"The JUnit alone cannot distinguish a self-skip from a real pass "
|
||||
f"(see module docstring). SKIPPED (not green).\n")
|
||||
return 3
|
||||
|
||||
try:
|
||||
rows = build_rows(
|
||||
run_dir, args.tests, args.env,
|
||||
args.chainsaw_version or pinned_chainsaw_version(CONF_ROOT),
|
||||
args.kube_version or server_kube_version(),
|
||||
git_sha(args.repo))
|
||||
except ET.ParseError as e:
|
||||
sys.stderr.write(f"[record] {junit} is not parsable JUnit: {e}\n")
|
||||
return 1
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"[record] cannot read the run inputs: {e}\n")
|
||||
return 1
|
||||
|
||||
if not rows:
|
||||
sys.stderr.write(f"[record] {junit} contains no <testcase> — nothing to record. "
|
||||
f"SKIPPED (not green).\n")
|
||||
return 3
|
||||
|
||||
for r in rows:
|
||||
print(f"[record] {r['layer']:4s} {r['proof']:28s} {r['test']} "
|
||||
f"({','.join(r['requirements']) or 'untagged'}) "
|
||||
f"junit={r['junit_result']} sentinel={r['sentinel'] or '—'}")
|
||||
counts: dict[str, int] = {}
|
||||
for r in rows:
|
||||
counts[r["proof"]] = counts.get(r["proof"], 0) + 1
|
||||
print(f"[record] {len(rows)} testcase(s): "
|
||||
+ ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
||||
|
||||
if args.dry_run:
|
||||
print("[record] --dry-run: ledger NOT written.")
|
||||
return 0
|
||||
try:
|
||||
append_rows(args.ledger, rows)
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"[record] cannot append to {args.ledger}: {e}\n")
|
||||
return 1
|
||||
print(f"[record] appended {len(rows)} row(s) -> {args.ledger}")
|
||||
print("[record] COMMIT the run dir and the ledger; an uncommitted ledger proves nothing.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+447
-39
@@ -15,26 +15,42 @@ This generator derives, from the REPOSITORY ALONE, three orthogonal axes per req
|
||||
from a Flux Kustomization entrypoint (i.e. actually reconciled)?
|
||||
2. VERIFICATION — is there a requirement-tagged test, at which layer, and is it an
|
||||
EXECUTABLE test or a TODO scaffold?
|
||||
3. PROOF — has that test actually passed? *** DELIBERATELY NOT DERIVED. ***
|
||||
3. PROOF — has that test actually passed? Derived ONLY from the committed
|
||||
run ledger, and ONLY as far as that ledger's evidence reaches.
|
||||
|
||||
WHAT THIS TOOL REFUSES TO DERIVE (and why)
|
||||
------------------------------------------
|
||||
PROOF-OF-PASS IS NOT DERIVABLE FROM REPO STATE. Nothing tracked in git records a test
|
||||
result: `conformance/_artifacts/` and `results/` are git-ignored, and the only prose
|
||||
record of a green run lives in DECISIONS-LOG.md — hand-maintained, unstructured, and
|
||||
exactly the kind of source the operator has ruled out. The project's own culture is that
|
||||
"merged" is never "done" (agents/definition-of-done.md, R-TEST-03), so this report will
|
||||
NOT infer, guess, or default a requirement to green. Every requirement's proof state is
|
||||
reported as `not-derivable`, and the headline says so. If an untracked JUnit artifact
|
||||
happens to be lying in the workspace it is surfaced ONLY as a quarantined advisory with
|
||||
its mtime — never counted, never mixed into a status.
|
||||
PROOF — WHAT CHANGED, AND WHAT DID NOT
|
||||
--------------------------------------
|
||||
This report used to say `not-derivable` for every requirement, because nothing tracked in
|
||||
git recorded a test result and the only prose record of a green run lived in
|
||||
DECISIONS-LOG.md — hand-maintained, unstructured, exactly the source the operator ruled
|
||||
out. The fix this docstring asked for now exists: `conformance/runs/ledger.jsonl`, an
|
||||
append-only ledger written by `lib/record_run.py` from a run's JUnit **and its run log**
|
||||
(see conformance/.gitignore for the negations that keep both in git).
|
||||
|
||||
The honest fix is a committed, machine-readable run ledger; until that exists, the most
|
||||
this report can say is "wired and testable", never "proven".
|
||||
THE REFUSAL POSTURE IS UNCHANGED. Evidence replaced "not derivable" only where evidence
|
||||
exists; nowhere is optimism substituted for it:
|
||||
|
||||
* A ledger row reaches `proven` only if `record_run.py` saw that test's own PASS
|
||||
sentinel token in the run log. The L2/L3 tests SELF-SKIP GREEN — Chainsaw records
|
||||
`passed` on `DR_RESTORE_SKIPPED_OK` exactly as on a real run, and its JUnit carries no
|
||||
stdout — so a skip token maps to `skipped` and can never become proof.
|
||||
* A proof is honoured only while it still describes the current code: if any of the
|
||||
requirement's implementing manifests or verifying tests changed in a commit that is
|
||||
NOT an ancestor of the run's `git_sha`, the row degrades to `stale-proof`, which is
|
||||
not green. A run recorded from a dirty tree (`<sha>-dirty`) is never honoured.
|
||||
* A requirement with no ledger row, or whose verifying tests did not ALL report proven,
|
||||
stays `not-derivable` / `unknown` — never rounded up.
|
||||
* The untracked `_artifacts/report.junit.xml` is STILL only a quarantined advisory with
|
||||
its mtime. It is not the ledger, it is not evidence, and it feeds no status.
|
||||
|
||||
With an empty or absent ledger this report behaves exactly as it did before: every
|
||||
requirement `not-derivable`, headline PROVEN 0, "the most this report can say is wired and
|
||||
testable, never proven".
|
||||
|
||||
READS (all read-only)
|
||||
requirements/registry.yaml the authoritative R-* mirror (via lib/coverage.py)
|
||||
tests/**/chainsaw-test.yaml requirement tags + layer + executable annotation
|
||||
runs/ledger.jsonl COMMITTED proof-of-pass rows (via lib/record_run.py)
|
||||
clusters/<env>/ Flux entrypoint -> the reconciliation graph
|
||||
`git ls-files` TRACKED files only; untracked scratch is not repo state
|
||||
../home-network-k3s/docs/00-vision-and-scope.md OPTIONAL registry<->spec parity check
|
||||
@@ -343,6 +359,169 @@ def spec_requirement_ids(spec_root: Path) -> set[str] | None:
|
||||
return set(RID_RE.findall(doc.read_text()))
|
||||
|
||||
|
||||
# ========================================================================== ledger
|
||||
# PROOF axis. Everything here reads `conformance/runs/ledger.jsonl` (written ONLY by an
|
||||
# explicit `lib/record_run.py` / `run.sh --record` invocation) and downgrades relentlessly.
|
||||
# Nothing in this section may ever turn absent, ambiguous or aged evidence into green.
|
||||
|
||||
DEFAULT_LEDGER = CONF_ROOT / "runs" / "ledger.jsonl"
|
||||
|
||||
# Severity ranking, WORST FIRST. A requirement's proof is the MINIMUM over its verifying
|
||||
# tests, so one failed/skipped/missing test drags the whole requirement down and a single
|
||||
# proven test can never carry the others. `stale-proof` ranks below `proven` on purpose:
|
||||
# it is a real past pass that no longer describes the current code, and it is NOT green.
|
||||
PROOF_ORDER = {
|
||||
"failed": 0, # a ledger row recorded a JUnit failure/error
|
||||
"unknown": 1, # ran, but the log could not evidence pass vs self-skip
|
||||
"not-derivable": 2, # no ledger row for this test at all (the pre-ledger default)
|
||||
"junit-skipped": 3, # JUnit <skipped/> — did not execute
|
||||
"skipped": 4, # SELF-SKIPPED green: exited 0 without exercising the behaviour
|
||||
"stale-proof": 5, # proven at a sha the implementation/test has since moved past
|
||||
"proven": 6, # the ONLY green state
|
||||
}
|
||||
# record_run.py's two "we cannot tell" states collapse to one reported state here; the
|
||||
# per-test detail in status.json keeps the distinction for whoever has to fix it.
|
||||
_PROOF_ALIASES = {
|
||||
"unknown-no-sentinel-declared": "unknown",
|
||||
"unknown-sentinel-absent": "unknown",
|
||||
}
|
||||
|
||||
|
||||
def read_ledger(path: Path) -> list[dict]:
|
||||
"""Parse the append-only JSONL ledger. A malformed line is SKIPPED, never guessed at.
|
||||
|
||||
Absent or empty file -> [] , which reproduces this report's pre-ledger behaviour
|
||||
exactly (every requirement `not-derivable`, headline PROVEN 0).
|
||||
"""
|
||||
if not path.is_file():
|
||||
return []
|
||||
rows: list[dict] = []
|
||||
for line in path.read_text(errors="replace").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict) and obj.get("test"):
|
||||
rows.append(obj)
|
||||
return rows
|
||||
|
||||
|
||||
def latest_by_test(rows: list[dict]) -> dict[str, dict]:
|
||||
"""Most recent ledger row per test name (ts_utc, then file order as the tiebreak).
|
||||
|
||||
Latest wins rather than best-ever: a test that has since gone back to self-skipping
|
||||
must stop reading as proven.
|
||||
"""
|
||||
latest: dict[str, dict] = {}
|
||||
for i, r in enumerate(rows):
|
||||
key = r["test"]
|
||||
prev = latest.get(key)
|
||||
if prev is None or (r.get("ts_utc", ""), i) >= (prev.get("ts_utc", ""),
|
||||
prev["_i"]):
|
||||
latest[key] = {**r, "_i": i}
|
||||
return latest
|
||||
|
||||
|
||||
class _GitOracle:
|
||||
"""Small cached wrapper for the two git questions the staleness rule asks."""
|
||||
|
||||
def __init__(self, repo: Path):
|
||||
self.repo = repo
|
||||
self._last: dict[tuple[str, ...], str | None] = {}
|
||||
self._anc: dict[tuple[str, str], bool] = {}
|
||||
|
||||
def _git(self, *args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", "-C", str(self.repo), *args],
|
||||
capture_output=True, text=True)
|
||||
|
||||
def last_change(self, paths: tuple[str, ...]) -> str | None:
|
||||
"""Commit that last touched any of `paths`; None if git knows none of them."""
|
||||
if not paths:
|
||||
return None
|
||||
if paths in self._last:
|
||||
return self._last[paths]
|
||||
p = self._git("log", "-1", "--format=%H", "--", *paths)
|
||||
sha = p.stdout.strip() if p.returncode == 0 else ""
|
||||
self._last[paths] = sha or None
|
||||
return self._last[paths]
|
||||
|
||||
def is_ancestor(self, older: str, newer: str) -> bool:
|
||||
"""True iff `older` is an ancestor of (or equal to) `newer`.
|
||||
|
||||
False whenever either rev is unresolvable — an unknown or `-dirty` sha (a run
|
||||
recorded from a modified working tree) can never satisfy the freshness test, which
|
||||
is the intended fail-closed direction.
|
||||
"""
|
||||
key = (older, newer)
|
||||
if key in self._anc:
|
||||
return self._anc[key]
|
||||
ok = self._git("merge-base", "--is-ancestor", older, newer).returncode == 0
|
||||
self._anc[key] = ok
|
||||
return ok
|
||||
|
||||
|
||||
def derive_proof(git: _GitOracle, rid: str, verifiers: list[dict],
|
||||
impl_files: list[dict], latest: dict[str, dict]) -> dict:
|
||||
"""PROOF for one requirement, from the ledger, with the staleness rule applied.
|
||||
|
||||
STALENESS RULE — a proof is a claim about a specific tree, not a permanent badge:
|
||||
relevant paths = every implementing artifact this report found for the requirement
|
||||
+ every verifying test's directory
|
||||
last_change = `git log -1 -- <relevant paths>`
|
||||
A row's `proven` is honoured only if `last_change` is an ANCESTOR of the row's
|
||||
`git_sha`, i.e. the run happened at or after the newest change to the code it
|
||||
claims to prove. Otherwise it degrades to `stale-proof` (not green). A `-dirty`
|
||||
sha, a sha this clone does not have (rebased/dropped), or a path git has never
|
||||
seen (an uncommitted test) all fail the ancestry test and degrade the same way.
|
||||
|
||||
The requirement's state is the MINIMUM over its verifying tests: every verifying test
|
||||
must be proven and fresh before the requirement reads `proven`. A test with no ledger
|
||||
row contributes `not-derivable`, so a partially-recorded run cannot manufacture green.
|
||||
"""
|
||||
paths = sorted({f["path"] for f in impl_files}
|
||||
| {os.path.dirname(v["path"]) for v in verifiers if v.get("path")})
|
||||
last_change = git.last_change(tuple(paths))
|
||||
|
||||
per_test: list[dict] = []
|
||||
for v in verifiers:
|
||||
row = latest.get(v["name"])
|
||||
if row is None:
|
||||
per_test.append({"test": v["name"], "state": "not-derivable",
|
||||
"reason": "no ledger row — this test has never been "
|
||||
"recorded by run.sh --record / record_run.py"})
|
||||
continue
|
||||
base = _PROOF_ALIASES.get(row.get("proof", ""), row.get("proof", "unknown"))
|
||||
if base not in PROOF_ORDER:
|
||||
base = "unknown"
|
||||
state, reason = base, row.get("note", "")
|
||||
if base == "proven":
|
||||
sha = row.get("git_sha", "")
|
||||
if last_change and not git.is_ancestor(last_change, sha):
|
||||
state = "stale-proof"
|
||||
reason = (f"proven at {sha[:12]}, but the implementing manifests / test "
|
||||
f"changed in {last_change[:12]}, which that run predates")
|
||||
per_test.append({"test": v["name"], "state": state, "reason": reason,
|
||||
"run_id": row.get("run_id"), "ts_utc": row.get("ts_utc"),
|
||||
"git_sha": row.get("git_sha"), "env": row.get("env"),
|
||||
"sentinel": row.get("sentinel"),
|
||||
"junit_result": row.get("junit_result")})
|
||||
|
||||
if not per_test:
|
||||
state = "not-derivable"
|
||||
reason = "no requirement-tagged test exists to be proven"
|
||||
else:
|
||||
worst = min(per_test, key=lambda d: PROOF_ORDER[d["state"]])
|
||||
state, reason = worst["state"], worst["reason"]
|
||||
if state == "not-derivable" and not reason:
|
||||
reason = "no committed ledger row for this requirement's verifying test(s)"
|
||||
|
||||
return {"state": state, "reason": reason, "last_change_sha": last_change,
|
||||
"evidence_paths": paths, "per_test": per_test}
|
||||
|
||||
|
||||
# ========================================================================== build
|
||||
IMPL_ORDER = {"none": 0, "off-cluster": 1, "unwired": 2, "reconciled-partial": 3,
|
||||
"reconciled": 4}
|
||||
@@ -369,7 +548,13 @@ LABELS = {
|
||||
|
||||
def build(repo: Path, registry: list[dict], tests: list[dict], impl: dict[str, list[dict]],
|
||||
applied: set[str], gitops_roots: set[str], spec_ids: set[str] | None,
|
||||
env: str) -> dict:
|
||||
env: str, ledger_rows: list[dict] | None = None) -> dict:
|
||||
# PROOF axis inputs. `ledger_rows=None` / `[]` reproduces the pre-ledger behaviour:
|
||||
# `latest` is empty, so every requirement derives `not-derivable` exactly as before.
|
||||
ledger_rows = ledger_rows or []
|
||||
latest = latest_by_test(ledger_rows)
|
||||
git = _GitOracle(repo)
|
||||
|
||||
classified = [classify_test(t) for t in tests]
|
||||
by_req_tests: dict[str, list[dict]] = {r["id"]: [] for r in registry}
|
||||
for t, c in zip(tests, classified):
|
||||
@@ -439,11 +624,9 @@ def build(repo: Path, registry: list[dict], tests: list[dict], impl: dict[str, l
|
||||
"blocked_by": sorted({t["blocked_by"] for t in ts if t["blocked_by"]}),
|
||||
"tests": ts,
|
||||
},
|
||||
# NEVER derived. See the module docstring.
|
||||
"proof": {"state": "not-derivable",
|
||||
"reason": "no committed, machine-readable record of a test run "
|
||||
"exists in the repo (_artifacts/ and results/ are "
|
||||
"git-ignored); DECISIONS-LOG.md prose is not a signal"},
|
||||
# Derived ONLY from the committed ledger, and only as far as it reaches.
|
||||
# With no ledger row this stays `not-derivable`, as it always was.
|
||||
"proof": derive_proof(git, rid, ts, ev, latest),
|
||||
"status": LABELS[(impl_state, verify_state)],
|
||||
}
|
||||
|
||||
@@ -492,8 +675,19 @@ def build(repo: Path, registry: list[dict], tests: list[dict], impl: dict[str, l
|
||||
and v["verification"]["state"] == "executable"),
|
||||
"safety_critical_total": count(lambda v: v["safety_critical"]),
|
||||
"safety_critical_without_executable_l3": len(safety_scaffold),
|
||||
"proven": 0,
|
||||
"proven_derivable": False,
|
||||
# PROVEN is now a real count — but only of requirements whose EVERY verifying
|
||||
# test has a committed ledger row that saw its pass sentinel, at a sha the
|
||||
# implementation has not moved past. `stale-proof` and `skipped` are counted
|
||||
# separately and are NOT green.
|
||||
"proven": count(lambda v: v["proof"]["state"] == "proven"),
|
||||
"proof_stale": count(lambda v: v["proof"]["state"] == "stale-proof"),
|
||||
"proof_skipped": count(lambda v: v["proof"]["state"] == "skipped"),
|
||||
"proof_failed": count(lambda v: v["proof"]["state"] == "failed"),
|
||||
"proof_unknown": count(lambda v: v["proof"]["state"] == "unknown"),
|
||||
"proof_not_derivable": count(
|
||||
lambda v: v["proof"]["state"] in ("not-derivable", "junit-skipped")),
|
||||
"ledger_rows": len(ledger_rows),
|
||||
"proven_derivable": bool(ledger_rows),
|
||||
"coverage_style_covered": count(lambda v: v["verification"]["state"] != "none"),
|
||||
}
|
||||
summary["uncited_by_any_reconciled_manifest"] = count(
|
||||
@@ -552,16 +746,39 @@ def render_md(rep: dict, nodes: list[dict], advisory: dict | None) -> str:
|
||||
o.append(f"| Safety-critical requirements whose L3 test is a scaffold | "
|
||||
f"**{s['safety_critical_without_executable_l3']}** of "
|
||||
f"{s['safety_critical_total']} |")
|
||||
o.append(f"| **PROVEN (a test actually passed)** | **{s['proven']} — NOT DERIVABLE** |")
|
||||
o.append("")
|
||||
o.append("> **Proof is deliberately not derived.** Nothing tracked in git records a "
|
||||
"test result: `conformance/_artifacts/` and `results/` are git-ignored, and "
|
||||
"the only record of a green run is DECISIONS-LOG.md prose — hand-maintained, "
|
||||
"which is the very thing this report replaces. Per "
|
||||
"`agents/definition-of-done.md` a requirement is done only when its tagged "
|
||||
"test **passes**, so the strongest claim this report can make is "
|
||||
"*wired and testable*. Closing the gap needs a committed, machine-readable "
|
||||
"run ledger; until then, treat every row below as unproven.\n")
|
||||
if s["proven_derivable"]:
|
||||
o.append(f"| **PROVEN (every verifying test passed, evidenced)** | "
|
||||
f"**{s['proven']}** |")
|
||||
o.append(f"| Proof exists but is STALE (code changed after the run) | "
|
||||
f"{s['proof_stale']} |")
|
||||
o.append(f"| Verifying test SELF-SKIPPED green (not proof) | {s['proof_skipped']} |")
|
||||
o.append(f"| Verifying test FAILED in the recorded run | {s['proof_failed']} |")
|
||||
o.append(f"| Ran, but the log could not evidence pass vs self-skip | "
|
||||
f"{s['proof_unknown']} |")
|
||||
o.append(f"| No committed ledger row yet | {s['proof_not_derivable']} |")
|
||||
o.append("")
|
||||
o.append(f"> **Proof comes from `conformance/runs/ledger.jsonl`** "
|
||||
f"({s['ledger_rows']} row(s)), written only by an explicit "
|
||||
f"`run.sh --record` / `lib/record_run.py` invocation. A row reaches "
|
||||
f"`proven` ONLY when the run log carried that test's own pass sentinel "
|
||||
f"token — the L2/L3 tests self-skip GREEN (Chainsaw records `passed` on "
|
||||
f"`DR_RESTORE_SKIPPED_OK` exactly as on a real run, and its JUnit carries "
|
||||
f"no stdout), so a skip token maps to `skipped` and can never become "
|
||||
f"proof. A proof is honoured only while the implementing manifests and "
|
||||
f"the test itself have not changed since the recorded `git_sha`; "
|
||||
f"otherwise it degrades to `stale-proof`, which is not green.\n")
|
||||
else:
|
||||
o.append(f"| **PROVEN (a test actually passed)** | **{s['proven']} — NOT DERIVABLE** |")
|
||||
o.append("")
|
||||
o.append("> **No proof is derivable: `conformance/runs/ledger.jsonl` is absent or "
|
||||
"empty.** The ledger is the committed, machine-readable run record; until "
|
||||
"a run is captured with `run.sh --record`, nothing tracked in git records "
|
||||
"a test result (`conformance/_artifacts/` and `results/` are git-ignored, "
|
||||
"and DECISIONS-LOG.md prose is hand-maintained — the very thing this "
|
||||
"report replaces). Per `agents/definition-of-done.md` a requirement is "
|
||||
"done only when its tagged test **passes**, so the strongest claim this "
|
||||
"report can make is *wired and testable*; treat every row below as "
|
||||
"unproven.\n")
|
||||
o.append(f"> For contrast, `coverage.md` counts **{s['coverage_style_covered']}** "
|
||||
f"requirements as covered — that count includes the "
|
||||
f"{s['verify_scaffold_only']} whose only verifying test is a TODO scaffold.\n")
|
||||
@@ -612,7 +829,15 @@ def render_md(rep: dict, nodes: list[dict], advisory: dict | None) -> str:
|
||||
if ver["blocked_by"]:
|
||||
vcell += f" · blocked-by: {'; '.join(ver['blocked_by'])}"
|
||||
sc = " ⚠safety" if v["safety_critical"] else ""
|
||||
o.append(f"| {rid}{sc} | **{v['status']}** | {icell} | {vcell} | not-derivable |")
|
||||
pr = v["proof"]
|
||||
if pr["state"] == "not-derivable":
|
||||
pcell = "not-derivable"
|
||||
else:
|
||||
runs = sorted({d["run_id"] for d in pr["per_test"] if d.get("run_id")})
|
||||
pcell = f"**{pr['state']}**" if pr["state"] == "proven" else pr["state"]
|
||||
if runs:
|
||||
pcell += f" ({', '.join(runs)})"
|
||||
o.append(f"| {rid}{sc} | **{v['status']}** | {icell} | {vcell} | {pcell} |")
|
||||
|
||||
if rep["safety_critical_without_executable_l3"]:
|
||||
o.append("\n## Safety-critical requirements whose L3 fault test is a SCAFFOLD "
|
||||
@@ -701,22 +926,76 @@ def render_md(rep: dict, nodes: list[dict], advisory: dict | None) -> str:
|
||||
o.append("| executable vs scaffold | TODO markers in the test body outrank the "
|
||||
"`executable` annotation; a real body without the annotation still counts "
|
||||
"and is flagged | conservative — ambiguity never upgrades |")
|
||||
o.append("| **test actually passed** | **NOT DERIVED** — nothing in git records a "
|
||||
"run result | refused |")
|
||||
o.append("| **test actually passed** | the committed `runs/ledger.jsonl` row for "
|
||||
"every verifying test: JUnit result **plus** that test's own pass sentinel "
|
||||
"token grepped from the run log | exact where a run was recorded, refused "
|
||||
"everywhere else — a self-skip token or a missing/unrecognised sentinel "
|
||||
"NEVER yields `proven` |")
|
||||
o.append("| proof still current | the row's `git_sha` must be a descendant of the "
|
||||
"last commit touching the requirement's implementing manifests or its test "
|
||||
"| conservative — an unresolvable or `-dirty` sha degrades to `stale-proof` |")
|
||||
o.append("| cluster liveness (pods/CRs healthy) | **NOT DERIVED** — no cluster is "
|
||||
"contacted; `reconciled` means Flux *would* apply it, not that it is "
|
||||
"healthy | refused |")
|
||||
o.append("| phase/board status | **NOT DERIVED** — `agents/task-board.md` is stale "
|
||||
"and is not a signal | refused |")
|
||||
|
||||
o.append("\n## Run ledger — the committed proof-of-pass evidence\n")
|
||||
if not s["proven_derivable"]:
|
||||
o.append("- `conformance/runs/ledger.jsonl` holds no rows. Capture a run with "
|
||||
"`./run.sh --record` (recording is never implicit: a run that silently "
|
||||
"records itself is how a bad run becomes a permanent false proof), then "
|
||||
"commit `conformance/runs/<run-id>/` **with its `run.log`** and the "
|
||||
"ledger. The log is the load-bearing half — Chainsaw's JUnit carries no "
|
||||
"stdout, so without it a self-skip is indistinguishable from a pass.")
|
||||
else:
|
||||
o.append(f"- {s['ledger_rows']} row(s) in `conformance/runs/ledger.jsonl`. "
|
||||
f"Per-test evidence for every non-`not-derivable` requirement below; "
|
||||
f"full rows (env, chainsaw/kube versions, sentinel, note) are in "
|
||||
f"`status.json` under `by_requirement.<R-ID>.proof.per_test`.\n")
|
||||
o.append("| Requirement | Proof | Test | Ledger state | Sentinel | Run |")
|
||||
o.append("|---|---|---|---|---|---|")
|
||||
for rid, v in rep["by_requirement"].items():
|
||||
pr = v["proof"]
|
||||
if pr["state"] == "not-derivable":
|
||||
continue
|
||||
for d in pr["per_test"]:
|
||||
o.append(f"| {rid} | {pr['state']} | `{d['test']}` | {d['state']} | "
|
||||
f"`{d.get('sentinel') or '—'}` | {d.get('run_id') or '—'} |")
|
||||
stale = [(rid, v) for rid, v in rep["by_requirement"].items()
|
||||
if v["proof"]["state"] == "stale-proof"]
|
||||
if stale:
|
||||
o.append("\n### Stale proofs — a real pass that no longer describes this tree\n")
|
||||
o.append("These requirements DID pass, but their implementing manifests or "
|
||||
"their verifying test changed in a commit the recorded run predates. "
|
||||
"A stale proof is not green; re-run with `--record`.\n")
|
||||
for rid, v in stale:
|
||||
pr = v["proof"]
|
||||
o.append(f"- **{rid}** — {pr['reason']}")
|
||||
skipped = [(rid, v) for rid, v in rep["by_requirement"].items()
|
||||
if v["proof"]["state"] == "skipped"]
|
||||
if skipped:
|
||||
o.append("\n### SELF-SKIPPED green — recorded, and deliberately NOT proof\n")
|
||||
o.append("Chainsaw reported `passed` for these, because the test exited 0 "
|
||||
"after emitting its skip token: a precondition (Longhorn "
|
||||
"StorageClass, Velero CRDs, >=3 Ready nodes) was absent, so the "
|
||||
"behaviour was never exercised. This is exactly the row a "
|
||||
"JUnit-only ledger would have recorded as green.\n")
|
||||
for rid, v in skipped:
|
||||
toks = ", ".join(sorted({d.get("sentinel") or "?"
|
||||
for d in v["proof"]["per_test"]
|
||||
if d["state"] == "skipped"}))
|
||||
o.append(f"- **{rid}** — skip token(s): `{toks}`")
|
||||
|
||||
o.append("\n## Advisory — untracked local run artifact (NOT counted)\n")
|
||||
if advisory:
|
||||
o.append(f"- `{advisory['path']}` (untracked, git-ignored), last written "
|
||||
f"{advisory['mtime']}: {advisory['tests']} testcase(s), "
|
||||
f"{advisory['failures']} failure(s), {advisory['skipped']} skipped.")
|
||||
o.append(f"- requirement IDs it names: {', '.join(advisory['requirements']) or '—'}")
|
||||
o.append("- This is one developer's local workspace, not repo state. It is shown "
|
||||
"for orientation only and contributes to NO status above.")
|
||||
o.append("- This is one developer's local workspace, not repo state, and it is NOT "
|
||||
"the ledger. It is shown for orientation only and contributes to NO "
|
||||
"status above — a JUnit alone cannot tell a self-skip from a pass.")
|
||||
else:
|
||||
o.append("- none present.")
|
||||
return "\n".join(o) + "\n"
|
||||
@@ -841,6 +1120,122 @@ def _selftest() -> int:
|
||||
classify_test({"name": "s", "path": str(f), "layer": "L2"})["state"]
|
||||
== "scaffold")
|
||||
|
||||
# ---------------------------------------------------------------- PROOF / ledger
|
||||
# These are the assertions that stand between this report and a fake green.
|
||||
check("empty ledger parses to no rows (pre-ledger behaviour is preserved)",
|
||||
read_ledger(Path("/nonexistent/ledger.jsonl")) == [])
|
||||
|
||||
def row(**kw):
|
||||
base = {"ts_utc": "2026-09-01T12:00:00Z", "git_sha": "deadbeef", "env": "dev",
|
||||
"chainsaw_version": "v0.2.12", "kube_version": "v1.36.4", "layer": "L3",
|
||||
"test": "dr-restore", "requirements": ["R-STATE-03"],
|
||||
"junit_result": "passed", "sentinel": None, "proof": "skipped",
|
||||
"run_id": "20260901T120000Z-deadbee", "note": ""}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
verifiers = [{"name": "dr-restore",
|
||||
"path": "conformance/tests/l3-fault-injection/dr-restore/"
|
||||
"chainsaw-test.yaml"}]
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
gr = Path(td)
|
||||
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
||||
|
||||
def git(*a):
|
||||
return subprocess.run(["git", "-C", str(gr), *a], capture_output=True,
|
||||
text=True, env=env)
|
||||
|
||||
git("init", "-q", "-b", "main")
|
||||
(gr / "platform").mkdir()
|
||||
(gr / "platform" / "velero.yaml").write_text("kind: X # R-STATE-03\n")
|
||||
git("add", "-A"); git("commit", "-qm", "one")
|
||||
sha_a = git("rev-parse", "HEAD").stdout.strip()
|
||||
(gr / "platform" / "velero.yaml").write_text("kind: X # R-STATE-03 v2\n")
|
||||
git("add", "-A"); git("commit", "-qm", "two")
|
||||
sha_b = git("rev-parse", "HEAD").stdout.strip()
|
||||
|
||||
oracle = _GitOracle(gr)
|
||||
impl = [{"path": "platform/velero.yaml"}]
|
||||
|
||||
# *** THE CASE THIS WHOLE FEATURE EXISTS FOR ***
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(proof="skipped", junit_result="passed",
|
||||
sentinel="DR_RESTORE_SKIPPED_OK",
|
||||
git_sha=sha_b)]))
|
||||
check("a SELF-SKIP row on a GREEN JUnit is 'skipped', NEVER 'proven'",
|
||||
pr["state"] == "skipped")
|
||||
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(proof="proven", git_sha=sha_b,
|
||||
sentinel="DR_RESTORED_OK")]))
|
||||
check("a proven row at the CURRENT sha is honoured", pr["state"] == "proven")
|
||||
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(proof="proven", git_sha=sha_a,
|
||||
sentinel="DR_RESTORED_OK")]))
|
||||
check("STALENESS: proven at a sha the implementation has moved past degrades "
|
||||
"to stale-proof", pr["state"] == "stale-proof")
|
||||
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(proof="proven", git_sha=f"{sha_b}-dirty",
|
||||
sentinel="DR_RESTORED_OK")]))
|
||||
check("a run recorded from a DIRTY tree is never honoured as proof",
|
||||
pr["state"] == "stale-proof")
|
||||
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(proof="proven", git_sha="0" * 40)]))
|
||||
check("a git_sha this clone cannot resolve degrades, never upgrades",
|
||||
pr["state"] == "stale-proof")
|
||||
|
||||
for bad, expect in (("failed", "failed"),
|
||||
("junit-skipped", "junit-skipped"),
|
||||
("unknown-sentinel-absent", "unknown"),
|
||||
("unknown-no-sentinel-declared", "unknown"),
|
||||
("something-new-from-the-future", "unknown")):
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(proof=bad, git_sha=sha_b)]))
|
||||
check(f"ledger state {bad!r} maps to {expect!r}, never to proven",
|
||||
pr["state"] == expect)
|
||||
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl, {})
|
||||
check("no ledger row at all stays not-derivable",
|
||||
pr["state"] == "not-derivable")
|
||||
|
||||
# Two verifying tests, only one proven: the requirement must NOT read proven.
|
||||
two = verifiers + [{"name": "data-survival",
|
||||
"path": "conformance/tests/l3-fault-injection/"
|
||||
"data-survival/chainsaw-test.yaml"}]
|
||||
pr = derive_proof(oracle, "R-STATE-03", two, impl,
|
||||
latest_by_test([row(proof="proven", git_sha=sha_b)]))
|
||||
check("a requirement is proven only when EVERY verifying test is",
|
||||
pr["state"] == "not-derivable")
|
||||
pr = derive_proof(oracle, "R-STATE-03", two, impl,
|
||||
latest_by_test([row(proof="proven", git_sha=sha_b),
|
||||
row(test="data-survival", proof="skipped",
|
||||
git_sha=sha_b)]))
|
||||
check("worst-of wins across verifying tests (skip drags proven down)",
|
||||
pr["state"] == "skipped")
|
||||
|
||||
# Latest row wins, not best-ever: a test that regressed to self-skipping must
|
||||
# stop reading as proven.
|
||||
pr = derive_proof(oracle, "R-STATE-03", verifiers, impl,
|
||||
latest_by_test([row(ts_utc="2026-09-01T10:00:00Z",
|
||||
proof="proven", git_sha=sha_b),
|
||||
row(ts_utc="2026-09-02T10:00:00Z",
|
||||
proof="skipped", git_sha=sha_b)]))
|
||||
check("the LATEST ledger row wins, not the best one ever recorded",
|
||||
pr["state"] == "skipped")
|
||||
|
||||
# A malformed ledger line is skipped, never guessed at.
|
||||
lf = gr / "ledger.jsonl"
|
||||
lf.write_text('{"test":"a","proof":"proven"}\nnot json at all\n\n'
|
||||
'{"proof":"proven"}\n')
|
||||
rows = read_ledger(lf)
|
||||
check("malformed / test-less ledger lines are dropped, not guessed at",
|
||||
len(rows) == 1 and rows[0]["test"] == "a")
|
||||
|
||||
if not ok:
|
||||
return 1
|
||||
sys.stderr.write("status.py selftest: PASS\n")
|
||||
@@ -865,6 +1260,10 @@ def main() -> int:
|
||||
help="exit 1 on defects this tool owns: registry/spec drift, "
|
||||
"executable-annotation conflicts, or a safety-critical "
|
||||
"requirement whose only L3 test is a scaffold")
|
||||
ap.add_argument("--ledger", type=Path, default=DEFAULT_LEDGER,
|
||||
help=f"committed proof-of-pass ledger written by lib/record_run.py "
|
||||
f"(default {DEFAULT_LEDGER}). Absent or empty => every "
|
||||
f"requirement stays `not-derivable`, as before the ledger existed")
|
||||
ap.add_argument("--selftest", action="store_true", help="run the built-in self-test")
|
||||
args = ap.parse_args()
|
||||
|
||||
@@ -895,7 +1294,12 @@ def main() -> int:
|
||||
applied, nodes = reconciled_files(repo, entry)
|
||||
gitops_roots = {n["resolved"].split("/", 1)[0] for n in nodes if n["exists"]}
|
||||
impl = scan_implementation(repo, tracked_files(repo))
|
||||
rep = build(repo, registry, tests, impl, applied, gitops_roots, spec_ids, args.env)
|
||||
ledger_rows = read_ledger(args.ledger)
|
||||
rep = build(repo, registry, tests, impl, applied, gitops_roots, spec_ids, args.env,
|
||||
ledger_rows)
|
||||
rep["ledger"] = {"path": os.path.relpath(args.ledger, REPO_ROOT),
|
||||
"rows": len(ledger_rows),
|
||||
"present": args.ledger.is_file()}
|
||||
rep["flux_graph"] = nodes
|
||||
rep["gitops_roots"] = sorted(gitops_roots)
|
||||
rep["reconciled_file_count"] = len(applied)
|
||||
@@ -915,7 +1319,11 @@ def main() -> int:
|
||||
f"(+{s['wired_or_partial_and_testable'] - s['wired_and_testable']} partial) | "
|
||||
f"scaffold-only {s['verify_scaffold_only']} | "
|
||||
f"safety-critical w/o executable L3 "
|
||||
f"{s['safety_critical_without_executable_l3']} | PROVEN: not derivable")
|
||||
f"{s['safety_critical_without_executable_l3']} | "
|
||||
+ (f"PROVEN: {s['proven']} (stale {s['proof_stale']}, self-skipped "
|
||||
f"{s['proof_skipped']}, failed {s['proof_failed']}, unknown "
|
||||
f"{s['proof_unknown']}) from {s['ledger_rows']} ledger row(s)"
|
||||
if s["proven_derivable"] else "PROVEN: not derivable (empty run ledger)"))
|
||||
print(f"[status] wrote {args.out/'status.md'} and {args.out/'status.json'}")
|
||||
|
||||
rc = 0
|
||||
|
||||
+93
-3
@@ -14,12 +14,33 @@
|
||||
# ./run.sh --status # DERIVED STATUS report only (no cluster): per
|
||||
# # requirement, is it implemented, is it actually
|
||||
# # reconciled by Flux, is its test executable or a
|
||||
# # scaffold. Proof-of-pass is deliberately NOT
|
||||
# # derived — see lib/status.py.
|
||||
# # scaffold, and whether the committed run ledger
|
||||
# # (runs/ledger.jsonl) actually proves it — see
|
||||
# # lib/status.py. No ledger row => not-derivable.
|
||||
# ./run.sh --lint # structural-validate test defs (no cluster)
|
||||
# ./run.sh --strict-coverage # hygiene gate: fail on tag/annotation drift
|
||||
# ./run.sh --requirements R-NET-01 # ALSO asserts those IDs are covered (merge gate)
|
||||
# ./run.sh --require-all # phase-exit: every requirement must be covered
|
||||
# ./run.sh --record # RUN AND CAPTURE: tee the whole run to
|
||||
# # runs/<UTC>-<short-sha>/run.log, copy the JUnit
|
||||
# # beside it, and append one ledger row per
|
||||
# # testcase to runs/ledger.jsonl (lib/record_run.py)
|
||||
#
|
||||
# --record — PROOF-OF-PASS, CAPTURED AT RUN TIME (why it is a separate flag)
|
||||
# --------------------------------------------------------------------------
|
||||
# Without it, nothing about a green run ever reaches repo state: _artifacts/ is gitignored
|
||||
# and the root .gitignore kills *.junit.xml globally, which is why `--status` reports
|
||||
# PROVEN 0 for all 51 requirements. `--record` writes the one tracked thing that fixes
|
||||
# that — `conformance/runs/` (see conformance/.gitignore for the negations).
|
||||
#
|
||||
# RECORDING IS DELIBERATELY EXPLICIT. A run that silently records itself is how a bad run
|
||||
# becomes a permanent false proof, so the plain `./run.sh` writes no ledger row at all.
|
||||
#
|
||||
# BOTH FILES ARE KEPT, and the LOG is the load-bearing one. The L2/L3 tests self-skip
|
||||
# GREEN: when a precondition is missing they echo e.g. `DR_RESTORE_SKIPPED_OK` and exit 0,
|
||||
# and Chainsaw records `passed` exactly as on a real run. Chainsaw's JUnit carries no
|
||||
# stdout, so the sentinel token in run.log is the ONLY thing that separates proven from
|
||||
# skipped. A JUnit committed on its own would record a skip as proof of R-STATE-03.
|
||||
#
|
||||
# Artifacts (always written under conformance/_artifacts/, gitignored):
|
||||
# _artifacts/report.junit.xml JUnit (Chainsaw output, or skipped-scaffold shim)
|
||||
@@ -28,6 +49,11 @@
|
||||
# _artifacts/status/status.md human requirement-status report (--status)
|
||||
# _artifacts/status/status.json machine requirement-status report (--status)
|
||||
#
|
||||
# Run ledger (TRACKED — commit these; --record only):
|
||||
# runs/<UTC>-<short-sha>/run.log full tee'd run output (carries the sentinels)
|
||||
# runs/<UTC>-<short-sha>/report.junit.xml the JUnit for that same run
|
||||
# runs/ledger.jsonl append-only, one row per testcase per run
|
||||
#
|
||||
# Phase 0: no cluster exists. The behavior tests are valid scaffolds that cannot
|
||||
# execute yet, so run.sh emits a *skipped* JUnit and a coverage report, and exits
|
||||
# green — the harness is proven reproducible while later phases fill in behavior.
|
||||
@@ -47,12 +73,17 @@ STATUS_OUT="${ARTIFACTS}/status"
|
||||
JUNIT_OUT="${ARTIFACTS}/report.junit.xml"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
# Every argument as given, so `--record` can re-exec this script unchanged (minus itself)
|
||||
# under `tee`. Captured BEFORE parsing so nothing is normalized away.
|
||||
ORIG_ARGS=("$@")
|
||||
|
||||
LAYER="all"
|
||||
REQS=""
|
||||
MODE="run" # run | coverage-only | status | lint
|
||||
STRICT_COVERAGE=0
|
||||
REQUIRE_ALL=0
|
||||
FORCE_NO_CLUSTER=0
|
||||
RECORD=0
|
||||
|
||||
# Documented, out-of-scope coverage gaps exempt from the completeness gate: requirements
|
||||
# owned by another workstream that WS-TEST does not yet scaffold. Listed here so the
|
||||
@@ -83,7 +114,9 @@ FORCE_NO_CLUSTER=0
|
||||
# longer a gap and has been removed from this list; the coverage report counts it as covered.
|
||||
DOCUMENTED_GAPS=""
|
||||
|
||||
usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }
|
||||
# Print the header block (line 2 through the closing `####` rule, which is dropped) as
|
||||
# help. Bounded by the rule rather than a fixed line number so the header can grow.
|
||||
usage() { sed -n '2,/^#####/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; }
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
@@ -95,6 +128,7 @@ while [ $# -gt 0 ]; do
|
||||
--strict-coverage) STRICT_COVERAGE=1; shift ;;
|
||||
--require-all) REQUIRE_ALL=1; shift ;;
|
||||
--no-cluster) FORCE_NO_CLUSTER=1; shift ;;
|
||||
--record) RECORD=1; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; usage; exit 2 ;;
|
||||
esac
|
||||
@@ -104,6 +138,62 @@ mkdir -p "${ARTIFACTS}" "${COVERAGE_OUT}" "${STATUS_OUT}"
|
||||
|
||||
log() { printf '\033[1m[run.sh]\033[0m %s\n' "$*"; }
|
||||
|
||||
###############################################################################
|
||||
# --record: capture this run as committed evidence, then hand it to the ledger.
|
||||
#
|
||||
# Implemented as a ONE-LEVEL re-exec of this same script under `tee`, so the run itself is
|
||||
# byte-for-byte the ordinary run (same code path, same artifacts) and the log is the exact
|
||||
# operator-visible output — no second, differently-behaving "recording mode" to drift.
|
||||
# CONF_RECORD_CHILD guards against recursion.
|
||||
#
|
||||
# The recorder runs REGARDLESS of the run's exit status: a red run must be recorded as red
|
||||
# (that is evidence too), and run.sh still exits with the child's real rc, so no gate is
|
||||
# softened by recording. `${PIPESTATUS[0]}` is what carries that rc past `tee`.
|
||||
###############################################################################
|
||||
if [ "${RECORD}" = 1 ] && [ -z "${CONF_RECORD_CHILD:-}" ]; then
|
||||
case "${MODE}" in
|
||||
run) ;;
|
||||
# --record is about EVIDENCE OF A RUN. --coverage-only/--status/--lint execute no
|
||||
# test, so a ledger row from one of them would be a row about nothing.
|
||||
*) echo "--record captures a TEST RUN; --${MODE} executes none." >&2; exit 2 ;;
|
||||
esac
|
||||
RUN_SHA="$(git -C "${CONF_ROOT}" rev-parse --short HEAD 2>/dev/null || echo nogit)"
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${RUN_SHA}"
|
||||
RUN_DIR="${CONF_ROOT}/runs/${RUN_ID}"
|
||||
mkdir -p "${RUN_DIR}"
|
||||
# Drop any JUnit left behind by a PREVIOUS run before starting, so this run dir can only
|
||||
# ever capture a report THIS run produced. Copying a stale artifact would pair someone
|
||||
# else's result with this run's log — a fabricated pairing, which is precisely the class
|
||||
# of false proof the ledger exists to prevent.
|
||||
rm -f "${JUNIT_OUT}"
|
||||
# Strip --record from the argv handed to the child; everything else passes through.
|
||||
CHILD_ARGS=()
|
||||
for a in ${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"}; do
|
||||
[ "$a" = "--record" ] || CHILD_ARGS+=("$a")
|
||||
done
|
||||
log "RECORD: capturing this run to runs/${RUN_ID}/ (run.log + report.junit.xml)"
|
||||
set +e
|
||||
CONF_RECORD_CHILD=1 "${BASH_SOURCE[0]}" ${CHILD_ARGS[@]+"${CHILD_ARGS[@]}"} 2>&1 \
|
||||
| tee "${RUN_DIR}/run.log"
|
||||
RUN_RC=${PIPESTATUS[0]}
|
||||
set -e
|
||||
if [ -f "${JUNIT_OUT}" ]; then
|
||||
cp "${JUNIT_OUT}" "${RUN_DIR}/report.junit.xml"
|
||||
else
|
||||
log "RECORD: no ${JUNIT_OUT} was produced — the ledger step will SKIP (never green)."
|
||||
fi
|
||||
# Explicit, separate step: read the JUnit + the log and append one row per testcase.
|
||||
# rc 3 = inputs absent (nothing recorded, not green); it must not mask the run's own rc.
|
||||
# CONF_LEDGER exists so the harness's own tests can exercise this path without writing
|
||||
# to the real ledger; operators should never set it.
|
||||
"${PYTHON}" "${CONF_ROOT}/lib/record_run.py" \
|
||||
--run-dir "${RUN_DIR}" --env "${CONF_ENV:-dev}" \
|
||||
--ledger "${CONF_LEDGER:-${CONF_ROOT}/runs/ledger.jsonl}" || true
|
||||
log "RECORD: run rc=${RUN_RC}. Commit conformance/runs/${RUN_ID}/ WITH its run.log and"
|
||||
log "RECORD: conformance/runs/ledger.jsonl — an uncommitted ledger proves nothing."
|
||||
exit "${RUN_RC}"
|
||||
fi
|
||||
|
||||
# --- Coverage report is always produced (cluster-independent, R-TEST-04) --------------
|
||||
gen_coverage() {
|
||||
# Pass --allow-gap only when there IS an exemption: an empty DOCUMENTED_GAPS means no
|
||||
|
||||
Whitespace-only changes.
@@ -12,7 +12,15 @@ kind: Kustomization
|
||||
resources:
|
||||
# ---- Phase 2 — Platform Core: Network, Storage, TLS, DNS, Ingress ----
|
||||
- kyverno # admission enforcer FIRST — policies present before other components reconcile
|
||||
- cilium # CNI (adopts the running Ansible-installed release; parity verified)
|
||||
# cilium (the CNI) is DELIBERATELY NOT here: it now reconciles in its OWN Flux Kustomization
|
||||
# (clusters/base/cilium.yaml, dependsOn flux-system, wait:true), and `platform` dependsOn IT.
|
||||
# Not the headlamp/oauth2-proxy rationale (a leaf that must not gate) but its mirror image:
|
||||
# the CNI is the one component that SHOULD gate everything, and inside this shared tier its
|
||||
# health was indistinguishable from eleven unrelated components, the only emergency valve was
|
||||
# freezing the whole tier, and the tier's 10m/5m interval/timeout are wrong for a 3-node
|
||||
# DaemonSet rollout. Gating is preserved via the explicit `platform.dependsOn: [cilium]` edge.
|
||||
# NOTE: platform/cilium/namespace.yaml is the ONLY declaration of Namespace/kube-system in
|
||||
# this tree — that is why the move required a prune-disabled transition. See DECISIONS-LOG.
|
||||
- longhorn # replicated block storage / CSI
|
||||
- external-snapshotter # CSI snapshot CRDs + snapshot-controller (VolumeSnapshot API; CDI smart-clone)
|
||||
- cert-manager # Let's Encrypt DNS-01 wildcard — needs SOPS `cloudflare-api-token`
|
||||
|
||||
Reference in new issue
Block a user