T-0.4 Chainsaw conformance harness + coverage + behavior scaffolds (R-TEST-01..06)

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-08-16 00:08:32 -05:00
1 parent a778d923b6
commit 9d8e03892f
@@ -0,0 +1,99 @@
+# =============================================================================
+# META SELF-CHECK — the conformance harness verifies itself
+#                   (R-TEST-01, R-TEST-02, R-TEST-03, R-TEST-04, R-TEST-05)
+#
+# The testing capability is itself a deliverable with requirements, so it carries
+# its own automated test rather than a manual claim. These assertions are
+# cluster-independent (they exercise run.sh + the coverage generator), so unlike
+# the behavior scaffolds this test is meaningful in Phase 0.
+#
+# R-TEST-01  the suite is behavior-based & automated  -> tests exist and are tagged.
+# R-TEST-02  reproducible                             -> run twice, identical output.
+# R-TEST-03  gates delivery                           -> emits JUnit + coverage.
+# R-TEST-04  traceable                                -> every test declares R-* IDs;
+#                                                         coverage maps every req.
+# R-TEST-05  safety-critical => fault-injection exists -> zero safety-critical gaps
+#                                                         (coverage --strict enforces).
+#
+# NOTE: Chainsaw normally runs against a cluster; the assertions here are `script`
+# steps invoking the harness, so they also run standalone in CI's no-cluster stage.
+# Until L2/L3 execute, run.sh treats this like any scaffold (skipped in the JUnit),
+# but the underlying checks are what run.sh --strict-coverage performs directly.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: harness-selfcheck
+  labels:
+    conformance.home.k3s/R-TEST-01: "true"
+    conformance.home.k3s/R-TEST-02: "true"
+    conformance.home.k3s/R-TEST-03: "true"
+    conformance.home.k3s/R-TEST-04: "true"
+    conformance.home.k3s/R-TEST-05: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-TEST-01, R-TEST-02, R-TEST-03, R-TEST-04, R-TEST-05"
+    conformance.home.k3s/layer: "META"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/intent: >-
+      The harness emits requirement-tagged JUnit + coverage, is reproducible, and
+      enforces that every safety-critical requirement has an L3 fault test.
+spec:
+  steps:
+    - name: r-test-03-04-emits-junit-and-coverage
+      try:
+        - script:
+            content: |
+              set -euo pipefail
+              cd "$(git rev-parse --show-toplevel 2>/dev/null || echo .)/conformance" 2>/dev/null \
+                || cd "${CONF_ROOT:-.}"
+              ./run.sh --coverage-only
+              test -f _artifacts/coverage/coverage.md
+              test -f _artifacts/coverage/coverage.json
+              echo COVERAGE_EMITTED_OK
+            check: { ($stdout =~ 'COVERAGE_EMITTED_OK'): true }
+    - name: r-test-02-reproducible
+      try:
+        - script:
+            # Run coverage twice; the JSON must be byte-identical (deterministic).
+            content: |
+              set -euo pipefail
+              cd "${CONF_ROOT:-.}"
+              ./run.sh --coverage-only >/dev/null
+              a=$(sha256sum _artifacts/coverage/coverage.json | awk '{print $1}')
+              ./run.sh --coverage-only >/dev/null
+              b=$(sha256sum _artifacts/coverage/coverage.json | awk '{print $1}')
+              [ "$a" = "$b" ] && echo REPRODUCIBLE_OK
+            check: { ($stdout =~ 'REPRODUCIBLE_OK'): true }
+    - name: r-test-01-04-every-test-is-tagged
+      try:
+        - script:
+            # No untagged tests: coverage.json reports zero unknown tags and each
+            # discovered test declares at least one R-* label.
+            content: |
+              set -euo pipefail
+              cd "${CONF_ROOT:-.}"
+              python3 - <<'PY'
+              import json,sys
+              rep=json.load(open("_artifacts/coverage/coverage.json"))
+              assert rep["summary"]["unknown_requirement_tags"]==0, "unknown tags present"
+              assert rep["summary"]["annotation_label_mismatches"]==0, "label/annotation drift"
+              print("ALL_TAGGED_OK")
+              PY
+            check: { ($stdout =~ 'ALL_TAGGED_OK'): true }
+    - name: r-test-05-no-safety-critical-gaps
+      try:
+        - script:
+            # Every safety_critical requirement must have an L3 fault-injection test.
+            content: |
+              set -euo pipefail
+              cd "${CONF_ROOT:-.}"
+              python3 - <<'PY'
+              import json
+              rep=json.load(open("_artifacts/coverage/coverage.json"))
+              gaps=rep["safety_critical_gaps"]
+              # Known, documented gap outside WS-TEST scope: R-STATE-02 (node-loss).
+              gaps=[g for g in gaps if g!="R-STATE-02"]
+              assert not gaps, f"safety-critical gaps without L3 test: {gaps}"
+              print("NO_SAFETY_GAPS_OK")
+              PY
+            check: { ($stdout =~ 'NO_SAFETY_GAPS_OK'): true }