diff --git a/conformance/runs/20260913T014923Z-9d0814a/report.junit.xml b/conformance/runs/20260913T014923Z-9d0814a/report.junit.xml new file mode 100644 index 0000000..d6b8a09 --- /dev/null +++ b/conformance/runs/20260913T014923Z-9d0814a/report.junit.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/conformance/runs/20260913T014923Z-9d0814a/run.log b/conformance/runs/20260913T014923Z-9d0814a/run.log new file mode 100644 index 0000000..66e419f --- /dev/null +++ b/conformance/runs/20260913T014923Z-9d0814a/run.log @@ -0,0 +1,237 @@ +[run.sh] Coverage report first (always) +[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s). +[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json +[run.sh] No L0/L1 offline checks match the selection (layer=l3 reqs='R-NET-06'). +[run.sh] LIGHT pass: 1 test(s), parallel=2 (pinned v0.2.12) +Version: 0.2.12 +Loading config (/work/.claude/conf-clone/conformance/config/chainsaw-config.yaml)... +- Using test file: chainsaw-test +- TestDirs [/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch] +- SkipDelete false +- FailFast false +- ReportFormat 'JUNIT-TEST' +- ReportName 'chainsaw-report-light' +- ReportPath '/work/.claude/conf-clone/conformance/_artifacts' +- Namespace '' +- FullName false +- IncludeTestRegex '' +- ExcludeTestRegex '' +- ApplyTimeout 1m30s +- AssertTimeout 3m0s +- CleanupTimeout 2m0s +- DeleteTimeout 2m0s +- ErrorTimeout 1m0s +- ExecTimeout 3m0s +- DeletionPropagationPolicy Background +- Parallel 2 +- Template true +- NoCluster false +- PauseOnFailure false +Loading tests... +- vpn-killswitch (/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch) +Loading values... +Running tests... +=== RUN chainsaw +=== PAUSE chainsaw +=== CONT chainsaw +=== RUN chainsaw/vpn-killswitch +=== PAUSE chainsaw/vpn-killswitch +=== CONT chainsaw/vpn-killswitch + l.go:52: | 01:49:24 | vpn-killswitch | @chainsaw  | CREATE | OK | v1/Namespace @ chainsaw-beloved-toad + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | TRY | BEGIN | + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + kubectl label namespace "$NAMESPACE" \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/warn=privileged \ + pod-security.kubernetes.io/audit=privileged \ + --overwrite + echo "NS_PRIVILEGED_OK" + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | SCRIPT | LOG | + === STDOUT + namespace/chainsaw-beloved-toad labeled + NS_PRIVILEGED_OK + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | SCRIPT | DONE | + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | APPLY | RUN | v1/Pod @ chainsaw-beloved-toad/egress-internet + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | CREATE | OK | v1/Pod @ chainsaw-beloved-toad/egress-internet + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | APPLY | DONE | v1/Pod @ chainsaw-beloved-toad/egress-internet + l.go:52: | 01:49:24 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | APPLY | RUN | v1/Pod @ chainsaw-beloved-toad/egress-vpn + l.go:52: | 01:49:25 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | CREATE | OK | v1/Pod @ chainsaw-beloved-toad/egress-vpn + l.go:52: | 01:49:25 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | APPLY | DONE | v1/Pod @ chainsaw-beloved-toad/egress-vpn + l.go:52: | 01:49:25 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | ASSERT | RUN | v1/Pod @ chainsaw-beloved-toad/egress-internet + l.go:52: | 01:49:26 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | ASSERT | DONE | v1/Pod @ chainsaw-beloved-toad/egress-internet + l.go:52: | 01:49:26 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | ASSERT | RUN | v1/Pod @ chainsaw-beloved-toad/egress-vpn + l.go:52: | 01:49:26 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | ASSERT | DONE | v1/Pod @ chainsaw-beloved-toad/egress-vpn + l.go:52: | 01:49:26 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + wan="" vpn="" + for i in $(seq 1 12); do + [ -n "$wan" ] || wan="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi + [ "$wan" = "$vpn" ] && vpn="" # not steered yet; re-poll + sleep 5 + done + echo "wan=$wan vpn=$vpn" + [ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; } + [ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; } + echo "TUNNEL_HEALTHY_OK" + l.go:52: | 01:49:27 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | SCRIPT | LOG | + === STDOUT + wan=136.51.115.252 vpn=158.173.241.11 + TUNNEL_HEALTHY_OK + l.go:52: | 01:49:27 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | SCRIPT | DONE | + l.go:52: | 01:49:27 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | TRY | END | + l.go:52: | 01:49:27 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | TRY | BEGIN | + l.go:52: | 01:49:27 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | APPLY | RUN | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop + l.go:52: | 01:49:27 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | CREATE | OK | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop + l.go:52: | 01:49:27 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | APPLY | DONE | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop + l.go:52: | 01:49:27 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | ASSERT | RUN | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop + l.go:52: | 01:49:31 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | ASSERT | DONE | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop + l.go:52: | 01:49:31 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -u + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + for i in $(seq 1 8); do + out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$out" ]; then + if [ -n "$wan" ] && [ "$out" = "$wan" ]; then + echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "attempt $i: vpn-class blackholed (no response), good" + sleep 3 + done + echo "NO_LEAK_OK" + l.go:52: | 01:51:01 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | SCRIPT | LOG | + === STDOUT + attempt 1: vpn-class blackholed (no response), good + attempt 2: vpn-class blackholed (no response), good + attempt 3: vpn-class blackholed (no response), good + attempt 4: vpn-class blackholed (no response), good + attempt 5: vpn-class blackholed (no response), good + attempt 6: vpn-class blackholed (no response), good + attempt 7: vpn-class blackholed (no response), good + attempt 8: vpn-class blackholed (no response), good + NO_LEAK_OK + l.go:52: | 01:51:01 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | SCRIPT | DONE | + l.go:52: | 01:51:01 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + ns="$NAMESPACE" + out="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + echo "internet_ip=$out" + [ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; } + echo "INTERNET_UNAFFECTED_OK" + l.go:52: | 01:51:02 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | SCRIPT | LOG | + === STDOUT + internet_ip=136.51.115.252 + INTERNET_UNAFFECTED_OK + l.go:52: | 01:51:02 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | SCRIPT | DONE | + l.go:52: | 01:51:02 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | TRY | END | + l.go:52: | 01:51:02 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | FINALLY | BEGIN | + l.go:52: | 01:51:02 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + ns="$NAMESPACE" + # Distinct Job name (tunnel-restart) so it never collides with the + # tunnel-stop Job. Left-aligned manifest so the block scalar yields + # valid YAML at column 0 at runtime. + cat <<'JOB' | kubectl -n "$ns" apply -f - + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-restart + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 3 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" + tolerations: + - operator: Exists + containers: + - name: nsenter + image: busybox:1.37.0 + securityContext: + privileged: true + command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"] + JOB + # Block teardown until the restart actually completed on the host — a + # leaked-down tunnel is unacceptable. `wait` exits non-zero on failure + # or timeout, which (set -e) fails the step and surfaces the problem. + kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart + echo "TUNNEL_SELF_HEAL_OK" + l.go:52: | 01:51:05 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | SCRIPT | LOG | + === STDOUT + job.batch/tunnel-restart created + job.batch/tunnel-restart condition met + TUNNEL_SELF_HEAL_OK + l.go:52: | 01:51:05 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | SCRIPT | DONE | + l.go:52: | 01:51:05 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | FINALLY | END | + l.go:52: | 01:51:05 | vpn-killswitch | when-tunnel-recovers-connectivity-returns | TRY | BEGIN | + l.go:52: | 01:51:05 | vpn-killswitch | when-tunnel-recovers-connectivity-returns | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + vpn="" + for i in $(seq 1 24); do + vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi + vpn="" + sleep 5 + done + echo "recovered_vpn=$vpn wan=$wan" + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; } + [ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; } + echo "TUNNEL_RECOVERED_OK" + l.go:52: | 01:51:08 | vpn-killswitch | when-tunnel-recovers-connectivity-returns | SCRIPT | LOG | + === STDOUT + recovered_vpn=158.173.241.11 wan=136.51.115.252 + TUNNEL_RECOVERED_OK + l.go:52: | 01:51:08 | vpn-killswitch | when-tunnel-recovers-connectivity-returns | SCRIPT | DONE | + l.go:52: | 01:51:08 | vpn-killswitch | when-tunnel-recovers-connectivity-returns | TRY | END | + l.go:52: | 01:51:08 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | CLEANUP | BEGIN | + l.go:52: | 01:51:08 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | DELETE | OK | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop + l.go:52: | 01:51:08 | vpn-killswitch | when-force-tunnel-down-then-no-leak  | CLEANUP | END | + l.go:52: | 01:51:08 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | CLEANUP | BEGIN | + l.go:52: | 01:51:08 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | DELETE | OK | v1/Pod @ chainsaw-beloved-toad/egress-vpn + l.go:52: | 01:51:39 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | DELETE | OK | v1/Pod @ chainsaw-beloved-toad/egress-internet + l.go:52: | 01:52:10 | vpn-killswitch | given-healthy-tunnel-and-two-workloads  | CLEANUP | END | + l.go:52: | 01:52:10 | vpn-killswitch | @chainsaw  | CLEANUP | BEGIN | + l.go:52: | 01:52:10 | vpn-killswitch | @chainsaw  | DELETE | OK | v1/Namespace @ chainsaw-beloved-toad + l.go:52: | 01:52:18 | vpn-killswitch | @chainsaw  | CLEANUP | END | +--- PASS: chainsaw (0.00s) + --- PASS: chainsaw/vpn-killswitch (173.47s) +PASS +Tests Summary... +- Passed tests 1 +- Failed tests 0 +- Skipped tests 0 +Done. +[run.sh] HEAVY pass: no heavy tests selected. +merge_junit: merged 1 report(s) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml (tests=1 failures=0 errors=0 skipped=0) +[run.sh] Done. JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md diff --git a/conformance/runs/ledger.jsonl b/conformance/runs/ledger.jsonl index 4d6d4f8..1848693 100644 --- a/conformance/runs/ledger.jsonl +++ b/conformance/runs/ledger.jsonl @@ -2,3 +2,4 @@ {"chainsaw_version": "v0.2.12", "env": "dev", "git_sha": "fdc3265585f4b775c0a6ae3fb33c837b0c643655", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (OFF_CLUSTER_TARGET_OK, BACKUPS_ARE_SCHEDULED_OK, DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-STATE-03"], "run_id": "20260902T001907Z-fdc3265", "sentinel": "DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK", "test": "dr-restore", "ts_utc": "2026-09-02T00:19:07Z"} {"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L1", "note": "L1 offline check exited 0; run_l1_offline.py renders its skips as JUnit , so a green JUnit is truthful without a sentinel", "proof": "proven", "requirements": ["R-NET-05", "R-NET-06", "R-NET-07"], "run_id": "20260913T014321Z-19fc8bc", "sentinel": null, "test": "egress-class-render", "ts_utc": "2026-09-13T01:43:21Z"} {"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6-dirty", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014437Z-19fc8bc", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:44:37Z"} +{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "9d0814ae61e6a0ba64d3789a71f1cd9c985efbf0", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014923Z-9d0814a", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:49:23Z"}