From 4c21f9f6e33563258f85be9cdf129f67c3f3fda6 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 14:09:28 -0500 Subject: [PATCH] conformance: fix invalid test-level spec.finally for chainsaw v0.2.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chainsaw v0.2.12 has no test-level `spec.finally` (finally exists only on a step), and a step-level `finally` accepts only catch-style ops (script/command/ delete/wait), NOT `apply`/`assert`. Both tests failed to parse ("unknown field"). - egress-class: drop the redundant test-level `spec.finally` delete-pods block — chainsaw auto-deletes applied fixtures on cleanup (skipDelete: false). The valid step-level `finally` that restores the egress-class label is kept. - vpn-killswitch: move the SAFETY-CRITICAL self-healing tunnel restart out of the invalid `spec.finally` and into the STEP-level `finally` of the fault step `when-force-tunnel-down-then-no-leak`, implemented as a `script` (finally can't use apply/assert) that `kubectl apply`s the privileged node-pinned restart Job (name `tunnel-restart`, distinct from `tunnel-stop`) and `kubectl wait`s for it to complete — the equivalent of `assert succeeded:1`, blocking teardown until the host confirms the tunnel is up. This finally runs even if the no-leak / internet-unaffected asserts in the same step's `try` fail, so no path leaves the gateway tunnel down. The recovery step is now poll-only (restart already happened in the prior step's finally). Validation with the pinned binary (~/.local/bin/chainsaw v0.2.12): `chainsaw lint test --file ...` => "The document is valid" for both; `chainsaw test ... --no-cluster` loads both past the parse stage (no "unknown field"), then fails only at runtime for lack of a cluster (expected). yamllint + run.sh --strict-coverage clean (RC=0). No live apply / fault injection, no spec edits. Co-Authored-By: Claude Opus 4.8 --- .../egress-class/chainsaw-test.yaml | 13 +- .../vpn-killswitch/chainsaw-test.yaml | 127 ++++++------------ 2 files changed, 41 insertions(+), 99 deletions(-) diff --git a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml index b8ffb14..d8bf598 100644 --- a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml @@ -158,13 +158,6 @@ spec: homelab.keithjackson.dev/egress-class=vpn --overwrite echo "restored egress-class=vpn" # --------------------------------------------------------------------------- - # Fixtures are deleted by Chainsaw's per-test namespace teardown; this explicit - # finally is belt-and-suspenders so a leaked pod is impossible even if the ns - # deletion is ever disabled. - finally: - - delete: - ref: - apiVersion: v1 - kind: Pod - labels: - app: egress-test + # Fixtures are torn down automatically by Chainsaw's per-test namespace deletion + # (config/chainsaw-config.yaml sets skipDelete: false), so no explicit fixture + # cleanup step is needed — and chainsaw v0.2.12 has no test-level spec.finally. diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml index 8b70601..e89bc3b 100644 --- a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml @@ -196,19 +196,32 @@ spec: echo "INTERNET_UNAFFECTED_OK" check: (contains($stdout, 'INTERNET_UNAFFECTED_OK')): true - # --------------------------------------------------------------------------- - - name: when-tunnel-recovers-connectivity-returns - try: - # -- Restart the host tunnel service and prove vpn-class egress comes back. -- - - apply: - resource: + # -- SELF-HEALING (STEP-LEVEL finally — chainsaw v0.2.12 has NO test-level + # spec.finally; finally lives on the step, and it only accepts catch-style ops + # like `script`, NOT `apply`/`assert`). This runs ALWAYS, even if a `try` + # assertion above FAILED, so a failed no-leak / internet-unaffected assertion + # can never leave the gateway tunnel down. This is the ONLY step that stops the + # tunnel, so restarting it here closes every path to a leaked-down gateway. + # We create the privileged restart Job with kubectl and `kubectl wait` for it — + # the equivalent of an `assert succeeded:1`: teardown blocks until the host + # confirms the unit is back up. Idempotent: `systemctl start` on an already- + # running unit is a no-op success. + finally: + - script: + content: | + set -eu + ns="$NAMESPACE" + # Distinct Job name (tunnel-restart) so it never collides with the + # tunnel-stop Job. Left-aligned manifest so the block scalar yields + # valid YAML at column 0 at runtime. + cat <<'JOB' | kubectl -n "$ns" apply -f - apiVersion: batch/v1 kind: Job metadata: - name: tunnel-start + name: tunnel-restart labels: { app: vpn-killswitch-fault } spec: - backoffLimit: 2 + backoffLimit: 3 activeDeadlineSeconds: 90 ttlSecondsAfterFinished: 600 template: @@ -226,28 +239,22 @@ spec: image: busybox:1.37.0 securityContext: privileged: true - command: - - nsenter - - "-t" - - "1" - - "-m" - - "-u" - - "-i" - - "-n" - - "-p" - - "--" - - systemctl - - start - - vpn-egress-tunnel - - assert: - resource: - apiVersion: batch/v1 - kind: Job - metadata: { name: tunnel-start } - status: { succeeded: 1 } + command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"] + JOB + # Block teardown until the restart actually completed on the host — a + # leaked-down tunnel is unacceptable. `wait` exits non-zero on failure + # or timeout, which (set -e) fails the step and surfaces the problem. + kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart + echo "TUNNEL_SELF_HEAL_OK" + check: + (contains($stdout, 'TUNNEL_SELF_HEAL_OK')): true + # --------------------------------------------------------------------------- + - name: when-tunnel-recovers-connectivity-returns + # The tunnel was already restarted by the previous step's self-healing + # `finally`; here we prove the EFFECT — vpn-class egress returns to a NON-WAN + # (PIA) IP within a bounded window (tunnel reconnect + datapath convergence). + try: - script: - # Poll (bounded) until the tunnel reconnects and vpn-class egress returns - # to a NON-WAN (PIA) IP — connectivity restored, still tunneled. content: | set -eu ns="$NAMESPACE" @@ -267,63 +274,5 @@ spec: echo "TUNNEL_RECOVERED_OK" check: (contains($stdout, 'TUNNEL_RECOVERED_OK')): true - # --------------------------------------------------------------------------- - # SELF-HEALING: no matter how any step above ended, ALWAYS start the tunnel so a - # mid-test failure never leaves the gateway down. Idempotent: `systemctl start` on - # an already-running unit is a no-op success. Distinct Job name so it never - # collides with the recovery-step Job. - finally: - - apply: - resource: - apiVersion: batch/v1 - kind: Job - metadata: - name: tunnel-start-finally - labels: { app: vpn-killswitch-fault } - spec: - backoffLimit: 3 - activeDeadlineSeconds: 90 - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: { app: vpn-killswitch-fault } - spec: - restartPolicy: Never - hostPID: true - nodeSelector: - homelab.keithjackson.dev/egress-gateway: "true" - tolerations: - - operator: Exists - containers: - - name: nsenter - image: busybox:1.37.0 - securityContext: - privileged: true - command: - - nsenter - - "-t" - - "1" - - "-m" - - "-u" - - "-i" - - "-n" - - "-p" - - "--" - - systemctl - - start - - vpn-egress-tunnel - # Block until the self-healing start actually completed on the host before the - # namespace (and this Job) is torn down — a leaked-down tunnel is unacceptable. - - assert: - resource: - apiVersion: batch/v1 - kind: Job - metadata: { name: tunnel-start-finally } - status: { succeeded: 1 } - # Clean up the fault Jobs (belt-and-suspenders; ns teardown also removes them). - - delete: - ref: - apiVersion: batch/v1 - kind: Job - labels: - app: vpn-killswitch-fault + # Fault Jobs + fixtures are torn down automatically by Chainsaw's per-test + # namespace deletion (config/chainsaw-config.yaml: skipDelete: false).