From e7a567db0b440ff87fe3a92a1f3c395c396e0130 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 13:12:07 -0500 Subject: [PATCH 1/3] =?UTF-8?q?WIP=20conformance(egress-class):=20ADR-0015?= =?UTF-8?q?=20assertions=20(partial=20=E2=80=94=20killswitch=20test=20TODO?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session-limit interrupted the WS-TEST agent. egress-class test updated; vpn-killswitch test + host fault-injection fixture still outstanding. Preserved for review/continuation. Co-Authored-By: Claude Opus 4.8 --- .../egress-class/chainsaw-test.yaml | 166 +++++++++++++++--- 1 file changed, 137 insertions(+), 29 deletions(-) diff --git a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml index 5c86475..b8ffb14 100644 --- a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml @@ -1,13 +1,24 @@ # ============================================================================= -# L2 BEHAVIORAL SCAFFOLD — Egress class (R-NET-05, R-NET-07) +# L2 BEHAVIORAL — Egress class (R-NET-05, R-NET-07) [RUNNABLE] # Catalog: docs/14 "Egress class (R-NET-05)" + "Egress flip (R-NET-07)" +# Spec: ADR-0015 (node-level tunnel steered by Cilium Egress Gateway), +# docs/03 "networking isolation & egress". # -# GIVEN one `egress: internet` and one `egress: vpn` workload, +# GIVEN one `egress-class: internet` and one `egress-class: vpn` workload, # WHEN each queries an external echo of its own public IP, -# THEN internet shows the home/WAN IP and vpn shows the VPN (PIA) IP. -# AND (R-NET-07) flipping the class in git re-routes with no workload rewrite. +# THEN the two public IPs are BOTH reachable and DIFFER — internet-class egresses +# the home/WAN stack, vpn-class is SNATed out the gateway node's tun0 to the +# PIA egress (R-NET-05). +# AND (R-NET-07) flipping only the `egress-class` label on the SAME pod re-routes +# its egress via the Cilium Egress Gateway policy with NO workload rewrite — +# same pod (metadata.uid unchanged), new egress path. # -# STATUS: scaffold — needs T-3.1/T-3.2 (egress gateway + PIA/Gluetun). +# ADR-0015 realizes `egress-class=vpn` with a HOST OpenVPN split-tunnel on the +# egress-gateway node (systemd `vpn-egress-tunnel`, iface tun0) selected by a +# CiliumEgressGatewayPolicy (platform/vpn-egress/egress-steering.yaml). There is NO +# in-cluster gluetun pod (superseded design). This test asserts BEHAVIOR (observed +# public IP), never the existence of the policy, and is IP-agnostic: it asserts the +# two classes DIFFER rather than hardcoding today's WAN/PIA addresses. # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -20,43 +31,140 @@ metadata: conformance.home.k3s/requirements: "R-NET-05, R-NET-07" conformance.home.k3s/layer: "L2" conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - internet-class egress shows WAN IP; vpn-class shows VPN IP; git flip re-routes. + internet-class egress and vpn-class egress reach different public IPs (WAN vs + PIA); flipping only the egress-class label on the same pod re-routes it with no + pod recreation (Cilium Egress Gateway, ADR-0015). spec: steps: + # --------------------------------------------------------------------------- - name: given-two-egress-classes try: - apply: - file: fixtures/internet-and-vpn-pods.yaml # TODO(T-3.1): author fixtures - - name: then-internet-shows-wan-ip + file: fixtures/internet-and-vpn-pods.yaml + # Both are bare pods (restartPolicy: Never) — Ready == Running here. + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-internet } + status: { phase: Running } + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-vpn } + status: { phase: Running } + # --------------------------------------------------------------------------- + - name: then-internet-and-vpn-egress-differ try: - script: - # external echo of public IP; assert it is NOT the known VPN egress range. + # R-NET-05: both classes reach an external public-IP echo, and the two + # observed IPs DIFFER. api.ipify.org echoes the caller's public IP. We + # poll (Cilium may still be programming the egress datapath just after + # the pods go Ready) and compare the two — an IP-agnostic assertion, so + # it stays green as PIA/WAN addresses rotate. `internet` is also the + # node/home WAN path, so it doubles as the "WAN reference" for R-NET-07. content: | - set -euo pipefail - ip=$(kubectl -n "$NAMESPACE" exec egress-internet -- \ - curl -fsS https://api.ipify.org) - echo "internet_ip=$ip" - # TODO: assert ip is in home/WAN set, not the PIA range. + set -eu + ns="$NAMESPACE" + get_ip() { + # echo the pod's public IP, or empty on failure (fail-open probe). + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + inet="" vpn="" + for i in $(seq 1 12); do + [ -n "$inet" ] || inet="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$inet" ] && [ -n "$vpn" ]; then + if [ "$inet" != "$vpn" ]; then break; fi + # Both resolved but equal — egress-gateway steering may not have + # converged yet; clear vpn and re-poll within the bounded window. + vpn="" + fi + sleep 5 + done + echo "internet_ip=$inet" + echo "vpn_ip=$vpn" + [ -n "$inet" ] || { echo "FAIL: internet-class egress never returned a public IP" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress never returned a public IP" >&2; exit 1; } + if [ "$inet" = "$vpn" ]; then + echo "FAIL: vpn-class egress IP ($vpn) equals internet/WAN IP — NOT steered through the tunnel" >&2 + exit 1 + fi + echo "EGRESS_CLASSES_DIFFER_OK" check: - (contains($stdout, 'internet_ip=')): true - - name: then-vpn-shows-vpn-ip + (contains($stdout, 'EGRESS_CLASSES_DIFFER_OK')): true + # --------------------------------------------------------------------------- + - name: when-flip-egress-class-then-rerouted-no-recreation try: - script: + # R-NET-07: change ONLY the `egress-class` label on the vpn pod + # (internet <- vpn), the exact declarative change a git/Flux edit makes. + # Cilium re-evaluates the CiliumEgressGatewayPolicy podSelector and stops + # steering this pod through tun0, so its public IP flips to the WAN path. + # The SAME pod must serve the new path: metadata.uid MUST be unchanged + # (no delete/recreate, no workload rewrite). We restore the label after. content: | - set -euo pipefail - ip=$(kubectl -n "$NAMESPACE" exec egress-vpn -- \ - curl -fsS https://api.ipify.org) - echo "vpn_ip=$ip" - # TODO: assert ip is a PIA egress address (distinct from WAN IP). + set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + uid_before="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')" + wan="$(get_ip egress-internet)" # WAN reference (unchanged internet-class pod) + vpn_before="$(get_ip egress-vpn)" + echo "uid_before=$uid_before wan=$wan vpn_before=$vpn_before" + [ -n "$wan" ] || { echo "FAIL: no WAN reference IP from internet-class pod" >&2; exit 1; } + + # -- The flip: relabel in place (simulates the git-driven class change). -- + kubectl -n "$ns" label pod egress-vpn \ + homelab.keithjackson.dev/egress-class=internet --overwrite + + # Poll (bounded) until Cilium re-programs egress: the pod's public IP + # must become the WAN IP (i.e. no longer routed out tun0). + now="" + for i in $(seq 1 18); do + now="$(get_ip egress-vpn)" + if [ -n "$now" ] && [ "$now" = "$wan" ]; then break; fi + sleep 5 + done + uid_after="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')" + echo "vpn_after=$now uid_after=$uid_after" + + # -- Assertions ----------------------------------------------------- + if [ "$now" != "$wan" ]; then + echo "FAIL: after flip to internet-class, egress IP ($now) != WAN ($wan) — did not re-route" >&2 + exit 1 + fi + if [ "$uid_before" != "$uid_after" ]; then + echo "FAIL: pod was recreated (uid $uid_before -> $uid_after) — flip must NOT rewrite the workload" >&2 + exit 1 + fi + echo "EGRESS_FLIP_REROUTED_SAME_POD_OK" check: - (contains($stdout, 'vpn_ip=')): true - - name: when-flip-egress-class-in-git - try: + (contains($stdout, 'EGRESS_FLIP_REROUTED_SAME_POD_OK')): true + finally: + # Restore the original class regardless of outcome (defensive — the fixture + # is torn down anyway, but keep the pod's declared state consistent if a + # later step is ever added). - script: - # R-NET-07: patch the egress class (simulating a git/Flux change) and - # assert the observed egress path changes with NO pod recreation. content: | - set -euo pipefail - echo "TODO(T-3.4): patch egress class, reconcile, re-check public IP," - echo "assert path changed and pod UID/generation unchanged (no rewrite)." + set +e + kubectl -n "$NAMESPACE" label pod egress-vpn \ + homelab.keithjackson.dev/egress-class=vpn --overwrite + echo "restored egress-class=vpn" + # --------------------------------------------------------------------------- + # Fixtures are deleted by Chainsaw's per-test namespace teardown; this explicit + # finally is belt-and-suspenders so a leaked pod is impossible even if the ns + # deletion is ever disabled. + finally: + - delete: + ref: + apiVersion: v1 + kind: Pod + labels: + app: egress-test From 0e69c63b55faedb9f4cb195783e013279dd585e5 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 13:58:40 -0500 Subject: [PATCH 2/3] conformance: complete vpn-killswitch L3 test + ADR-0015 RBAC/fixtures Turn the vpn-killswitch scaffold into a runnable SAFETY-CRITICAL L3 test for R-NET-06 under ADR-0015 (node-level host OpenVPN tunnel steered by Cilium Egress Gateway with a host nftables kill-switch), replacing the obsolete in-cluster gluetun design. - fixtures/vpn-and-internet-pods.yaml: two egress-class probe pods + documented RBAC/PodSecurity/namespace requirements for node-level fault injection. - vpn-killswitch/chainsaw-test.yaml: GIVEN healthy tunnel (vpn egress != WAN); WHEN the host `vpn-egress-tunnel` systemd unit is STOPPED via a privileged, hostPID, node-pinned nsenter Job THEN vpn-class egress is blackholed (curl fails, never returns the WAN IP = no leak) while internet-class is unaffected; WHEN restarted THEN vpn egress recovers. spec-level `finally` ALWAYS runs an idempotent start Job (self-healing) so a mid-test failure never leaves the gateway tunnel down. - incluster/rbac.yaml: grant the runner least-privilege for the delivered tests (namespaces create/delete/patch for the PSA relabel, pods/services/configmaps/ pods-exec/pods-log, batch/jobs) and replace the gluetun-scale rule with the ADR-0015 host-systemd mechanism note (privilege via PodSecurity, not RBAC). Offline validation: python yaml parse (all docs), `run.sh --lint` (structural), `run.sh --coverage-only --strict-coverage` (no tag/annotation drift; R-NET-05/06/ 07 covered, R-NET-06 has its L3 test), yamllint. No live apply, no cluster run. Co-Authored-By: Claude Opus 4.8 --- conformance/incluster/rbac.yaml | 48 ++- .../vpn-killswitch/chainsaw-test.yaml | 331 ++++++++++++++++-- .../fixtures/vpn-and-internet-pods.yaml | 68 ++++ 3 files changed, 395 insertions(+), 52 deletions(-) create mode 100644 conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml diff --git a/conformance/incluster/rbac.yaml b/conformance/incluster/rbac.yaml index 5ddcdc7..4ebad1e 100644 --- a/conformance/incluster/rbac.yaml +++ b/conformance/incluster/rbac.yaml @@ -47,28 +47,46 @@ metadata: name: conformance-runner labels: app.kubernetes.io/part-of: conformance -rules: [] +rules: # ---- Hermetic fixtures (all L2/L3): create/delete throwaway test namespaces + fixtures - # - apiGroups: [""] - # resources: [namespaces] - # verbs: [create, delete, get, list, watch] - # - apiGroups: [""] - # resources: [pods, services, configmaps, secrets, persistentvolumeclaims] - # verbs: [create, delete, get, list, watch, patch] - # - apiGroups: [""] - # resources: [pods/exec] - # verbs: [create] # probe exec for behavior assertions + - apiGroups: [""] + resources: [namespaces] + # `patch` is required by R-NET-06 (vpn-killswitch): the test relabels its own + # ephemeral namespace to PodSecurity enforce=privileged so it can launch the + # node-pinned privileged fault Job. Scoped to the per-test namespace only. + verbs: [create, delete, get, list, watch, patch] + - apiGroups: [""] + resources: [pods, services, configmaps, secrets, persistentvolumeclaims] + verbs: [create, delete, get, list, watch, patch] + - apiGroups: [""] + resources: [pods/exec] + verbs: [create] # probe exec for behavior assertions + - apiGroups: [""] + resources: [pods/log] + verbs: [get] # read fault/probe Job logs on failure + # ---- Probe / fault Jobs (R-TLS-01/03 tls-probe; R-NET-06 tunnel stop/start) + # `create` cannot be constrained by resourceNames, and the fault Job names are + # fixed per-test (tunnel-stop/tunnel-start/tunnel-start-finally); scoped by the + # in-namespace blast radius of the ephemeral test namespace. + - apiGroups: [batch] + resources: [jobs] + verbs: [create, delete, get, list, watch, patch] # # ---- R-NET-01/02 (default-deny, deny-undeclared): apply/patch/delete allow policies # - apiGroups: [cilium.io] # resources: [ciliumnetworkpolicies] # verbs: [create, delete, get, list, patch] # - # ---- R-NET-06 (VPN kill-switch): force the shared Gluetun gateway down/up - # - apiGroups: [apps] - # resources: [deployments, deployments/scale] - # resourceNames: [gluetun-gateway] # SCOPE to the gateway only - # verbs: [get, patch, update] + # ---- R-NET-06 (VPN kill-switch, ADR-0015): the fault is a HOST systemd action + # (stop/start `vpn-egress-tunnel` on the egress-gateway node), NOT scaling an + # in-cluster gluetun Deployment (superseded design). It is driven by a privileged, + # hostPID, node-pinned Job (nsenter -t 1 -- systemctl ...). That power comes from + # PodSecurity (the test namespace relabeled to enforce=privileged) + the pod's + # securityContext, NOT from RBAC — the fault Job pod needs NO Kubernetes API + # rights. The runner only needs `jobs` create/delete + `namespaces` patch above. + # PREREQUISITE: if a cluster-wide Kyverno disallow-privileged / disallow-host- + # namespaces policy is enforced, the `chainsaw-conf-*` test namespaces MUST be in + # its exclusion set (documented, audited exception — R-SEC-02). # # ---- R-STATE-02 (node loss): cordon/drain/uncordon a node (NOT in WS-TEST scope yet) # - apiGroups: [""] diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml index 8008804..8b70601 100644 --- a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml @@ -1,18 +1,34 @@ # ============================================================================= -# L3 FAULT-INJECTION SCAFFOLD — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL* -# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)"; ADR-0013 (PIA/Gluetun). +# L3 FAULT-INJECTION — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL* [RUNNABLE] +# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)". +# Spec: ADR-0015 (node-level host tunnel + Cilium Egress Gateway + host nftables +# kill-switch), docs/03 "networking isolation & egress" (fail-closed). # -# GIVEN a `vpn`-class workload with a HEALTHY tunnel, -# WHEN the tunnel is FORCED DOWN (fault injection), -# THEN the workload has ZERO egress to the plain internet (fail-closed), and -# `internet`-class workloads are UNAFFECTED; -# WHEN the tunnel recovers, THEN connectivity returns. +# GIVEN a `vpn`-class workload with a HEALTHY host tunnel (its egress is the PIA IP, +# distinct from the WAN), +# WHEN the tunnel is FORCED DOWN — the host `vpn-egress-tunnel` systemd unit on the +# egress-gateway node is STOPPED (fault injection) — +# THEN the vpn-class workload reaches NOTHING on the plain internet: its curl FAILS +# and, critically, does NOT return the WAN IP (a WAN IP = LEAK = test FAILS). +# The host nftables kill-switch blackholes egress-class=vpn traffic while tun0 +# is absent (ADR-0015 invariant 3/4: vpn fails CLOSED). +# AND the `internet`-class workload is UNAFFECTED (invariant 3: internet fails OPEN). +# WHEN the tunnel RECOVERS (service restarted), THEN vpn-class egress returns (PIA IP). # -# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check -# does not satisfy R-NET-06. `finally` ALWAYS restores the tunnel so a failure -# mid-test never leaves the gateway down (self-healing fixture). +# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check does +# not satisfy R-NET-06. # -# STATUS: scaffold — needs T-3.0 (fault harness) + T-3.2 (Gluetun gateway). +# FAULT MECHANISM (ADR-0015): the tunnel is HOST infra, not a pod, so the fault is a +# host systemd action. A Chainsaw script runs in-cluster, so stopping a host service +# needs a privileged, node-pinned pod: a Job with nodeSelector egress-gateway=true, +# hostPID: true, a privileged container running +# nsenter -t 1 -m -u -i -n -p -- systemctl stop|start vpn-egress-tunnel +# (enters PID 1's namespaces to drive the host systemctl). See +# fixtures/vpn-and-internet-pods.yaml for the RBAC / PodSecurity / namespace notes. +# +# SELF-HEALING: the spec-level `finally` ALWAYS runs a `systemctl start` Job so a +# failure mid-test NEVER leaves the gateway tunnel down (idempotent — start on an +# already-running unit is a no-op success). # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -24,49 +40,290 @@ metadata: conformance.home.k3s/requirements: "R-NET-06" conformance.home.k3s/layer: "L3" conformance.home.k3s/safety-critical: "true" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - Tunnel down => vpn workload leaks nothing; internet workloads unaffected; - recovery restores connectivity. + Stopping the host vpn-egress-tunnel service blackholes vpn-class egress (no + leak, no WAN IP) while internet-class is unaffected; restarting recovers it. spec: + timeouts: + apply: 90s + assert: 180s + exec: 180s steps: + # --------------------------------------------------------------------------- - name: given-healthy-tunnel-and-two-workloads try: - - apply: - file: fixtures/vpn-and-internet-pods.yaml # TODO(T-3.2): author + # The fault Jobs below need privileged + hostPID, which the platform-default + # `restricted` PSA on the ephemeral test namespace forbids. Relabel THIS + # per-test namespace only (torn down with it) so the blast radius is scoped. - script: - content: 'echo "TODO: assert tunnel healthy AND vpn workload egress works"' + content: | + set -eu + kubectl label namespace "$NAMESPACE" \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/warn=privileged \ + pod-security.kubernetes.io/audit=privileged \ + --overwrite + echo "NS_PRIVILEGED_OK" + check: + (contains($stdout, 'NS_PRIVILEGED_OK')): true + - apply: + file: fixtures/vpn-and-internet-pods.yaml + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-internet } + status: { phase: Running } + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-vpn } + status: { phase: Running } + - script: + # Baseline: tunnel HEALTHY. vpn-class egress works AND differs from WAN + # (proves it is steered through tun0, not leaking). Record the WAN IP for + # the leak check below. IP-agnostic (compares classes, no hardcoded IPs). + content: | + set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + wan="" vpn="" + for i in $(seq 1 12); do + [ -n "$wan" ] || wan="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi + [ "$wan" = "$vpn" ] && vpn="" # not steered yet; re-poll + sleep 5 + done + echo "wan=$wan vpn=$vpn" + [ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; } + [ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; } + echo "TUNNEL_HEALTHY_OK" + check: + (contains($stdout, 'TUNNEL_HEALTHY_OK')): true + # --------------------------------------------------------------------------- - name: when-force-tunnel-down-then-no-leak try: - # FAULT INJECTION: force the shared Gluetun tunnel down (scale/patch/kill). + # -- FAULT INJECTION: stop the host `vpn-egress-tunnel` unit on the gateway -- + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-stop + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 2 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true # see PID 1 to enter its namespaces + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" # pin to gateway node + tolerations: + - operator: Exists # schedule even if the gateway node is tainted + containers: + - name: nsenter + # busybox ships an `nsenter` applet; if a build lacks it, swap + # for a util-linux image (e.g. debian:12-slim + util-linux). + image: busybox:1.37.0 + securityContext: + privileged: true # CAP_SYS_ADMIN for setns() + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - stop + - vpn-egress-tunnel + - assert: + # The stop actually ran on the host (Job completed). + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-stop } + status: { succeeded: 1 } - script: + # THEN: vpn-class egress must reach NOTHING. Poll — ANY successful response + # is a LEAK (with tun0 down the kill-switch blackholes the class), and a + # response equal to the WAN IP is the specific leak we must never see. content: | - set -euo pipefail - echo "TODO(T-3.0): scale gluetun to 0 / drop wg iface (force tunnel DOWN)" - - script: - # THEN: vpn workload must reach NOTHING on the plain internet. - content: | - set -euo pipefail - if kubectl -n "$NAMESPACE" exec egress-vpn -- \ - curl -fsS --max-time 5 https://api.ipify.org; then - echo "LEAK: vpn workload reached internet with tunnel down" >&2; exit 1 - fi - echo NO_LEAK_OK + set -u + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + for i in $(seq 1 8); do + out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$out" ]; then + if [ -n "$wan" ] && [ "$out" = "$wan" ]; then + echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "attempt $i: vpn-class blackholed (no response), good" + sleep 3 + done + echo "NO_LEAK_OK" check: (contains($stdout, 'NO_LEAK_OK')): true - script: - # AND: internet-class workload is unaffected. + # AND: internet-class egress is UNAFFECTED (fails OPEN — kill-switch is + # scoped to vpn-class only, ADR-0015 invariant 3). content: | - set -euo pipefail - kubectl -n "$NAMESPACE" exec egress-internet -- \ - curl -fsS --max-time 5 https://api.ipify.org && echo INTERNET_UNAFFECTED_OK + set -eu + ns="$NAMESPACE" + out="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + echo "internet_ip=$out" + [ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; } + echo "INTERNET_UNAFFECTED_OK" check: (contains($stdout, 'INTERNET_UNAFFECTED_OK')): true + # --------------------------------------------------------------------------- - name: when-tunnel-recovers-connectivity-returns try: + # -- Restart the host tunnel service and prove vpn-class egress comes back. -- + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-start + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 2 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" + tolerations: + - operator: Exists + containers: + - name: nsenter + image: busybox:1.37.0 + securityContext: + privileged: true + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - start + - vpn-egress-tunnel + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-start } + status: { succeeded: 1 } - script: - content: 'echo "TODO(T-3.0): restore gluetun; assert vpn egress returns"' - # ALWAYS restore the tunnel, even if an assertion above failed. - finally: - - script: + # Poll (bounded) until the tunnel reconnects and vpn-class egress returns + # to a NON-WAN (PIA) IP — connectivity restored, still tunneled. content: | - echo "TODO(T-3.0): guarantee gluetun scaled back / wg iface restored" + set -eu + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + vpn="" + for i in $(seq 1 24); do + vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi + vpn="" + sleep 5 + done + echo "recovered_vpn=$vpn wan=$wan" + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; } + [ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; } + echo "TUNNEL_RECOVERED_OK" + check: + (contains($stdout, 'TUNNEL_RECOVERED_OK')): true + # --------------------------------------------------------------------------- + # SELF-HEALING: no matter how any step above ended, ALWAYS start the tunnel so a + # mid-test failure never leaves the gateway down. Idempotent: `systemctl start` on + # an already-running unit is a no-op success. Distinct Job name so it never + # collides with the recovery-step Job. + finally: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-start-finally + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 3 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" + tolerations: + - operator: Exists + containers: + - name: nsenter + image: busybox:1.37.0 + securityContext: + privileged: true + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - start + - vpn-egress-tunnel + # Block until the self-healing start actually completed on the host before the + # namespace (and this Job) is torn down — a leaked-down tunnel is unacceptable. + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-start-finally } + status: { succeeded: 1 } + # Clean up the fault Jobs (belt-and-suspenders; ns teardown also removes them). + - delete: + ref: + apiVersion: batch/v1 + kind: Job + labels: + app: vpn-killswitch-fault diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml new file mode 100644 index 0000000..aa042dc --- /dev/null +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml @@ -0,0 +1,68 @@ +# ============================================================================= +# Fixture for the vpn-killswitch L3 fault-injection test (R-NET-06, ADR-0015). +# +# Two probe workloads — identical to the egress-class fixture — plus the two +# node-pinned PRIVILEGED fault Jobs that STOP / START the host `vpn-egress-tunnel` +# systemd unit on the egress-gateway node. Under ADR-0015 the VPN tunnel is HOST +# infrastructure (OpenVPN split-tunnel on the node), NOT an in-cluster gluetun pod, +# so "force the tunnel down" means stopping a host systemd service — which requires +# entering the host namespaces (nsenter -t 1) from a privileged, hostPID pod pinned +# to that node. +# +# The Jobs are authored here as fixtures (not applied by this file directly) so the +# test can `apply` the stop Job at the fault point and the start Job in `finally`. +# They are pinned to the gateway node via nodeSelector; hostPID + privileged are +# required so nsenter can reach PID 1's mount/uts/ipc/net/pid namespaces and run the +# host `systemctl`. +# +# --------------------------------------------------------------------------------- +# NAMESPACE / PodSecurity / RBAC REQUIREMENTS (see incluster/rbac.yaml): +# * The namespace these Jobs run in MUST permit privileged + hostPID pods. Chainsaw +# runs each test in an ephemeral namespace that inherits the platform default +# (pod-security.kubernetes.io/enforce=restricted), which FORBIDS both. The test +# therefore relabels its OWN ephemeral namespace to `enforce=privileged` BEFORE +# applying these Jobs (and it is torn down with the namespace). This is scoped to +# the single per-test namespace — the platform default stays restricted. +# * If a cluster-wide Kyverno "disallow-privileged"/"disallow-host-namespaces" +# policy is in force, the conformance test namespace (prefix `chainsaw-conf-`) +# MUST be in that policy's exclusion set, or these Jobs are the documented, sole +# sanctioned exception for node-level fault injection (R-SEC-02 audit item). +# * The runner ServiceAccount (conformance-runner) needs: namespaces patch (to set +# the PSA label), and jobs create/delete/get/list/watch (to drive the fault). +# * The Job PODS run under the ephemeral namespace's `default` ServiceAccount and +# need NO Kubernetes API permissions — they act on the HOST via nsenter, not the +# API. Their power comes from PodSecurity (privileged) + hostPID, not RBAC. +# ============================================================================= +apiVersion: v1 +kind: Pod +metadata: + name: egress-internet + labels: + app: egress-test + homelab.keithjackson.dev/egress-class: internet +spec: + containers: + - name: curl + image: curlimages/curl:8.11.1 + command: ["sleep", "3600"] + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { memory: 32Mi } + restartPolicy: Never +--- +apiVersion: v1 +kind: Pod +metadata: + name: egress-vpn + labels: + app: egress-test + homelab.keithjackson.dev/egress-class: vpn +spec: + containers: + - name: curl + image: curlimages/curl:8.11.1 + command: ["sleep", "3600"] + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { memory: 32Mi } + restartPolicy: Never From 4c21f9f6e33563258f85be9cdf129f67c3f3fda6 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 14:09:28 -0500 Subject: [PATCH 3/3] conformance: fix invalid test-level spec.finally for chainsaw v0.2.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chainsaw v0.2.12 has no test-level `spec.finally` (finally exists only on a step), and a step-level `finally` accepts only catch-style ops (script/command/ delete/wait), NOT `apply`/`assert`. Both tests failed to parse ("unknown field"). - egress-class: drop the redundant test-level `spec.finally` delete-pods block — chainsaw auto-deletes applied fixtures on cleanup (skipDelete: false). The valid step-level `finally` that restores the egress-class label is kept. - vpn-killswitch: move the SAFETY-CRITICAL self-healing tunnel restart out of the invalid `spec.finally` and into the STEP-level `finally` of the fault step `when-force-tunnel-down-then-no-leak`, implemented as a `script` (finally can't use apply/assert) that `kubectl apply`s the privileged node-pinned restart Job (name `tunnel-restart`, distinct from `tunnel-stop`) and `kubectl wait`s for it to complete — the equivalent of `assert succeeded:1`, blocking teardown until the host confirms the tunnel is up. This finally runs even if the no-leak / internet-unaffected asserts in the same step's `try` fail, so no path leaves the gateway tunnel down. The recovery step is now poll-only (restart already happened in the prior step's finally). Validation with the pinned binary (~/.local/bin/chainsaw v0.2.12): `chainsaw lint test --file ...` => "The document is valid" for both; `chainsaw test ... --no-cluster` loads both past the parse stage (no "unknown field"), then fails only at runtime for lack of a cluster (expected). yamllint + run.sh --strict-coverage clean (RC=0). No live apply / fault injection, no spec edits. Co-Authored-By: Claude Opus 4.8 --- .../egress-class/chainsaw-test.yaml | 13 +- .../vpn-killswitch/chainsaw-test.yaml | 127 ++++++------------ 2 files changed, 41 insertions(+), 99 deletions(-) diff --git a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml index b8ffb14..d8bf598 100644 --- a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml @@ -158,13 +158,6 @@ spec: homelab.keithjackson.dev/egress-class=vpn --overwrite echo "restored egress-class=vpn" # --------------------------------------------------------------------------- - # Fixtures are deleted by Chainsaw's per-test namespace teardown; this explicit - # finally is belt-and-suspenders so a leaked pod is impossible even if the ns - # deletion is ever disabled. - finally: - - delete: - ref: - apiVersion: v1 - kind: Pod - labels: - app: egress-test + # Fixtures are torn down automatically by Chainsaw's per-test namespace deletion + # (config/chainsaw-config.yaml sets skipDelete: false), so no explicit fixture + # cleanup step is needed — and chainsaw v0.2.12 has no test-level spec.finally. diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml index 8b70601..e89bc3b 100644 --- a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml @@ -196,19 +196,32 @@ spec: echo "INTERNET_UNAFFECTED_OK" check: (contains($stdout, 'INTERNET_UNAFFECTED_OK')): true - # --------------------------------------------------------------------------- - - name: when-tunnel-recovers-connectivity-returns - try: - # -- Restart the host tunnel service and prove vpn-class egress comes back. -- - - apply: - resource: + # -- SELF-HEALING (STEP-LEVEL finally — chainsaw v0.2.12 has NO test-level + # spec.finally; finally lives on the step, and it only accepts catch-style ops + # like `script`, NOT `apply`/`assert`). This runs ALWAYS, even if a `try` + # assertion above FAILED, so a failed no-leak / internet-unaffected assertion + # can never leave the gateway tunnel down. This is the ONLY step that stops the + # tunnel, so restarting it here closes every path to a leaked-down gateway. + # We create the privileged restart Job with kubectl and `kubectl wait` for it — + # the equivalent of an `assert succeeded:1`: teardown blocks until the host + # confirms the unit is back up. Idempotent: `systemctl start` on an already- + # running unit is a no-op success. + finally: + - script: + content: | + set -eu + ns="$NAMESPACE" + # Distinct Job name (tunnel-restart) so it never collides with the + # tunnel-stop Job. Left-aligned manifest so the block scalar yields + # valid YAML at column 0 at runtime. + cat <<'JOB' | kubectl -n "$ns" apply -f - apiVersion: batch/v1 kind: Job metadata: - name: tunnel-start + name: tunnel-restart labels: { app: vpn-killswitch-fault } spec: - backoffLimit: 2 + backoffLimit: 3 activeDeadlineSeconds: 90 ttlSecondsAfterFinished: 600 template: @@ -226,28 +239,22 @@ spec: image: busybox:1.37.0 securityContext: privileged: true - command: - - nsenter - - "-t" - - "1" - - "-m" - - "-u" - - "-i" - - "-n" - - "-p" - - "--" - - systemctl - - start - - vpn-egress-tunnel - - assert: - resource: - apiVersion: batch/v1 - kind: Job - metadata: { name: tunnel-start } - status: { succeeded: 1 } + command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"] + JOB + # Block teardown until the restart actually completed on the host — a + # leaked-down tunnel is unacceptable. `wait` exits non-zero on failure + # or timeout, which (set -e) fails the step and surfaces the problem. + kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart + echo "TUNNEL_SELF_HEAL_OK" + check: + (contains($stdout, 'TUNNEL_SELF_HEAL_OK')): true + # --------------------------------------------------------------------------- + - name: when-tunnel-recovers-connectivity-returns + # The tunnel was already restarted by the previous step's self-healing + # `finally`; here we prove the EFFECT — vpn-class egress returns to a NON-WAN + # (PIA) IP within a bounded window (tunnel reconnect + datapath convergence). + try: - script: - # Poll (bounded) until the tunnel reconnects and vpn-class egress returns - # to a NON-WAN (PIA) IP — connectivity restored, still tunneled. content: | set -eu ns="$NAMESPACE" @@ -267,63 +274,5 @@ spec: echo "TUNNEL_RECOVERED_OK" check: (contains($stdout, 'TUNNEL_RECOVERED_OK')): true - # --------------------------------------------------------------------------- - # SELF-HEALING: no matter how any step above ended, ALWAYS start the tunnel so a - # mid-test failure never leaves the gateway down. Idempotent: `systemctl start` on - # an already-running unit is a no-op success. Distinct Job name so it never - # collides with the recovery-step Job. - finally: - - apply: - resource: - apiVersion: batch/v1 - kind: Job - metadata: - name: tunnel-start-finally - labels: { app: vpn-killswitch-fault } - spec: - backoffLimit: 3 - activeDeadlineSeconds: 90 - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: { app: vpn-killswitch-fault } - spec: - restartPolicy: Never - hostPID: true - nodeSelector: - homelab.keithjackson.dev/egress-gateway: "true" - tolerations: - - operator: Exists - containers: - - name: nsenter - image: busybox:1.37.0 - securityContext: - privileged: true - command: - - nsenter - - "-t" - - "1" - - "-m" - - "-u" - - "-i" - - "-n" - - "-p" - - "--" - - systemctl - - start - - vpn-egress-tunnel - # Block until the self-healing start actually completed on the host before the - # namespace (and this Job) is torn down — a leaked-down tunnel is unacceptable. - - assert: - resource: - apiVersion: batch/v1 - kind: Job - metadata: { name: tunnel-start-finally } - status: { succeeded: 1 } - # Clean up the fault Jobs (belt-and-suspenders; ns teardown also removes them). - - delete: - ref: - apiVersion: batch/v1 - kind: Job - labels: - app: vpn-killswitch-fault + # Fault Jobs + fixtures are torn down automatically by Chainsaw's per-test + # namespace deletion (config/chainsaw-config.yaml: skipDelete: false).