diff --git a/conformance/incluster/rbac.yaml b/conformance/incluster/rbac.yaml index 5ddcdc7..4ebad1e 100644 --- a/conformance/incluster/rbac.yaml +++ b/conformance/incluster/rbac.yaml @@ -47,28 +47,46 @@ metadata: name: conformance-runner labels: app.kubernetes.io/part-of: conformance -rules: [] +rules: # ---- Hermetic fixtures (all L2/L3): create/delete throwaway test namespaces + fixtures - # - apiGroups: [""] - # resources: [namespaces] - # verbs: [create, delete, get, list, watch] - # - apiGroups: [""] - # resources: [pods, services, configmaps, secrets, persistentvolumeclaims] - # verbs: [create, delete, get, list, watch, patch] - # - apiGroups: [""] - # resources: [pods/exec] - # verbs: [create] # probe exec for behavior assertions + - apiGroups: [""] + resources: [namespaces] + # `patch` is required by R-NET-06 (vpn-killswitch): the test relabels its own + # ephemeral namespace to PodSecurity enforce=privileged so it can launch the + # node-pinned privileged fault Job. Scoped to the per-test namespace only. + verbs: [create, delete, get, list, watch, patch] + - apiGroups: [""] + resources: [pods, services, configmaps, secrets, persistentvolumeclaims] + verbs: [create, delete, get, list, watch, patch] + - apiGroups: [""] + resources: [pods/exec] + verbs: [create] # probe exec for behavior assertions + - apiGroups: [""] + resources: [pods/log] + verbs: [get] # read fault/probe Job logs on failure + # ---- Probe / fault Jobs (R-TLS-01/03 tls-probe; R-NET-06 tunnel stop/start) + # `create` cannot be constrained by resourceNames, and the fault Job names are + # fixed per-test (tunnel-stop/tunnel-start/tunnel-start-finally); scoped by the + # in-namespace blast radius of the ephemeral test namespace. + - apiGroups: [batch] + resources: [jobs] + verbs: [create, delete, get, list, watch, patch] # # ---- R-NET-01/02 (default-deny, deny-undeclared): apply/patch/delete allow policies # - apiGroups: [cilium.io] # resources: [ciliumnetworkpolicies] # verbs: [create, delete, get, list, patch] # - # ---- R-NET-06 (VPN kill-switch): force the shared Gluetun gateway down/up - # - apiGroups: [apps] - # resources: [deployments, deployments/scale] - # resourceNames: [gluetun-gateway] # SCOPE to the gateway only - # verbs: [get, patch, update] + # ---- R-NET-06 (VPN kill-switch, ADR-0015): the fault is a HOST systemd action + # (stop/start `vpn-egress-tunnel` on the egress-gateway node), NOT scaling an + # in-cluster gluetun Deployment (superseded design). It is driven by a privileged, + # hostPID, node-pinned Job (nsenter -t 1 -- systemctl ...). That power comes from + # PodSecurity (the test namespace relabeled to enforce=privileged) + the pod's + # securityContext, NOT from RBAC — the fault Job pod needs NO Kubernetes API + # rights. The runner only needs `jobs` create/delete + `namespaces` patch above. + # PREREQUISITE: if a cluster-wide Kyverno disallow-privileged / disallow-host- + # namespaces policy is enforced, the `chainsaw-conf-*` test namespaces MUST be in + # its exclusion set (documented, audited exception — R-SEC-02). # # ---- R-STATE-02 (node loss): cordon/drain/uncordon a node (NOT in WS-TEST scope yet) # - apiGroups: [""] diff --git a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml index 5c86475..d8bf598 100644 --- a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml @@ -1,13 +1,24 @@ # ============================================================================= -# L2 BEHAVIORAL SCAFFOLD — Egress class (R-NET-05, R-NET-07) +# L2 BEHAVIORAL — Egress class (R-NET-05, R-NET-07) [RUNNABLE] # Catalog: docs/14 "Egress class (R-NET-05)" + "Egress flip (R-NET-07)" +# Spec: ADR-0015 (node-level tunnel steered by Cilium Egress Gateway), +# docs/03 "networking isolation & egress". # -# GIVEN one `egress: internet` and one `egress: vpn` workload, +# GIVEN one `egress-class: internet` and one `egress-class: vpn` workload, # WHEN each queries an external echo of its own public IP, -# THEN internet shows the home/WAN IP and vpn shows the VPN (PIA) IP. -# AND (R-NET-07) flipping the class in git re-routes with no workload rewrite. +# THEN the two public IPs are BOTH reachable and DIFFER — internet-class egresses +# the home/WAN stack, vpn-class is SNATed out the gateway node's tun0 to the +# PIA egress (R-NET-05). +# AND (R-NET-07) flipping only the `egress-class` label on the SAME pod re-routes +# its egress via the Cilium Egress Gateway policy with NO workload rewrite — +# same pod (metadata.uid unchanged), new egress path. # -# STATUS: scaffold — needs T-3.1/T-3.2 (egress gateway + PIA/Gluetun). +# ADR-0015 realizes `egress-class=vpn` with a HOST OpenVPN split-tunnel on the +# egress-gateway node (systemd `vpn-egress-tunnel`, iface tun0) selected by a +# CiliumEgressGatewayPolicy (platform/vpn-egress/egress-steering.yaml). There is NO +# in-cluster gluetun pod (superseded design). This test asserts BEHAVIOR (observed +# public IP), never the existence of the policy, and is IP-agnostic: it asserts the +# two classes DIFFER rather than hardcoding today's WAN/PIA addresses. # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -20,43 +31,133 @@ metadata: conformance.home.k3s/requirements: "R-NET-05, R-NET-07" conformance.home.k3s/layer: "L2" conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - internet-class egress shows WAN IP; vpn-class shows VPN IP; git flip re-routes. + internet-class egress and vpn-class egress reach different public IPs (WAN vs + PIA); flipping only the egress-class label on the same pod re-routes it with no + pod recreation (Cilium Egress Gateway, ADR-0015). spec: steps: + # --------------------------------------------------------------------------- - name: given-two-egress-classes try: - apply: - file: fixtures/internet-and-vpn-pods.yaml # TODO(T-3.1): author fixtures - - name: then-internet-shows-wan-ip + file: fixtures/internet-and-vpn-pods.yaml + # Both are bare pods (restartPolicy: Never) — Ready == Running here. + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-internet } + status: { phase: Running } + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-vpn } + status: { phase: Running } + # --------------------------------------------------------------------------- + - name: then-internet-and-vpn-egress-differ try: - script: - # external echo of public IP; assert it is NOT the known VPN egress range. + # R-NET-05: both classes reach an external public-IP echo, and the two + # observed IPs DIFFER. api.ipify.org echoes the caller's public IP. We + # poll (Cilium may still be programming the egress datapath just after + # the pods go Ready) and compare the two — an IP-agnostic assertion, so + # it stays green as PIA/WAN addresses rotate. `internet` is also the + # node/home WAN path, so it doubles as the "WAN reference" for R-NET-07. content: | - set -euo pipefail - ip=$(kubectl -n "$NAMESPACE" exec egress-internet -- \ - curl -fsS https://api.ipify.org) - echo "internet_ip=$ip" - # TODO: assert ip is in home/WAN set, not the PIA range. + set -eu + ns="$NAMESPACE" + get_ip() { + # echo the pod's public IP, or empty on failure (fail-open probe). + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + inet="" vpn="" + for i in $(seq 1 12); do + [ -n "$inet" ] || inet="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$inet" ] && [ -n "$vpn" ]; then + if [ "$inet" != "$vpn" ]; then break; fi + # Both resolved but equal — egress-gateway steering may not have + # converged yet; clear vpn and re-poll within the bounded window. + vpn="" + fi + sleep 5 + done + echo "internet_ip=$inet" + echo "vpn_ip=$vpn" + [ -n "$inet" ] || { echo "FAIL: internet-class egress never returned a public IP" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress never returned a public IP" >&2; exit 1; } + if [ "$inet" = "$vpn" ]; then + echo "FAIL: vpn-class egress IP ($vpn) equals internet/WAN IP — NOT steered through the tunnel" >&2 + exit 1 + fi + echo "EGRESS_CLASSES_DIFFER_OK" check: - (contains($stdout, 'internet_ip=')): true - - name: then-vpn-shows-vpn-ip + (contains($stdout, 'EGRESS_CLASSES_DIFFER_OK')): true + # --------------------------------------------------------------------------- + - name: when-flip-egress-class-then-rerouted-no-recreation try: - script: + # R-NET-07: change ONLY the `egress-class` label on the vpn pod + # (internet <- vpn), the exact declarative change a git/Flux edit makes. + # Cilium re-evaluates the CiliumEgressGatewayPolicy podSelector and stops + # steering this pod through tun0, so its public IP flips to the WAN path. + # The SAME pod must serve the new path: metadata.uid MUST be unchanged + # (no delete/recreate, no workload rewrite). We restore the label after. content: | - set -euo pipefail - ip=$(kubectl -n "$NAMESPACE" exec egress-vpn -- \ - curl -fsS https://api.ipify.org) - echo "vpn_ip=$ip" - # TODO: assert ip is a PIA egress address (distinct from WAN IP). + set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + uid_before="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')" + wan="$(get_ip egress-internet)" # WAN reference (unchanged internet-class pod) + vpn_before="$(get_ip egress-vpn)" + echo "uid_before=$uid_before wan=$wan vpn_before=$vpn_before" + [ -n "$wan" ] || { echo "FAIL: no WAN reference IP from internet-class pod" >&2; exit 1; } + + # -- The flip: relabel in place (simulates the git-driven class change). -- + kubectl -n "$ns" label pod egress-vpn \ + homelab.keithjackson.dev/egress-class=internet --overwrite + + # Poll (bounded) until Cilium re-programs egress: the pod's public IP + # must become the WAN IP (i.e. no longer routed out tun0). + now="" + for i in $(seq 1 18); do + now="$(get_ip egress-vpn)" + if [ -n "$now" ] && [ "$now" = "$wan" ]; then break; fi + sleep 5 + done + uid_after="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')" + echo "vpn_after=$now uid_after=$uid_after" + + # -- Assertions ----------------------------------------------------- + if [ "$now" != "$wan" ]; then + echo "FAIL: after flip to internet-class, egress IP ($now) != WAN ($wan) — did not re-route" >&2 + exit 1 + fi + if [ "$uid_before" != "$uid_after" ]; then + echo "FAIL: pod was recreated (uid $uid_before -> $uid_after) — flip must NOT rewrite the workload" >&2 + exit 1 + fi + echo "EGRESS_FLIP_REROUTED_SAME_POD_OK" check: - (contains($stdout, 'vpn_ip=')): true - - name: when-flip-egress-class-in-git - try: + (contains($stdout, 'EGRESS_FLIP_REROUTED_SAME_POD_OK')): true + finally: + # Restore the original class regardless of outcome (defensive — the fixture + # is torn down anyway, but keep the pod's declared state consistent if a + # later step is ever added). - script: - # R-NET-07: patch the egress class (simulating a git/Flux change) and - # assert the observed egress path changes with NO pod recreation. content: | - set -euo pipefail - echo "TODO(T-3.4): patch egress class, reconcile, re-check public IP," - echo "assert path changed and pod UID/generation unchanged (no rewrite)." + set +e + kubectl -n "$NAMESPACE" label pod egress-vpn \ + homelab.keithjackson.dev/egress-class=vpn --overwrite + echo "restored egress-class=vpn" + # --------------------------------------------------------------------------- + # Fixtures are torn down automatically by Chainsaw's per-test namespace deletion + # (config/chainsaw-config.yaml sets skipDelete: false), so no explicit fixture + # cleanup step is needed — and chainsaw v0.2.12 has no test-level spec.finally. diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml index 8008804..e89bc3b 100644 --- a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml @@ -1,18 +1,34 @@ # ============================================================================= -# L3 FAULT-INJECTION SCAFFOLD — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL* -# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)"; ADR-0013 (PIA/Gluetun). +# L3 FAULT-INJECTION — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL* [RUNNABLE] +# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)". +# Spec: ADR-0015 (node-level host tunnel + Cilium Egress Gateway + host nftables +# kill-switch), docs/03 "networking isolation & egress" (fail-closed). # -# GIVEN a `vpn`-class workload with a HEALTHY tunnel, -# WHEN the tunnel is FORCED DOWN (fault injection), -# THEN the workload has ZERO egress to the plain internet (fail-closed), and -# `internet`-class workloads are UNAFFECTED; -# WHEN the tunnel recovers, THEN connectivity returns. +# GIVEN a `vpn`-class workload with a HEALTHY host tunnel (its egress is the PIA IP, +# distinct from the WAN), +# WHEN the tunnel is FORCED DOWN — the host `vpn-egress-tunnel` systemd unit on the +# egress-gateway node is STOPPED (fault injection) — +# THEN the vpn-class workload reaches NOTHING on the plain internet: its curl FAILS +# and, critically, does NOT return the WAN IP (a WAN IP = LEAK = test FAILS). +# The host nftables kill-switch blackholes egress-class=vpn traffic while tun0 +# is absent (ADR-0015 invariant 3/4: vpn fails CLOSED). +# AND the `internet`-class workload is UNAFFECTED (invariant 3: internet fails OPEN). +# WHEN the tunnel RECOVERS (service restarted), THEN vpn-class egress returns (PIA IP). # -# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check -# does not satisfy R-NET-06. `finally` ALWAYS restores the tunnel so a failure -# mid-test never leaves the gateway down (self-healing fixture). +# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check does +# not satisfy R-NET-06. # -# STATUS: scaffold — needs T-3.0 (fault harness) + T-3.2 (Gluetun gateway). +# FAULT MECHANISM (ADR-0015): the tunnel is HOST infra, not a pod, so the fault is a +# host systemd action. A Chainsaw script runs in-cluster, so stopping a host service +# needs a privileged, node-pinned pod: a Job with nodeSelector egress-gateway=true, +# hostPID: true, a privileged container running +# nsenter -t 1 -m -u -i -n -p -- systemctl stop|start vpn-egress-tunnel +# (enters PID 1's namespaces to drive the host systemctl). See +# fixtures/vpn-and-internet-pods.yaml for the RBAC / PodSecurity / namespace notes. +# +# SELF-HEALING: the spec-level `finally` ALWAYS runs a `systemctl start` Job so a +# failure mid-test NEVER leaves the gateway tunnel down (idempotent — start on an +# already-running unit is a no-op success). # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -24,49 +40,239 @@ metadata: conformance.home.k3s/requirements: "R-NET-06" conformance.home.k3s/layer: "L3" conformance.home.k3s/safety-critical: "true" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - Tunnel down => vpn workload leaks nothing; internet workloads unaffected; - recovery restores connectivity. + Stopping the host vpn-egress-tunnel service blackholes vpn-class egress (no + leak, no WAN IP) while internet-class is unaffected; restarting recovers it. spec: + timeouts: + apply: 90s + assert: 180s + exec: 180s steps: + # --------------------------------------------------------------------------- - name: given-healthy-tunnel-and-two-workloads try: - - apply: - file: fixtures/vpn-and-internet-pods.yaml # TODO(T-3.2): author + # The fault Jobs below need privileged + hostPID, which the platform-default + # `restricted` PSA on the ephemeral test namespace forbids. Relabel THIS + # per-test namespace only (torn down with it) so the blast radius is scoped. - script: - content: 'echo "TODO: assert tunnel healthy AND vpn workload egress works"' + content: | + set -eu + kubectl label namespace "$NAMESPACE" \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/warn=privileged \ + pod-security.kubernetes.io/audit=privileged \ + --overwrite + echo "NS_PRIVILEGED_OK" + check: + (contains($stdout, 'NS_PRIVILEGED_OK')): true + - apply: + file: fixtures/vpn-and-internet-pods.yaml + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-internet } + status: { phase: Running } + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-vpn } + status: { phase: Running } + - script: + # Baseline: tunnel HEALTHY. vpn-class egress works AND differs from WAN + # (proves it is steered through tun0, not leaking). Record the WAN IP for + # the leak check below. IP-agnostic (compares classes, no hardcoded IPs). + content: | + set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + wan="" vpn="" + for i in $(seq 1 12); do + [ -n "$wan" ] || wan="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi + [ "$wan" = "$vpn" ] && vpn="" # not steered yet; re-poll + sleep 5 + done + echo "wan=$wan vpn=$vpn" + [ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; } + [ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; } + echo "TUNNEL_HEALTHY_OK" + check: + (contains($stdout, 'TUNNEL_HEALTHY_OK')): true + # --------------------------------------------------------------------------- - name: when-force-tunnel-down-then-no-leak try: - # FAULT INJECTION: force the shared Gluetun tunnel down (scale/patch/kill). + # -- FAULT INJECTION: stop the host `vpn-egress-tunnel` unit on the gateway -- + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-stop + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 2 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true # see PID 1 to enter its namespaces + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" # pin to gateway node + tolerations: + - operator: Exists # schedule even if the gateway node is tainted + containers: + - name: nsenter + # busybox ships an `nsenter` applet; if a build lacks it, swap + # for a util-linux image (e.g. debian:12-slim + util-linux). + image: busybox:1.37.0 + securityContext: + privileged: true # CAP_SYS_ADMIN for setns() + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - stop + - vpn-egress-tunnel + - assert: + # The stop actually ran on the host (Job completed). + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-stop } + status: { succeeded: 1 } - script: + # THEN: vpn-class egress must reach NOTHING. Poll — ANY successful response + # is a LEAK (with tun0 down the kill-switch blackholes the class), and a + # response equal to the WAN IP is the specific leak we must never see. content: | - set -euo pipefail - echo "TODO(T-3.0): scale gluetun to 0 / drop wg iface (force tunnel DOWN)" - - script: - # THEN: vpn workload must reach NOTHING on the plain internet. - content: | - set -euo pipefail - if kubectl -n "$NAMESPACE" exec egress-vpn -- \ - curl -fsS --max-time 5 https://api.ipify.org; then - echo "LEAK: vpn workload reached internet with tunnel down" >&2; exit 1 - fi - echo NO_LEAK_OK + set -u + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + for i in $(seq 1 8); do + out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$out" ]; then + if [ -n "$wan" ] && [ "$out" = "$wan" ]; then + echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "attempt $i: vpn-class blackholed (no response), good" + sleep 3 + done + echo "NO_LEAK_OK" check: (contains($stdout, 'NO_LEAK_OK')): true - script: - # AND: internet-class workload is unaffected. + # AND: internet-class egress is UNAFFECTED (fails OPEN — kill-switch is + # scoped to vpn-class only, ADR-0015 invariant 3). content: | - set -euo pipefail - kubectl -n "$NAMESPACE" exec egress-internet -- \ - curl -fsS --max-time 5 https://api.ipify.org && echo INTERNET_UNAFFECTED_OK + set -eu + ns="$NAMESPACE" + out="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + echo "internet_ip=$out" + [ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; } + echo "INTERNET_UNAFFECTED_OK" check: (contains($stdout, 'INTERNET_UNAFFECTED_OK')): true - - name: when-tunnel-recovers-connectivity-returns - try: - - script: - content: 'echo "TODO(T-3.0): restore gluetun; assert vpn egress returns"' - # ALWAYS restore the tunnel, even if an assertion above failed. + # -- SELF-HEALING (STEP-LEVEL finally — chainsaw v0.2.12 has NO test-level + # spec.finally; finally lives on the step, and it only accepts catch-style ops + # like `script`, NOT `apply`/`assert`). This runs ALWAYS, even if a `try` + # assertion above FAILED, so a failed no-leak / internet-unaffected assertion + # can never leave the gateway tunnel down. This is the ONLY step that stops the + # tunnel, so restarting it here closes every path to a leaked-down gateway. + # We create the privileged restart Job with kubectl and `kubectl wait` for it — + # the equivalent of an `assert succeeded:1`: teardown blocks until the host + # confirms the unit is back up. Idempotent: `systemctl start` on an already- + # running unit is a no-op success. finally: - script: content: | - echo "TODO(T-3.0): guarantee gluetun scaled back / wg iface restored" + set -eu + ns="$NAMESPACE" + # Distinct Job name (tunnel-restart) so it never collides with the + # tunnel-stop Job. Left-aligned manifest so the block scalar yields + # valid YAML at column 0 at runtime. + cat <<'JOB' | kubectl -n "$ns" apply -f - + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-restart + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 3 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" + tolerations: + - operator: Exists + containers: + - name: nsenter + image: busybox:1.37.0 + securityContext: + privileged: true + command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"] + JOB + # Block teardown until the restart actually completed on the host — a + # leaked-down tunnel is unacceptable. `wait` exits non-zero on failure + # or timeout, which (set -e) fails the step and surfaces the problem. + kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart + echo "TUNNEL_SELF_HEAL_OK" + check: + (contains($stdout, 'TUNNEL_SELF_HEAL_OK')): true + # --------------------------------------------------------------------------- + - name: when-tunnel-recovers-connectivity-returns + # The tunnel was already restarted by the previous step's self-healing + # `finally`; here we prove the EFFECT — vpn-class egress returns to a NON-WAN + # (PIA) IP within a bounded window (tunnel reconnect + datapath convergence). + try: + - script: + content: | + set -eu + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + vpn="" + for i in $(seq 1 24); do + vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi + vpn="" + sleep 5 + done + echo "recovered_vpn=$vpn wan=$wan" + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; } + [ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; } + echo "TUNNEL_RECOVERED_OK" + check: + (contains($stdout, 'TUNNEL_RECOVERED_OK')): true + # Fault Jobs + fixtures are torn down automatically by Chainsaw's per-test + # namespace deletion (config/chainsaw-config.yaml: skipDelete: false). diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml new file mode 100644 index 0000000..aa042dc --- /dev/null +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml @@ -0,0 +1,68 @@ +# ============================================================================= +# Fixture for the vpn-killswitch L3 fault-injection test (R-NET-06, ADR-0015). +# +# Two probe workloads — identical to the egress-class fixture — plus the two +# node-pinned PRIVILEGED fault Jobs that STOP / START the host `vpn-egress-tunnel` +# systemd unit on the egress-gateway node. Under ADR-0015 the VPN tunnel is HOST +# infrastructure (OpenVPN split-tunnel on the node), NOT an in-cluster gluetun pod, +# so "force the tunnel down" means stopping a host systemd service — which requires +# entering the host namespaces (nsenter -t 1) from a privileged, hostPID pod pinned +# to that node. +# +# The Jobs are authored here as fixtures (not applied by this file directly) so the +# test can `apply` the stop Job at the fault point and the start Job in `finally`. +# They are pinned to the gateway node via nodeSelector; hostPID + privileged are +# required so nsenter can reach PID 1's mount/uts/ipc/net/pid namespaces and run the +# host `systemctl`. +# +# --------------------------------------------------------------------------------- +# NAMESPACE / PodSecurity / RBAC REQUIREMENTS (see incluster/rbac.yaml): +# * The namespace these Jobs run in MUST permit privileged + hostPID pods. Chainsaw +# runs each test in an ephemeral namespace that inherits the platform default +# (pod-security.kubernetes.io/enforce=restricted), which FORBIDS both. The test +# therefore relabels its OWN ephemeral namespace to `enforce=privileged` BEFORE +# applying these Jobs (and it is torn down with the namespace). This is scoped to +# the single per-test namespace — the platform default stays restricted. +# * If a cluster-wide Kyverno "disallow-privileged"/"disallow-host-namespaces" +# policy is in force, the conformance test namespace (prefix `chainsaw-conf-`) +# MUST be in that policy's exclusion set, or these Jobs are the documented, sole +# sanctioned exception for node-level fault injection (R-SEC-02 audit item). +# * The runner ServiceAccount (conformance-runner) needs: namespaces patch (to set +# the PSA label), and jobs create/delete/get/list/watch (to drive the fault). +# * The Job PODS run under the ephemeral namespace's `default` ServiceAccount and +# need NO Kubernetes API permissions — they act on the HOST via nsenter, not the +# API. Their power comes from PodSecurity (privileged) + hostPID, not RBAC. +# ============================================================================= +apiVersion: v1 +kind: Pod +metadata: + name: egress-internet + labels: + app: egress-test + homelab.keithjackson.dev/egress-class: internet +spec: + containers: + - name: curl + image: curlimages/curl:8.11.1 + command: ["sleep", "3600"] + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { memory: 32Mi } + restartPolicy: Never +--- +apiVersion: v1 +kind: Pod +metadata: + name: egress-vpn + labels: + app: egress-test + homelab.keithjackson.dev/egress-class: vpn +spec: + containers: + - name: curl + image: curlimages/curl:8.11.1 + command: ["sleep", "3600"] + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { memory: 32Mi } + restartPolicy: Never