From e7a567db0b440ff87fe3a92a1f3c395c396e0130 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 13:12:07 -0500 Subject: [PATCH] =?UTF-8?q?WIP=20conformance(egress-class):=20ADR-0015=20a?= =?UTF-8?q?ssertions=20(partial=20=E2=80=94=20killswitch=20test=20TODO)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session-limit interrupted the WS-TEST agent. egress-class test updated; vpn-killswitch test + host fault-injection fixture still outstanding. Preserved for review/continuation. Co-Authored-By: Claude Opus 4.8 --- .../egress-class/chainsaw-test.yaml | 166 +++++++++++++++--- 1 file changed, 137 insertions(+), 29 deletions(-) diff --git a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml index 5c86475..b8ffb14 100644 --- a/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/egress-class/chainsaw-test.yaml @@ -1,13 +1,24 @@ # ============================================================================= -# L2 BEHAVIORAL SCAFFOLD — Egress class (R-NET-05, R-NET-07) +# L2 BEHAVIORAL — Egress class (R-NET-05, R-NET-07) [RUNNABLE] # Catalog: docs/14 "Egress class (R-NET-05)" + "Egress flip (R-NET-07)" +# Spec: ADR-0015 (node-level tunnel steered by Cilium Egress Gateway), +# docs/03 "networking isolation & egress". # -# GIVEN one `egress: internet` and one `egress: vpn` workload, +# GIVEN one `egress-class: internet` and one `egress-class: vpn` workload, # WHEN each queries an external echo of its own public IP, -# THEN internet shows the home/WAN IP and vpn shows the VPN (PIA) IP. -# AND (R-NET-07) flipping the class in git re-routes with no workload rewrite. +# THEN the two public IPs are BOTH reachable and DIFFER — internet-class egresses +# the home/WAN stack, vpn-class is SNATed out the gateway node's tun0 to the +# PIA egress (R-NET-05). +# AND (R-NET-07) flipping only the `egress-class` label on the SAME pod re-routes +# its egress via the Cilium Egress Gateway policy with NO workload rewrite — +# same pod (metadata.uid unchanged), new egress path. # -# STATUS: scaffold — needs T-3.1/T-3.2 (egress gateway + PIA/Gluetun). +# ADR-0015 realizes `egress-class=vpn` with a HOST OpenVPN split-tunnel on the +# egress-gateway node (systemd `vpn-egress-tunnel`, iface tun0) selected by a +# CiliumEgressGatewayPolicy (platform/vpn-egress/egress-steering.yaml). There is NO +# in-cluster gluetun pod (superseded design). This test asserts BEHAVIOR (observed +# public IP), never the existence of the policy, and is IP-agnostic: it asserts the +# two classes DIFFER rather than hardcoding today's WAN/PIA addresses. # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -20,43 +31,140 @@ metadata: conformance.home.k3s/requirements: "R-NET-05, R-NET-07" conformance.home.k3s/layer: "L2" conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - internet-class egress shows WAN IP; vpn-class shows VPN IP; git flip re-routes. + internet-class egress and vpn-class egress reach different public IPs (WAN vs + PIA); flipping only the egress-class label on the same pod re-routes it with no + pod recreation (Cilium Egress Gateway, ADR-0015). spec: steps: + # --------------------------------------------------------------------------- - name: given-two-egress-classes try: - apply: - file: fixtures/internet-and-vpn-pods.yaml # TODO(T-3.1): author fixtures - - name: then-internet-shows-wan-ip + file: fixtures/internet-and-vpn-pods.yaml + # Both are bare pods (restartPolicy: Never) — Ready == Running here. + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-internet } + status: { phase: Running } + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-vpn } + status: { phase: Running } + # --------------------------------------------------------------------------- + - name: then-internet-and-vpn-egress-differ try: - script: - # external echo of public IP; assert it is NOT the known VPN egress range. + # R-NET-05: both classes reach an external public-IP echo, and the two + # observed IPs DIFFER. api.ipify.org echoes the caller's public IP. We + # poll (Cilium may still be programming the egress datapath just after + # the pods go Ready) and compare the two — an IP-agnostic assertion, so + # it stays green as PIA/WAN addresses rotate. `internet` is also the + # node/home WAN path, so it doubles as the "WAN reference" for R-NET-07. content: | - set -euo pipefail - ip=$(kubectl -n "$NAMESPACE" exec egress-internet -- \ - curl -fsS https://api.ipify.org) - echo "internet_ip=$ip" - # TODO: assert ip is in home/WAN set, not the PIA range. + set -eu + ns="$NAMESPACE" + get_ip() { + # echo the pod's public IP, or empty on failure (fail-open probe). + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + inet="" vpn="" + for i in $(seq 1 12); do + [ -n "$inet" ] || inet="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$inet" ] && [ -n "$vpn" ]; then + if [ "$inet" != "$vpn" ]; then break; fi + # Both resolved but equal — egress-gateway steering may not have + # converged yet; clear vpn and re-poll within the bounded window. + vpn="" + fi + sleep 5 + done + echo "internet_ip=$inet" + echo "vpn_ip=$vpn" + [ -n "$inet" ] || { echo "FAIL: internet-class egress never returned a public IP" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress never returned a public IP" >&2; exit 1; } + if [ "$inet" = "$vpn" ]; then + echo "FAIL: vpn-class egress IP ($vpn) equals internet/WAN IP — NOT steered through the tunnel" >&2 + exit 1 + fi + echo "EGRESS_CLASSES_DIFFER_OK" check: - (contains($stdout, 'internet_ip=')): true - - name: then-vpn-shows-vpn-ip + (contains($stdout, 'EGRESS_CLASSES_DIFFER_OK')): true + # --------------------------------------------------------------------------- + - name: when-flip-egress-class-then-rerouted-no-recreation try: - script: + # R-NET-07: change ONLY the `egress-class` label on the vpn pod + # (internet <- vpn), the exact declarative change a git/Flux edit makes. + # Cilium re-evaluates the CiliumEgressGatewayPolicy podSelector and stops + # steering this pod through tun0, so its public IP flips to the WAN path. + # The SAME pod must serve the new path: metadata.uid MUST be unchanged + # (no delete/recreate, no workload rewrite). We restore the label after. content: | - set -euo pipefail - ip=$(kubectl -n "$NAMESPACE" exec egress-vpn -- \ - curl -fsS https://api.ipify.org) - echo "vpn_ip=$ip" - # TODO: assert ip is a PIA egress address (distinct from WAN IP). + set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + uid_before="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')" + wan="$(get_ip egress-internet)" # WAN reference (unchanged internet-class pod) + vpn_before="$(get_ip egress-vpn)" + echo "uid_before=$uid_before wan=$wan vpn_before=$vpn_before" + [ -n "$wan" ] || { echo "FAIL: no WAN reference IP from internet-class pod" >&2; exit 1; } + + # -- The flip: relabel in place (simulates the git-driven class change). -- + kubectl -n "$ns" label pod egress-vpn \ + homelab.keithjackson.dev/egress-class=internet --overwrite + + # Poll (bounded) until Cilium re-programs egress: the pod's public IP + # must become the WAN IP (i.e. no longer routed out tun0). + now="" + for i in $(seq 1 18); do + now="$(get_ip egress-vpn)" + if [ -n "$now" ] && [ "$now" = "$wan" ]; then break; fi + sleep 5 + done + uid_after="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')" + echo "vpn_after=$now uid_after=$uid_after" + + # -- Assertions ----------------------------------------------------- + if [ "$now" != "$wan" ]; then + echo "FAIL: after flip to internet-class, egress IP ($now) != WAN ($wan) — did not re-route" >&2 + exit 1 + fi + if [ "$uid_before" != "$uid_after" ]; then + echo "FAIL: pod was recreated (uid $uid_before -> $uid_after) — flip must NOT rewrite the workload" >&2 + exit 1 + fi + echo "EGRESS_FLIP_REROUTED_SAME_POD_OK" check: - (contains($stdout, 'vpn_ip=')): true - - name: when-flip-egress-class-in-git - try: + (contains($stdout, 'EGRESS_FLIP_REROUTED_SAME_POD_OK')): true + finally: + # Restore the original class regardless of outcome (defensive — the fixture + # is torn down anyway, but keep the pod's declared state consistent if a + # later step is ever added). - script: - # R-NET-07: patch the egress class (simulating a git/Flux change) and - # assert the observed egress path changes with NO pod recreation. content: | - set -euo pipefail - echo "TODO(T-3.4): patch egress class, reconcile, re-check public IP," - echo "assert path changed and pod UID/generation unchanged (no rewrite)." + set +e + kubectl -n "$NAMESPACE" label pod egress-vpn \ + homelab.keithjackson.dev/egress-class=vpn --overwrite + echo "restored egress-class=vpn" + # --------------------------------------------------------------------------- + # Fixtures are deleted by Chainsaw's per-test namespace teardown; this explicit + # finally is belt-and-suspenders so a leaked pod is impossible even if the ns + # deletion is ever disabled. + finally: + - delete: + ref: + apiVersion: v1 + kind: Pod + labels: + app: egress-test