WIP conformance(egress-class): ADR-0015 assertions (partial — killswitch test TODO)
Session-limit interrupted the WS-TEST agent. egress-class test updated; vpn-killswitch test + host fault-injection fixture still outstanding. Preserved for review/continuation. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed
1 parent
e14048e7fe
commit
e7a567db0b
1 file changed
+137
-29
@@ -1,13 +1,24 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL SCAFFOLD — Egress class (R-NET-05, R-NET-07)
|
||||
# L2 BEHAVIORAL — Egress class (R-NET-05, R-NET-07) [RUNNABLE]
|
||||
# Catalog: docs/14 "Egress class (R-NET-05)" + "Egress flip (R-NET-07)"
|
||||
# Spec: ADR-0015 (node-level tunnel steered by Cilium Egress Gateway),
|
||||
# docs/03 "networking isolation & egress".
|
||||
#
|
||||
# GIVEN one `egress: internet` and one `egress: vpn` workload,
|
||||
# GIVEN one `egress-class: internet` and one `egress-class: vpn` workload,
|
||||
# WHEN each queries an external echo of its own public IP,
|
||||
# THEN internet shows the home/WAN IP and vpn shows the VPN (PIA) IP.
|
||||
# AND (R-NET-07) flipping the class in git re-routes with no workload rewrite.
|
||||
# THEN the two public IPs are BOTH reachable and DIFFER — internet-class egresses
|
||||
# the home/WAN stack, vpn-class is SNATed out the gateway node's tun0 to the
|
||||
# PIA egress (R-NET-05).
|
||||
# AND (R-NET-07) flipping only the `egress-class` label on the SAME pod re-routes
|
||||
# its egress via the Cilium Egress Gateway policy with NO workload rewrite —
|
||||
# same pod (metadata.uid unchanged), new egress path.
|
||||
#
|
||||
# STATUS: scaffold — needs T-3.1/T-3.2 (egress gateway + PIA/Gluetun).
|
||||
# ADR-0015 realizes `egress-class=vpn` with a HOST OpenVPN split-tunnel on the
|
||||
# egress-gateway node (systemd `vpn-egress-tunnel`, iface tun0) selected by a
|
||||
# CiliumEgressGatewayPolicy (platform/vpn-egress/egress-steering.yaml). There is NO
|
||||
# in-cluster gluetun pod (superseded design). This test asserts BEHAVIOR (observed
|
||||
# public IP), never the existence of the policy, and is IP-agnostic: it asserts the
|
||||
# two classes DIFFER rather than hardcoding today's WAN/PIA addresses.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
@@ -20,43 +31,140 @@ metadata:
|
||||
conformance.home.k3s/requirements: "R-NET-05, R-NET-07"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
internet-class egress shows WAN IP; vpn-class shows VPN IP; git flip re-routes.
|
||||
internet-class egress and vpn-class egress reach different public IPs (WAN vs
|
||||
PIA); flipping only the egress-class label on the same pod re-routes it with no
|
||||
pod recreation (Cilium Egress Gateway, ADR-0015).
|
||||
spec:
|
||||
steps:
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: given-two-egress-classes
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/internet-and-vpn-pods.yaml # TODO(T-3.1): author fixtures
|
||||
- name: then-internet-shows-wan-ip
|
||||
file: fixtures/internet-and-vpn-pods.yaml
|
||||
# Both are bare pods (restartPolicy: Never) — Ready == Running here.
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: { name: egress-internet }
|
||||
status: { phase: Running }
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: { name: egress-vpn }
|
||||
status: { phase: Running }
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: then-internet-and-vpn-egress-differ
|
||||
try:
|
||||
- script:
|
||||
# external echo of public IP; assert it is NOT the known VPN egress range.
|
||||
# R-NET-05: both classes reach an external public-IP echo, and the two
|
||||
# observed IPs DIFFER. api.ipify.org echoes the caller's public IP. We
|
||||
# poll (Cilium may still be programming the egress datapath just after
|
||||
# the pods go Ready) and compare the two — an IP-agnostic assertion, so
|
||||
# it stays green as PIA/WAN addresses rotate. `internet` is also the
|
||||
# node/home WAN path, so it doubles as the "WAN reference" for R-NET-07.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
ip=$(kubectl -n "$NAMESPACE" exec egress-internet -- \
|
||||
curl -fsS https://api.ipify.org)
|
||||
echo "internet_ip=$ip"
|
||||
# TODO: assert ip is in home/WAN set, not the PIA range.
|
||||
set -eu
|
||||
ns="$NAMESPACE"
|
||||
get_ip() {
|
||||
# echo the pod's public IP, or empty on failure (fail-open probe).
|
||||
kubectl -n "$ns" exec "$1" -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
|
||||
}
|
||||
inet="" vpn=""
|
||||
for i in $(seq 1 12); do
|
||||
[ -n "$inet" ] || inet="$(get_ip egress-internet)"
|
||||
[ -n "$vpn" ] || vpn="$(get_ip egress-vpn)"
|
||||
if [ -n "$inet" ] && [ -n "$vpn" ]; then
|
||||
if [ "$inet" != "$vpn" ]; then break; fi
|
||||
# Both resolved but equal — egress-gateway steering may not have
|
||||
# converged yet; clear vpn and re-poll within the bounded window.
|
||||
vpn=""
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "internet_ip=$inet"
|
||||
echo "vpn_ip=$vpn"
|
||||
[ -n "$inet" ] || { echo "FAIL: internet-class egress never returned a public IP" >&2; exit 1; }
|
||||
[ -n "$vpn" ] || { echo "FAIL: vpn-class egress never returned a public IP" >&2; exit 1; }
|
||||
if [ "$inet" = "$vpn" ]; then
|
||||
echo "FAIL: vpn-class egress IP ($vpn) equals internet/WAN IP — NOT steered through the tunnel" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "EGRESS_CLASSES_DIFFER_OK"
|
||||
check:
|
||||
(contains($stdout, 'internet_ip=')): true
|
||||
- name: then-vpn-shows-vpn-ip
|
||||
(contains($stdout, 'EGRESS_CLASSES_DIFFER_OK')): true
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: when-flip-egress-class-then-rerouted-no-recreation
|
||||
try:
|
||||
- script:
|
||||
# R-NET-07: change ONLY the `egress-class` label on the vpn pod
|
||||
# (internet <- vpn), the exact declarative change a git/Flux edit makes.
|
||||
# Cilium re-evaluates the CiliumEgressGatewayPolicy podSelector and stops
|
||||
# steering this pod through tun0, so its public IP flips to the WAN path.
|
||||
# The SAME pod must serve the new path: metadata.uid MUST be unchanged
|
||||
# (no delete/recreate, no workload rewrite). We restore the label after.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
ip=$(kubectl -n "$NAMESPACE" exec egress-vpn -- \
|
||||
curl -fsS https://api.ipify.org)
|
||||
echo "vpn_ip=$ip"
|
||||
# TODO: assert ip is a PIA egress address (distinct from WAN IP).
|
||||
set -eu
|
||||
ns="$NAMESPACE"
|
||||
get_ip() {
|
||||
kubectl -n "$ns" exec "$1" -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
|
||||
}
|
||||
uid_before="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')"
|
||||
wan="$(get_ip egress-internet)" # WAN reference (unchanged internet-class pod)
|
||||
vpn_before="$(get_ip egress-vpn)"
|
||||
echo "uid_before=$uid_before wan=$wan vpn_before=$vpn_before"
|
||||
[ -n "$wan" ] || { echo "FAIL: no WAN reference IP from internet-class pod" >&2; exit 1; }
|
||||
|
||||
# -- The flip: relabel in place (simulates the git-driven class change). --
|
||||
kubectl -n "$ns" label pod egress-vpn \
|
||||
homelab.keithjackson.dev/egress-class=internet --overwrite
|
||||
|
||||
# Poll (bounded) until Cilium re-programs egress: the pod's public IP
|
||||
# must become the WAN IP (i.e. no longer routed out tun0).
|
||||
now=""
|
||||
for i in $(seq 1 18); do
|
||||
now="$(get_ip egress-vpn)"
|
||||
if [ -n "$now" ] && [ "$now" = "$wan" ]; then break; fi
|
||||
sleep 5
|
||||
done
|
||||
uid_after="$(kubectl -n "$ns" get pod egress-vpn -o jsonpath='{.metadata.uid}')"
|
||||
echo "vpn_after=$now uid_after=$uid_after"
|
||||
|
||||
# -- Assertions -----------------------------------------------------
|
||||
if [ "$now" != "$wan" ]; then
|
||||
echo "FAIL: after flip to internet-class, egress IP ($now) != WAN ($wan) — did not re-route" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$uid_before" != "$uid_after" ]; then
|
||||
echo "FAIL: pod was recreated (uid $uid_before -> $uid_after) — flip must NOT rewrite the workload" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "EGRESS_FLIP_REROUTED_SAME_POD_OK"
|
||||
check:
|
||||
(contains($stdout, 'vpn_ip=')): true
|
||||
- name: when-flip-egress-class-in-git
|
||||
try:
|
||||
(contains($stdout, 'EGRESS_FLIP_REROUTED_SAME_POD_OK')): true
|
||||
finally:
|
||||
# Restore the original class regardless of outcome (defensive — the fixture
|
||||
# is torn down anyway, but keep the pod's declared state consistent if a
|
||||
# later step is ever added).
|
||||
- script:
|
||||
# R-NET-07: patch the egress class (simulating a git/Flux change) and
|
||||
# assert the observed egress path changes with NO pod recreation.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO(T-3.4): patch egress class, reconcile, re-check public IP,"
|
||||
echo "assert path changed and pod UID/generation unchanged (no rewrite)."
|
||||
set +e
|
||||
kubectl -n "$NAMESPACE" label pod egress-vpn \
|
||||
homelab.keithjackson.dev/egress-class=vpn --overwrite
|
||||
echo "restored egress-class=vpn"
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures are deleted by Chainsaw's per-test namespace teardown; this explicit
|
||||
# finally is belt-and-suspenders so a leaked pod is impossible even if the ns
|
||||
# deletion is ever disabled.
|
||||
finally:
|
||||
- delete:
|
||||
ref:
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
labels:
|
||||
app: egress-test
|
||||
Reference in new issue
Block a user