diff --git a/conformance/incluster/rbac.yaml b/conformance/incluster/rbac.yaml index 5ddcdc7..4ebad1e 100644 --- a/conformance/incluster/rbac.yaml +++ b/conformance/incluster/rbac.yaml @@ -47,28 +47,46 @@ metadata: name: conformance-runner labels: app.kubernetes.io/part-of: conformance -rules: [] +rules: # ---- Hermetic fixtures (all L2/L3): create/delete throwaway test namespaces + fixtures - # - apiGroups: [""] - # resources: [namespaces] - # verbs: [create, delete, get, list, watch] - # - apiGroups: [""] - # resources: [pods, services, configmaps, secrets, persistentvolumeclaims] - # verbs: [create, delete, get, list, watch, patch] - # - apiGroups: [""] - # resources: [pods/exec] - # verbs: [create] # probe exec for behavior assertions + - apiGroups: [""] + resources: [namespaces] + # `patch` is required by R-NET-06 (vpn-killswitch): the test relabels its own + # ephemeral namespace to PodSecurity enforce=privileged so it can launch the + # node-pinned privileged fault Job. Scoped to the per-test namespace only. + verbs: [create, delete, get, list, watch, patch] + - apiGroups: [""] + resources: [pods, services, configmaps, secrets, persistentvolumeclaims] + verbs: [create, delete, get, list, watch, patch] + - apiGroups: [""] + resources: [pods/exec] + verbs: [create] # probe exec for behavior assertions + - apiGroups: [""] + resources: [pods/log] + verbs: [get] # read fault/probe Job logs on failure + # ---- Probe / fault Jobs (R-TLS-01/03 tls-probe; R-NET-06 tunnel stop/start) + # `create` cannot be constrained by resourceNames, and the fault Job names are + # fixed per-test (tunnel-stop/tunnel-start/tunnel-start-finally); scoped by the + # in-namespace blast radius of the ephemeral test namespace. + - apiGroups: [batch] + resources: [jobs] + verbs: [create, delete, get, list, watch, patch] # # ---- R-NET-01/02 (default-deny, deny-undeclared): apply/patch/delete allow policies # - apiGroups: [cilium.io] # resources: [ciliumnetworkpolicies] # verbs: [create, delete, get, list, patch] # - # ---- R-NET-06 (VPN kill-switch): force the shared Gluetun gateway down/up - # - apiGroups: [apps] - # resources: [deployments, deployments/scale] - # resourceNames: [gluetun-gateway] # SCOPE to the gateway only - # verbs: [get, patch, update] + # ---- R-NET-06 (VPN kill-switch, ADR-0015): the fault is a HOST systemd action + # (stop/start `vpn-egress-tunnel` on the egress-gateway node), NOT scaling an + # in-cluster gluetun Deployment (superseded design). It is driven by a privileged, + # hostPID, node-pinned Job (nsenter -t 1 -- systemctl ...). That power comes from + # PodSecurity (the test namespace relabeled to enforce=privileged) + the pod's + # securityContext, NOT from RBAC — the fault Job pod needs NO Kubernetes API + # rights. The runner only needs `jobs` create/delete + `namespaces` patch above. + # PREREQUISITE: if a cluster-wide Kyverno disallow-privileged / disallow-host- + # namespaces policy is enforced, the `chainsaw-conf-*` test namespaces MUST be in + # its exclusion set (documented, audited exception — R-SEC-02). # # ---- R-STATE-02 (node loss): cordon/drain/uncordon a node (NOT in WS-TEST scope yet) # - apiGroups: [""] diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml index 8008804..8b70601 100644 --- a/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml @@ -1,18 +1,34 @@ # ============================================================================= -# L3 FAULT-INJECTION SCAFFOLD — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL* -# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)"; ADR-0013 (PIA/Gluetun). +# L3 FAULT-INJECTION — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL* [RUNNABLE] +# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)". +# Spec: ADR-0015 (node-level host tunnel + Cilium Egress Gateway + host nftables +# kill-switch), docs/03 "networking isolation & egress" (fail-closed). # -# GIVEN a `vpn`-class workload with a HEALTHY tunnel, -# WHEN the tunnel is FORCED DOWN (fault injection), -# THEN the workload has ZERO egress to the plain internet (fail-closed), and -# `internet`-class workloads are UNAFFECTED; -# WHEN the tunnel recovers, THEN connectivity returns. +# GIVEN a `vpn`-class workload with a HEALTHY host tunnel (its egress is the PIA IP, +# distinct from the WAN), +# WHEN the tunnel is FORCED DOWN — the host `vpn-egress-tunnel` systemd unit on the +# egress-gateway node is STOPPED (fault injection) — +# THEN the vpn-class workload reaches NOTHING on the plain internet: its curl FAILS +# and, critically, does NOT return the WAN IP (a WAN IP = LEAK = test FAILS). +# The host nftables kill-switch blackholes egress-class=vpn traffic while tun0 +# is absent (ADR-0015 invariant 3/4: vpn fails CLOSED). +# AND the `internet`-class workload is UNAFFECTED (invariant 3: internet fails OPEN). +# WHEN the tunnel RECOVERS (service restarted), THEN vpn-class egress returns (PIA IP). # -# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check -# does not satisfy R-NET-06. `finally` ALWAYS restores the tunnel so a failure -# mid-test never leaves the gateway down (self-healing fixture). +# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check does +# not satisfy R-NET-06. # -# STATUS: scaffold — needs T-3.0 (fault harness) + T-3.2 (Gluetun gateway). +# FAULT MECHANISM (ADR-0015): the tunnel is HOST infra, not a pod, so the fault is a +# host systemd action. A Chainsaw script runs in-cluster, so stopping a host service +# needs a privileged, node-pinned pod: a Job with nodeSelector egress-gateway=true, +# hostPID: true, a privileged container running +# nsenter -t 1 -m -u -i -n -p -- systemctl stop|start vpn-egress-tunnel +# (enters PID 1's namespaces to drive the host systemctl). See +# fixtures/vpn-and-internet-pods.yaml for the RBAC / PodSecurity / namespace notes. +# +# SELF-HEALING: the spec-level `finally` ALWAYS runs a `systemctl start` Job so a +# failure mid-test NEVER leaves the gateway tunnel down (idempotent — start on an +# already-running unit is a no-op success). # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -24,49 +40,290 @@ metadata: conformance.home.k3s/requirements: "R-NET-06" conformance.home.k3s/layer: "L3" conformance.home.k3s/safety-critical: "true" + conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- - Tunnel down => vpn workload leaks nothing; internet workloads unaffected; - recovery restores connectivity. + Stopping the host vpn-egress-tunnel service blackholes vpn-class egress (no + leak, no WAN IP) while internet-class is unaffected; restarting recovers it. spec: + timeouts: + apply: 90s + assert: 180s + exec: 180s steps: + # --------------------------------------------------------------------------- - name: given-healthy-tunnel-and-two-workloads try: - - apply: - file: fixtures/vpn-and-internet-pods.yaml # TODO(T-3.2): author + # The fault Jobs below need privileged + hostPID, which the platform-default + # `restricted` PSA on the ephemeral test namespace forbids. Relabel THIS + # per-test namespace only (torn down with it) so the blast radius is scoped. - script: - content: 'echo "TODO: assert tunnel healthy AND vpn workload egress works"' + content: | + set -eu + kubectl label namespace "$NAMESPACE" \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/warn=privileged \ + pod-security.kubernetes.io/audit=privileged \ + --overwrite + echo "NS_PRIVILEGED_OK" + check: + (contains($stdout, 'NS_PRIVILEGED_OK')): true + - apply: + file: fixtures/vpn-and-internet-pods.yaml + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-internet } + status: { phase: Running } + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: { name: egress-vpn } + status: { phase: Running } + - script: + # Baseline: tunnel HEALTHY. vpn-class egress works AND differs from WAN + # (proves it is steered through tun0, not leaking). Record the WAN IP for + # the leak check below. IP-agnostic (compares classes, no hardcoded IPs). + content: | + set -eu + ns="$NAMESPACE" + get_ip() { + kubectl -n "$ns" exec "$1" -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true + } + wan="" vpn="" + for i in $(seq 1 12); do + [ -n "$wan" ] || wan="$(get_ip egress-internet)" + [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)" + if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi + [ "$wan" = "$vpn" ] && vpn="" # not steered yet; re-poll + sleep 5 + done + echo "wan=$wan vpn=$vpn" + [ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; } + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; } + [ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; } + echo "TUNNEL_HEALTHY_OK" + check: + (contains($stdout, 'TUNNEL_HEALTHY_OK')): true + # --------------------------------------------------------------------------- - name: when-force-tunnel-down-then-no-leak try: - # FAULT INJECTION: force the shared Gluetun tunnel down (scale/patch/kill). + # -- FAULT INJECTION: stop the host `vpn-egress-tunnel` unit on the gateway -- + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-stop + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 2 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true # see PID 1 to enter its namespaces + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" # pin to gateway node + tolerations: + - operator: Exists # schedule even if the gateway node is tainted + containers: + - name: nsenter + # busybox ships an `nsenter` applet; if a build lacks it, swap + # for a util-linux image (e.g. debian:12-slim + util-linux). + image: busybox:1.37.0 + securityContext: + privileged: true # CAP_SYS_ADMIN for setns() + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - stop + - vpn-egress-tunnel + - assert: + # The stop actually ran on the host (Job completed). + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-stop } + status: { succeeded: 1 } - script: + # THEN: vpn-class egress must reach NOTHING. Poll — ANY successful response + # is a LEAK (with tun0 down the kill-switch blackholes the class), and a + # response equal to the WAN IP is the specific leak we must never see. content: | - set -euo pipefail - echo "TODO(T-3.0): scale gluetun to 0 / drop wg iface (force tunnel DOWN)" - - script: - # THEN: vpn workload must reach NOTHING on the plain internet. - content: | - set -euo pipefail - if kubectl -n "$NAMESPACE" exec egress-vpn -- \ - curl -fsS --max-time 5 https://api.ipify.org; then - echo "LEAK: vpn workload reached internet with tunnel down" >&2; exit 1 - fi - echo NO_LEAK_OK + set -u + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + for i in $(seq 1 8); do + out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$out" ]; then + if [ -n "$wan" ] && [ "$out" = "$wan" ]; then + echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1 + fi + echo "attempt $i: vpn-class blackholed (no response), good" + sleep 3 + done + echo "NO_LEAK_OK" check: (contains($stdout, 'NO_LEAK_OK')): true - script: - # AND: internet-class workload is unaffected. + # AND: internet-class egress is UNAFFECTED (fails OPEN — kill-switch is + # scoped to vpn-class only, ADR-0015 invariant 3). content: | - set -euo pipefail - kubectl -n "$NAMESPACE" exec egress-internet -- \ - curl -fsS --max-time 5 https://api.ipify.org && echo INTERNET_UNAFFECTED_OK + set -eu + ns="$NAMESPACE" + out="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + echo "internet_ip=$out" + [ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; } + echo "INTERNET_UNAFFECTED_OK" check: (contains($stdout, 'INTERNET_UNAFFECTED_OK')): true + # --------------------------------------------------------------------------- - name: when-tunnel-recovers-connectivity-returns try: + # -- Restart the host tunnel service and prove vpn-class egress comes back. -- + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-start + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 2 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" + tolerations: + - operator: Exists + containers: + - name: nsenter + image: busybox:1.37.0 + securityContext: + privileged: true + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - start + - vpn-egress-tunnel + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-start } + status: { succeeded: 1 } - script: - content: 'echo "TODO(T-3.0): restore gluetun; assert vpn egress returns"' - # ALWAYS restore the tunnel, even if an assertion above failed. - finally: - - script: + # Poll (bounded) until the tunnel reconnects and vpn-class egress returns + # to a NON-WAN (PIA) IP — connectivity restored, still tunneled. content: | - echo "TODO(T-3.0): guarantee gluetun scaled back / wg iface restored" + set -eu + ns="$NAMESPACE" + wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \ + curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)" + vpn="" + for i in $(seq 1 24); do + vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \ + curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)" + if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi + vpn="" + sleep 5 + done + echo "recovered_vpn=$vpn wan=$wan" + [ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; } + [ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; } + echo "TUNNEL_RECOVERED_OK" + check: + (contains($stdout, 'TUNNEL_RECOVERED_OK')): true + # --------------------------------------------------------------------------- + # SELF-HEALING: no matter how any step above ended, ALWAYS start the tunnel so a + # mid-test failure never leaves the gateway down. Idempotent: `systemctl start` on + # an already-running unit is a no-op success. Distinct Job name so it never + # collides with the recovery-step Job. + finally: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: tunnel-start-finally + labels: { app: vpn-killswitch-fault } + spec: + backoffLimit: 3 + activeDeadlineSeconds: 90 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: { app: vpn-killswitch-fault } + spec: + restartPolicy: Never + hostPID: true + nodeSelector: + homelab.keithjackson.dev/egress-gateway: "true" + tolerations: + - operator: Exists + containers: + - name: nsenter + image: busybox:1.37.0 + securityContext: + privileged: true + command: + - nsenter + - "-t" + - "1" + - "-m" + - "-u" + - "-i" + - "-n" + - "-p" + - "--" + - systemctl + - start + - vpn-egress-tunnel + # Block until the self-healing start actually completed on the host before the + # namespace (and this Job) is torn down — a leaked-down tunnel is unacceptable. + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: { name: tunnel-start-finally } + status: { succeeded: 1 } + # Clean up the fault Jobs (belt-and-suspenders; ns teardown also removes them). + - delete: + ref: + apiVersion: batch/v1 + kind: Job + labels: + app: vpn-killswitch-fault diff --git a/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml b/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml new file mode 100644 index 0000000..aa042dc --- /dev/null +++ b/conformance/tests/l3-fault-injection/vpn-killswitch/fixtures/vpn-and-internet-pods.yaml @@ -0,0 +1,68 @@ +# ============================================================================= +# Fixture for the vpn-killswitch L3 fault-injection test (R-NET-06, ADR-0015). +# +# Two probe workloads — identical to the egress-class fixture — plus the two +# node-pinned PRIVILEGED fault Jobs that STOP / START the host `vpn-egress-tunnel` +# systemd unit on the egress-gateway node. Under ADR-0015 the VPN tunnel is HOST +# infrastructure (OpenVPN split-tunnel on the node), NOT an in-cluster gluetun pod, +# so "force the tunnel down" means stopping a host systemd service — which requires +# entering the host namespaces (nsenter -t 1) from a privileged, hostPID pod pinned +# to that node. +# +# The Jobs are authored here as fixtures (not applied by this file directly) so the +# test can `apply` the stop Job at the fault point and the start Job in `finally`. +# They are pinned to the gateway node via nodeSelector; hostPID + privileged are +# required so nsenter can reach PID 1's mount/uts/ipc/net/pid namespaces and run the +# host `systemctl`. +# +# --------------------------------------------------------------------------------- +# NAMESPACE / PodSecurity / RBAC REQUIREMENTS (see incluster/rbac.yaml): +# * The namespace these Jobs run in MUST permit privileged + hostPID pods. Chainsaw +# runs each test in an ephemeral namespace that inherits the platform default +# (pod-security.kubernetes.io/enforce=restricted), which FORBIDS both. The test +# therefore relabels its OWN ephemeral namespace to `enforce=privileged` BEFORE +# applying these Jobs (and it is torn down with the namespace). This is scoped to +# the single per-test namespace — the platform default stays restricted. +# * If a cluster-wide Kyverno "disallow-privileged"/"disallow-host-namespaces" +# policy is in force, the conformance test namespace (prefix `chainsaw-conf-`) +# MUST be in that policy's exclusion set, or these Jobs are the documented, sole +# sanctioned exception for node-level fault injection (R-SEC-02 audit item). +# * The runner ServiceAccount (conformance-runner) needs: namespaces patch (to set +# the PSA label), and jobs create/delete/get/list/watch (to drive the fault). +# * The Job PODS run under the ephemeral namespace's `default` ServiceAccount and +# need NO Kubernetes API permissions — they act on the HOST via nsenter, not the +# API. Their power comes from PodSecurity (privileged) + hostPID, not RBAC. +# ============================================================================= +apiVersion: v1 +kind: Pod +metadata: + name: egress-internet + labels: + app: egress-test + homelab.keithjackson.dev/egress-class: internet +spec: + containers: + - name: curl + image: curlimages/curl:8.11.1 + command: ["sleep", "3600"] + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { memory: 32Mi } + restartPolicy: Never +--- +apiVersion: v1 +kind: Pod +metadata: + name: egress-vpn + labels: + app: egress-test + homelab.keithjackson.dev/egress-class: vpn +spec: + containers: + - name: curl + image: curlimages/curl:8.11.1 + command: ["sleep", "3600"] + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { memory: 32Mi } + restartPolicy: Never