# ============================================================================= # L2 BEHAVIORAL — the cluster under test IS an HA k3s control plane (R-PLAT-01) [RUNNABLE] # Catalog: docs/14 L2 row ("real runtime behavior, probes against a cluster") # Spec: docs/00 R-PLAT-01 ("The platform MUST run on k3s across 3 nodes with # production-grade HA control plane in prod; dev/test MAY run fewer nodes but MUST # exercise the same multi-node topology.") # docs/01 "Kubernetes distro | **k3s** (HA, embedded etcd) | R-PLAT-01/02" # docs/02 table: "Control plane | HA embedded etcd (k3s server on all 3) | HA # embedded etcd (k3s server on all 3)" # "Node count | 3 VMs (mirror prod topology) | 3 physical nodes" # docs/02 §"k3s topology decisions": "HA control plane with **embedded etcd** across # all 3 nodes (no external datastore)"; "Nodes are both control-plane and workload # (3-node converged cluster)". # # WHY L2 — AND WHAT L0 ALREADY DID # -------------------------------- # tests/l0-static/ha-control-plane-as-code (same requirement label) proves the AUTHORED # half offline: the repo builds a 3-server embedded-etcd control plane in both environments, # one founder, no external datastore, a leader-elected control-plane-only VIP in the serving # cert's SANs. It cannot prove that any of it is TRUE OF A RUNNING CLUSTER — an inventory # entry is not a node, and a templated DaemonSet is not a floating VIP. That is this test: # every assertion below is read from the cluster the suite is pointed at. # # ENVIRONMENT-AGNOSTIC (R-ENV-02): no address, hostname, node count or range is written # here. The expected node count comes from the cluster's OWN declared identity # (flux-system/cluster-vars CLUSTER_ENV, via a test-level chainsaw binding — the same # ConfigMap Flux substitutes from); the control-plane VIP is read out of the kube-vip # DaemonSet the cluster is actually running; the server addresses come from the Node # objects; and the LoadBalancer pool the VIP must stay outside of is read from cluster-vars # too. R-PLAT-01 itself distinguishes prod from dev/test ("3 nodes ... in prod; dev/test MAY # run fewer"), so the assertion is necessarily environment-AWARE — but it takes the # environment from the cluster, never from a flag or a literal, which is precisely the # mechanism the tests/README.md convention prescribes. # # GIVEN the cluster this suite is pointed at, # WHEN its Nodes, its kube-vip DaemonSet and its kube-apiserver endpoints are inspected, # THEN DISTRO: every node's kubelet is a k3s build (`+k3s`) — the distro really is k3s; # SERVERS: every Ready node is BOTH control-plane AND etcd (the converged 3-node # topology docs/02 fixes — no agent-only node, no control-plane-only node), # the count is ODD and quorum-capable (tolerates >= 1 loss), and it is # EXACTLY three when the cluster declares itself prod; # ETCD: every one of those nodes is a REGISTERED embedded-etcd member (k3s stamps # `etcd.k3s.cattle.io/node-name` on a node that joined the etcd cluster), so # the datastore is embedded and replicated across all of them; # VIP: kube-vip is Ready on EVERY control-plane node, is control-plane-only # (svc_enable=false — it must not contend with Cilium LB-IPAM for ARP), and # the address it floats is neither a node address, nor the ingress Gateway # VIP, nor inside the Cilium LoadBalancer pool; # ENDPOINT: the kube-apiserver answers `/readyz` THROUGH THAT VIP over TLS that # validates against the cluster CA for the VIP itself — i.e. the VIP really # fronts the API and the serving cert really carries it; # EVERY SERVER SERVES: the apiserver also answers `/readyz` on EVERY server's own # address, again with a CA-valid cert for that address. This is the assertion # that separates a real HA control plane from one server plus two joiners: # if only the founder ran an apiserver, the other addresses would refuse. # # THE PROBE RUNS IN-CLUSTER, DELIBERATELY. An earlier draft dialled the VIP and the node # addresses from the RUNNER. That works from a host on the node LAN (verified against prod) # and FAILS against dev, whose nodes sit behind a NAT the runner reaches only via a # forwarded port — i.e. it would have made the test a property of where it was launched # from. A Job on the cluster is on the node network in both environments, so the probe is # the same probe everywhere. The Job is created imperatively (kubectl heredoc) rather than # as a fixture file because its address list is DISCOVERED from the cluster at run time — # writing it into a fixture is exactly the per-environment literal R-ENV-02 forbids. # # WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them): # - FAILOVER / SURVIVAL. Nothing here removes a node. That etcd keeps quorum, that the VIP # migrates, and that the API stays reachable when a server dies is an L3 fault-injection # claim and is proven NOWHERE in this suite yet (tests/l3-fault-injection/node-loss # drains a node for R-STATE-02's data-availability claim and asserts nothing about the # control plane). This is the largest remaining R-PLAT-01 gap. # - THE NEXT environment. That the repo would build an HA control plane again is the L0 # half (tests/l0-static/ha-control-plane-as-code). # - "PRODUCTION-GRADE" as a capacity/performance judgement (operator's, docs/02 + §2 of # infra/PROD-PROVISIONING.md). # - THAT THE RUNNING cilium-agents ACTUALLY DIAL THIS VIP. As of 2026-09-06 the repository # authors `K3S_API_HOST` = the control-plane VIP in both environments (asserted at L0 by # tests/l0-static/ha-control-plane-as-code, on both sides of the [parity] contract), but # this test does not read the agents' live `KUBERNETES_SERVICE_HOST`. Until the operator # applies that change the LIVE agents still carry the PRIMARY NODE's address — measured # 2026-09-06: all three prod agents on 192.168.2.10 while kube-vip held 192.168.2.9. An # in-cluster assertion on the agents' endpoint belongs here and is NOT yet written; it is # the obvious follow-up once the change is applied and would catch drift the L0 check # cannot see (a hand-edited DaemonSet, a Helm value overridden out of band). # - etcd health beyond membership registration (raft leader, DB size, alarms) — k3s exposes # it only on the node-local etcd metrics port, which is not reachable through the # Kubernetes API this test is allowed to use. # # CLEANUP: the probe Job lives in chainsaw's per-test namespace and is deleted with it # (config/chainsaw-config.yaml skipDelete: false). Everything else is read-only: this test # creates nothing outside its own namespace and mutates no platform object. # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: name: ha-control-plane labels: conformance.home.k3s/R-PLAT-01: "true" annotations: conformance.home.k3s/requirements: "R-PLAT-01" conformance.home.k3s/layer: "L2" conformance.home.k3s/safety-critical: "false" conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- Given the cluster under test, when its Nodes, kube-vip DaemonSet and apiserver endpoints are probed, then every node is a k3s server that is both control-plane and a registered embedded-etcd member, the count is odd and quorum-capable (exactly three when the cluster declares itself prod), kube-vip floats a control-plane-only VIP that collides with no allocator, and the apiserver answers /readyz through that VIP AND on every server's own address with a CA-valid cert. spec: timeouts: apply: 60s assert: 2m exec: 3m # -- R-ENV-02: the environment's identity and its address allocations are READ FROM THE # CLUSTER UNDER TEST. flux-system/cluster-vars is the ConfigMap Flux itself substitutes # from, so these are the same values the platform was rendered with. Test-level, so # they are in scope for every step (step outputs/bindings are step-scoped). bindings: - name: clusterEnv value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.CLUSTER_ENV) - name: gatewayVip value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.GATEWAY_VIP) - name: lbPoolStart value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.CILIUM_LB_POOL_START) - name: lbPoolStop value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.CILIUM_LB_POOL_STOP) steps: # ----------------------------------------------------------------------------------- - name: then-the-node-set-is-a-quorum-capable-converged-k3s-control-plane description: >- Read the Node objects: k3s on every node, every node both control-plane and a registered embedded-etcd member, an odd quorum-capable count, exactly three in prod. try: - script: env: - name: CLUSTER_ENV value: ($clusterEnv) content: | set -eu fail() { echo "FAIL: $*" >&2; exit 1; } total="$(kubectl get nodes -o name | grep -c . || true)" [ "$total" -ge 1 ] || fail "the cluster reports NO nodes at all" # --- k3s, on every node (R-PLAT-01 names the distro; docs/01 ADR-0001) ----- # A non-k3s kubelet here would mean the platform is not running on k3s. notk3s="$(kubectl get nodes \ -o jsonpath='{range .items[*]}{.metadata.name}{"="}{.status.nodeInfo.kubeletVersion}{"\n"}{end}' \ | grep -v '+k3s' | grep -c . || true)" [ "$notk3s" = "0" ] \ || fail "$notk3s node(s) do not run a k3s kubelet — R-PLAT-01 requires the platform to run on k3s" # --- every node is BOTH control-plane AND etcd (docs/02 converged topology) - # Counted by label EXISTENCE, so it does not matter whether k3s writes "true" # or "". A control-plane node without the etcd role would be a server with an # external/absent datastore; an etcd node without control-plane would be an # isolated datastore member. Both are excluded by requiring all three counts # to be equal. cp="$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o name | grep -c . || true)" etcd="$(kubectl get nodes -l node-role.kubernetes.io/etcd -o name | grep -c . || true)" echo "nodes_total=$total control_plane=$cp etcd=$etcd cluster_env=${CLUSTER_ENV:-}" [ "$cp" = "$total" ] \ || fail "$cp of $total nodes carry node-role.kubernetes.io/control-plane — docs/02 fixes a converged cluster where every node is a control-plane member ('k3s server on all 3')" [ "$etcd" = "$total" ] \ || fail "$etcd of $total nodes carry node-role.kubernetes.io/etcd — docs/02 requires embedded etcd 'across all 3 nodes (no external datastore)'" # --- every node is a REGISTERED embedded-etcd member ----------------------- # k3s stamps etcd.k3s.cattle.io/node-name on a node that actually joined the # embedded-etcd cluster. The role label alone is a label; this annotation is # the datastore's own record of membership. members="$(kubectl get nodes \ -o jsonpath='{range .items[*]}{.metadata.annotations.etcd\.k3s\.cattle\.io/node-name}{"\n"}{end}' \ | grep -c . || true)" [ "$members" = "$total" ] \ || fail "$members of $total nodes are registered embedded-etcd members (etcd.k3s.cattle.io/node-name) — the datastore is not replicated across every server" # --- Ready --------------------------------------------------------------- notready="$(kubectl get nodes \ -o jsonpath='{range .items[*]}{.metadata.name}{"="}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \ | grep -v '=True$' | grep -c . || true)" [ "$notready" = "0" ] \ || fail "$notready node(s) are not Ready — a control plane missing a member is not the HA topology under test" # --- quorum arithmetic --------------------------------------------------- # Embedded etcd needs floor(N/2)+1 members to commit. Fault tolerance is # N - quorum; 'production-grade HA' must survive at least one loss, which # rules out 1 and 2 members. An EVEN count buys no extra tolerance and adds # a tie, so it is also rejected. quorum=$(( total / 2 + 1 )) tolerance=$(( total - quorum )) echo "quorum=$quorum tolerance=$tolerance" [ "$tolerance" -ge 1 ] \ || fail "$total etcd member(s) give quorum $quorum and tolerate $tolerance loss(es) — an HA control plane must survive losing a server" [ $(( total % 2 )) -eq 1 ] \ || fail "$total etcd members is EVEN: quorum is the same as for $(( total - 1 )) members, so the extra member adds a tie instead of tolerance" # --- the count R-PLAT-01 itself fixes, per environment -------------------- # The environment is the cluster's OWN CLUSTER_ENV (cluster-vars), never a # flag: 'k3s across 3 nodes ... in prod; dev/test MAY run fewer nodes but MUST # exercise the same multi-node topology'. case "${CLUSTER_ENV:-}" in prod) [ "$total" = "3" ] \ || fail "this cluster declares CLUSTER_ENV=prod and has $total nodes — R-PLAT-01 requires 'k3s across 3 nodes ... in prod' (docs/02: '3 physical nodes')" ;; "") fail "flux-system/cluster-vars has no CLUSTER_ENV — the cluster does not declare which environment it is, so R-PLAT-01's prod clause has no referent (and no literal may be substituted for it)" ;; *) [ "$total" -ge 2 ] \ || fail "this cluster declares CLUSTER_ENV=${CLUSTER_ENV} and has $total node(s) — dev/test may run fewer nodes than prod but MUST 'exercise the same multi-node topology'" ;; esac echo "HA_TOPOLOGY_OK" check: (contains($stdout, 'HA_TOPOLOGY_OK')): true # ----------------------------------------------------------------------------------- - name: then-a-control-plane-only-vip-is-floated-on-every-server description: >- Read the running kube-vip DaemonSet: Ready on every control-plane node, control-plane-only, and floating an address that collides with no other allocator. try: - script: env: - name: GATEWAY_VIP value: ($gatewayVip) - name: LB_POOL_START value: ($lbPoolStart) - name: LB_POOL_STOP value: ($lbPoolStop) content: | set -eu fail() { echo "FAIL: $*" >&2; exit 1; } # dotted quad -> integer, for range comparison without ipcalc/python. n() { echo "$1" | awk -F. '{printf "%d", ($1*16777216)+($2*65536)+($3*256)+$4}'; } # The DaemonSet is found by the label roles/kube_vip stamps on it, not by a # hardcoded name in a namespace this test guessed. ds="$(kubectl -n kube-system get ds -l app.kubernetes.io/name=kube-vip-ds -o name | head -1)" [ -n "$ds" ] \ || fail "no kube-vip DaemonSet is running in kube-system — the kube-apiserver has no floating endpoint, so losing the founder takes the API endpoint with it" desired="$(kubectl -n kube-system get "$ds" -o jsonpath='{.status.desiredNumberScheduled}')" ready="$(kubectl -n kube-system get "$ds" -o jsonpath='{.status.numberReady}')" cp="$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o name | grep -c . || true)" vip="$(kubectl -n kube-system get "$ds" \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="address")].value}')" cp_enable="$(kubectl -n kube-system get "$ds" \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="cp_enable")].value}')" svc_enable="$(kubectl -n kube-system get "$ds" \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="svc_enable")].value}')" echo "kube_vip=$ds ready=$ready/$desired control_plane_nodes=$cp vip=$vip cp_enable=$cp_enable svc_enable=$svc_enable" [ -n "$vip" ] \ || fail "the kube-vip DaemonSet announces no address — nothing is floated" [ "$cp_enable" = "true" ] \ || fail "kube-vip runs with cp_enable=$cp_enable — control-plane mode is off, so it floats no API VIP" [ "$svc_enable" = "false" ] \ || fail "kube-vip runs with svc_enable=$svc_enable — it would also serve Service LoadBalancers and contend with Cilium LB-IPAM for ARP on the same addresses (ADR-0002 gives Service LB to Cilium)" [ "$desired" = "$cp" ] \ || fail "kube-vip is scheduled on $desired node(s) but there are $cp control-plane nodes — the VIP must be able to move to ANY server, or its failover set is smaller than the control plane" [ "$ready" = "$desired" ] \ || fail "kube-vip is Ready on $ready of $desired nodes — a VIP holder that is not running cannot take the address over" # --- the VIP collides with nothing -------------------------------------- node_ips="$(kubectl get nodes \ -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' \ | grep -c "^${vip}$" || true)" [ "$node_ips" = "0" ] \ || fail "the floated VIP $vip is ALSO a node's own InternalIP — a VIP that is a node's address cannot float off that node" [ "$vip" != "${GATEWAY_VIP:-}" ] \ || fail "the control-plane VIP and the ingress Gateway VIP are the same address ($vip) — kube-vip and Cilium would both ARP for it" if [ -n "${LB_POOL_START:-}" ] && [ -n "${LB_POOL_STOP:-}" ]; then v="$(n "$vip")"; lo="$(n "$LB_POOL_START")"; hi="$(n "$LB_POOL_STOP")" [ "$lo" -le "$hi" ] \ || fail "the cluster's Cilium LB pool ${LB_POOL_START}-${LB_POOL_STOP} is inverted, so the VIP cannot be proven to be outside it" if [ "$v" -ge "$lo" ] && [ "$v" -le "$hi" ]; then fail "the control-plane VIP $vip lies INSIDE this cluster's Cilium LoadBalancer pool ${LB_POOL_START}-${LB_POOL_STOP} — LB-IPAM could hand the kube-API's VIP to a Service" fi else fail "flux-system/cluster-vars declares no CILIUM_LB_POOL_START/STOP — the separation between the control-plane VIP and the Service-LB allocator cannot be checked, and no literal range may be substituted for it (R-ENV-02)" fi echo "HA_VIP_CONTROL_PLANE_ONLY_OK" check: (contains($stdout, 'HA_VIP_CONTROL_PLANE_ONLY_OK')): true # ----------------------------------------------------------------------------------- - name: then-the-apiserver-answers-through-the-vip-and-on-every-server description: >- From inside the cluster, probe /readyz on the floated VIP and on every server's own address, validating TLS against the cluster CA for the address dialled. try: - script: # The address list is DISCOVERED here (Nodes + the kube-vip DaemonSet), so the # Job cannot be a static fixture — writing the addresses down is the # per-environment literal R-ENV-02 forbids. Hence the imperative heredoc, the # same pattern tests/l3-fault-injection/dr-restore uses for run-time-shaped # fixtures. The Job lives in chainsaw's per-test namespace and is torn down with # it; nothing outside that namespace is created or changed. content: | set -eu ns="$NAMESPACE" fail() { echo "FAIL: $*" >&2; exit 1; } ds="$(kubectl -n kube-system get ds -l app.kubernetes.io/name=kube-vip-ds -o name | head -1)" vip="$(kubectl -n kube-system get "$ds" \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="address")].value}')" servers="$(kubectl get nodes -l node-role.kubernetes.io/control-plane \ -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{" "}{end}')" [ -n "$vip" ] || fail "no floated VIP to probe" [ -n "$servers" ] || fail "no control-plane node addresses to probe" echo "probing vip=$vip servers=$servers" # The probe pod authenticates with its own ServiceAccount token and validates # TLS against the cluster CA, both from the projected SA volume. /readyz is # authorized for any authenticated identity via the built-in # system:public-info-viewer role — anonymous access is NOT relied on (verified: # an unauthenticated /readyz on this platform answers 401). kubectl -n "$ns" apply -f - <&1)" || rc=1 echo "vip \$VIP -> [\$out]" if [ "\$rc" != "0" ] || [ "\$out" != "ok" ]; then echo "FAIL: the kube-apiserver did not answer /readyz through the floated VIP \$VIP with a CA-valid cert for that address — the VIP does not front the API (or it is missing from the serving cert's SANs)" exit 1 fi for s in \$SERVERS; do rc=0 out="\$(probe "\$s" 2>&1)" || rc=1 echo "server \$s -> [\$out]" if [ "\$rc" != "0" ] || [ "\$out" != "ok" ]; then echo "FAIL: server \$s does not answer /readyz with a CA-valid cert for its own address — it is not running a kube-apiserver, so the control plane is not HA across every node" exit 1 fi done echo "HA_API_ENDPOINT_OK" EOF # Bounded wait; on failure surface the probe's own output before failing. if ! kubectl -n "$ns" wait --for=condition=complete --timeout=140s job/ha-api-endpoint-probe; then kubectl -n "$ns" logs job/ha-api-endpoint-probe --tail=40 || true fail "the in-cluster API-endpoint probe did not complete — see its log above" fi logs="$(kubectl -n "$ns" logs job/ha-api-endpoint-probe --tail=40)" echo "$logs" case "$logs" in *HA_API_ENDPOINT_OK*) echo "HA_API_ENDPOINT_REACHABLE_OK" ;; *) fail "the probe completed without reporting success" ;; esac check: (contains($stdout, 'HA_API_ENDPOINT_REACHABLE_OK')): true