Merge feature/conformance-r-state-02-node-loss: L3 node-loss test (R-STATE-02, closes last safety-critical gap)

conformance/README.md

@@ -30,9 +30,12 @@ behavioral / L3 fault-injection** (this harness).
 - `--requirements R-…` — **merge gate**: selects the tagged tests to run **and** asserts
   those touched requirements are covered (a listed safety-critical one must have an L3 test).
 - `--require-all` — **phase-exit gate**: every registry requirement must be covered.
-- Documented out-of-scope gaps (e.g. `R-STATE-02` node-loss, owned by WS-STORAGE) are
+- Documented out-of-scope gaps (e.g. `R-ENV-02` dev→prod promote and `R-GITOPS-01`
-  exempted from the completeness gates via `--allow-gap`, wired in `run.sh` as
+  state-in-git, owned by WS-INFRA/WS-GITOPS and keyed wholesale by the coarse path map)
-  `DOCUMENTED_GAPS` so the exemption is explicit and reviewable.
+  are exempted from the completeness gates via `--allow-gap`, wired in `run.sh` as
+  `DOCUMENTED_GAPS` so the exemption is explicit and reviewable. (`R-STATE-02` node-loss
+  used to be here; it now has a real L3 test — `tests/l3-fault-injection/node-loss` — and
+  is covered, not exempted.)
 
 `run.sh` is the **only** entrypoint. Local runs and the in-cluster CI Chainsaw Job call
 the *same* script with the same config, so results are identical.

conformance/incluster/rbac.yaml

@@ -88,13 +88,15 @@ rules:
   # namespaces policy is enforced, the `chainsaw-conf-*` test namespaces MUST be in
   # its exclusion set (documented, audited exception — R-SEC-02).
   #
-  # ---- R-STATE-02 (node loss): cordon/drain/uncordon a node  (NOT in WS-TEST scope yet)
+  # ---- R-STATE-02 (node loss, tests/l3-fault-injection/node-loss): the fault is
-  # - apiGroups: [""]
+  # `kubectl cordon` + label of ONE node, reverted by `kubectl uncordon` + label removal
-  #   resources: [nodes]
+  # in the test's `finally`. All three (cordon/uncordon/label) are a `patch` on the node
-  #   verbs: [get, list, patch]             # cordon = patch spec.unschedulable
+  # object; `get`/`list` locate the victim + the finally cleanup selector. The test evicts
-  # - apiGroups: [""]
+  # only its OWN writer pod via `delete pod` (already granted above) — it deliberately does
-  #   resources: [pods/eviction]
+  # NOT `kubectl drain`, so no eviction rights and no reach over unrelated pods are needed.
-  #   verbs: [create]                       # drain
+  - apiGroups: [""]
+    resources: [nodes]
+    verbs: [get, list, patch]             # cordon/uncordon/label = patch spec.unschedulable + metadata.labels
   #
   # ---- R-UPD-02 (bad-update auto-rollback): inject a bad image tag via automation
   # - apiGroups: [image.toolkit.fluxcd.io]

conformance/lib/coverage.py

@@ -30,7 +30,7 @@ Exit status / gating (R-TEST-03 gates are *scoped*, not "all 51 always"):
   --require-all    FULL completeness gate (phase-exit / final): fails if ANY registry
                    requirement is uncovered or any safety-critical one lacks an L3 test.
   --allow-gap IDs  exempt specific IDs from the completeness gates (documented, out-of-
-                   scope gaps such as R-STATE-02 node-loss owned by another workstream).
+                   scope gaps such as R-ENV-02/R-GITOPS-01 owned by another workstream).
 
 Pure standard library (no PyYAML) so the harness has no pip dependency: the registry and
 the test label/annotation lines we care about are parsed with line-oriented regexes, and

conformance/run.sh

@@ -49,7 +49,6 @@ FORCE_NO_CLUSTER=0
 # Documented, out-of-scope coverage gaps exempt from the completeness gate: requirements
 # owned by another workstream that WS-TEST does not yet scaffold. Listed here so the
 # exemption is explicit and reviewable, not silent.
-#   R-STATE-02 (node-loss)         — delivered with Longhorn/infra (WS-STORAGE).
 #   R-ENV-02   (dev->prod promote) — owned by WS-INFRA/WS-GITOPS; unscaffolded (a GAP in
 #                                    coverage.md). The coarse path->requirement map keys
 #                                    clusters/* AND ci/* to it and the R-ENV-02 doc token
@@ -71,8 +70,12 @@ FORCE_NO_CLUSTER=0
 # authoritative signal is the `conformance.home.k3s/R-*` label on a test, which still is.
 # The PATH-MAP half remains: ci/touched-requirements.sh keys clusters/* and ci/* to
 # R-ENV-02 + R-GITOPS-01 wholesale, which is why those two must stay exempted here. Tighten
-# the path map (or scaffold the two requirements) and this list shrinks to R-STATE-02.
+# the path map (or scaffold the two requirements) and this list empties.
-DOCUMENTED_GAPS="R-STATE-02 R-ENV-02 R-GITOPS-01"
+#
+# R-STATE-02 (node-loss) WAS listed here (owned by WS-STORAGE, "delivered with Longhorn/infra")
+# but now has a real verifying L3 test — tests/l3-fault-injection/node-loss — so it is no
+# longer a gap and has been removed from this list; the coverage report counts it as covered.
+DOCUMENTED_GAPS="R-ENV-02 R-GITOPS-01"
 
 usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }
 
@@ -154,7 +157,7 @@ select_dirs() {
 #     DIRECTORY basename (stable); override via CONF_HEAVY_TESTS to add/remove without code
 #     changes. Listed here even for tests still scaffolded (vm-lifecycle, dr-restore, ...) so
 #     the serialization is already correct the moment they grow real KubeVirt/Longhorn bodies.
-CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore}"
+CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore node-loss}"
 
 conf_is_heavy_dir() {
   # $1 = test dir; true if its basename is in the heavy set.

conformance/tests/l3-fault-injection/node-loss/chainsaw-test.yaml

@@ -0,0 +1,210 @@
+# =============================================================================
+# L3 FAULT-INJECTION — Volume survives node loss  (R-STATE-02)  [RUNNABLE / self-skips]
+#                      *SAFETY-CRITICAL*
+# Catalog: docs/14 "Node loss (R-STATE-02, L3)"
+# Spec:    docs/04-storage-and-state.md ("Longhorn — primary persistent storage";
+#          Default StorageClass = 3 replicas so any single node loss keeps data available)
+#
+# GIVEN a 3-replica Longhorn volume with written data, attached on some node N,
+# WHEN  node N is REMOVED from service (cordoned + the attached workload evicted off it,
+#       forcing the volume to detach from N),
+# THEN  the volume stays AVAILABLE — a fresh workload scheduled on a SURVIVING node
+#       re-attaches the same volume (served from its replicas on the other nodes) and
+#       reads the data intact — and the cluster reconverges.
+#
+# R-TEST-05 (fault-injection, not happy-path): the availability guarantee is proven by
+# actually taking a node out from under the attached volume and asserting the data is
+# still served from a different node — not by re-reading the still-attached volume.
+#
+# SAFE-BY-CONSTRUCTION (small 3-node embedded-etcd dev cluster):
+#   * Disruption is scoped to EXACTLY ONE node, and only at the SCHEDULING layer:
+#     `kubectl cordon` (spec.unschedulable=true) + eviction of THIS test's own workload
+#     pod. The node's kubelet/etcd member stay UP — etcd quorum is never touched, so this
+#     cannot starve embedded-etcd I/O the way a real power-off / NotReady node can. That
+#     harsher "node truly down" drill is an OPERATOR-run exercise for a maintenance window,
+#     NOT this automated test (see report).
+#   * We evict ONLY our own `writer` pod (delete), never `kubectl drain` — so no unrelated
+#     workload or system pod on that node is disturbed.
+#   * The victim node is labelled `conformance.home.k3s/cordoned-by=<this-test-namespace>`
+#     at cordon time; the `finally` step ALWAYS uncordons every node carrying THAT unique
+#     label and removes it — so the node is restored to schedulable even if the test fails,
+#     panics, or times out, and we never uncordon a node an operator cordoned deliberately.
+#
+# RUNNABLE / SELF-SKIPPING: skips CLEANLY (green) until BOTH a `longhorn` StorageClass
+# exists AND the cluster has >= 3 Ready nodes (a 3-replica volume + one-node-loss headroom
+# is not meaningful otherwise); runs the full node-loss survival check the moment it does —
+# no edit needed. Hermetic: PVC + pods live in the chainsaw per-test namespace and are torn
+# down with it; the only out-of-namespace mutation (the node cordon/label) is reverted in
+# `finally`.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: node-loss
+  labels:
+    conformance.home.k3s/R-STATE-02: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-STATE-02"
+    conformance.home.k3s/layer: "L3"
+    conformance.home.k3s/safety-critical: "true"
+    # executable => runs against a cluster; self-skips (green) until longhorn + 3 nodes exist.
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      Take one node out of service (cordon + evict the attached workload) under a 3-replica
+      Longhorn volume; a fresh workload on a surviving node re-attaches the volume and reads
+      the data intact — the volume stays available and the cluster reconverges. Skips cleanly
+      until a longhorn StorageClass and >= 3 Ready nodes exist.
+spec:
+  # Longhorn detach-from-victim + re-attach-on-survivor (rebuild/replica scheduling) is far
+  # slower than a same-node pod restart; give the eventual-consistency waits real headroom.
+  timeouts:
+    exec: 20m
+  steps:
+    - name: volume-stays-available-across-node-loss
+      try:
+        - script:
+            content: |
+              set -euo pipefail
+              NS="$NAMESPACE"
+
+              # -- Gate 1: skip cleanly (green) if the longhorn tier is not present yet. --
+              if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
+                echo "SKIP: 'longhorn' StorageClass absent — node-loss not applicable on this cluster yet"
+                echo NODE_LOSS_SKIPPED_OK
+                exit 0
+              fi
+
+              # -- Gate 2: a 3-replica volume + single-node-loss headroom needs >= 3 Ready nodes. --
+              READY="$(kubectl get nodes \
+                -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \
+                | grep -c '^True$' || true)"
+              echo "ready nodes: ${READY}"
+              if [ "${READY:-0}" -lt 3 ]; then
+                echo "SKIP: only ${READY:-0} Ready node(s) (<3) — node-loss resilience not meaningful yet"
+                echo NODE_LOSS_SKIPPED_OK
+                exit 0
+              fi
+
+              CANARY="node-loss-canary-$(date +%s%N)"
+
+              # -- GIVEN: a Longhorn PVC (platform default SC = 3 replicas across nodes) + a
+              #    writer pod that writes the canary and flushes it to the replicated volume. --
+              kubectl -n "$NS" apply -f - <<EOF
+              apiVersion: v1
+              kind: PersistentVolumeClaim
+              metadata:
+                name: survivor
+              spec:
+                accessModes: [ReadWriteOnce]
+                storageClassName: longhorn
+                resources:
+                  requests:
+                    storage: 1Gi
+              EOF
+
+              apply_pod() {
+                kubectl -n "$NS" apply -f - <<EOF
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: $1
+              spec:
+                restartPolicy: Never
+                containers:
+                  - name: app
+                    image: busybox:1.36.1
+                    command: ["sh", "-c", "sleep 3600"]
+                    volumeMounts:
+                      - name: data
+                        mountPath: /data
+                volumes:
+                  - name: data
+                    persistentVolumeClaim:
+                      claimName: survivor
+              EOF
+              }
+
+              apply_pod writer
+              kubectl -n "$NS" wait --for=condition=Ready pod/writer --timeout=240s
+              printf '%s' "$CANARY" | kubectl -n "$NS" exec -i writer -- sh -c 'cat > /data/canary && sync'
+              echo "wrote canary: $(kubectl -n "$NS" exec writer -- cat /data/canary)"
+
+              # The victim node is the one the attached volume currently lives on.
+              VICTIM="$(kubectl -n "$NS" get pod writer -o jsonpath='{.spec.nodeName}')"
+              [ -n "$VICTIM" ] || { echo "FAIL: could not determine writer's node" >&2; exit 1; }
+              echo "victim node (currently hosts the attached volume): $VICTIM"
+
+              # Best-effort precondition log: if the Longhorn Volume CR is readable, show the
+              # volume is genuinely 3-replica/healthy before we take a node. Non-fatal (the
+              # runner is least-privilege and may not have longhorn.io read rights) — the
+              # data-availability assertion below is the authoritative proof, not this.
+              PVNAME="$(kubectl -n "$NS" get pvc survivor -o jsonpath='{.spec.volumeName}' 2>/dev/null || true)"
+              if [ -n "$PVNAME" ]; then
+                kubectl -n longhorn-system get volumes.longhorn.io "$PVNAME" \
+                  -o jsonpath='pre-fault robustness={.status.robustness} replicas={.spec.numberOfReplicas}{"\n"}' \
+                  2>/dev/null || echo "(Longhorn Volume CR not readable — continuing; data read is the proof)"
+              fi
+
+              # -- WHEN: FAULT INJECTION — remove the victim node FROM SERVICE. --
+              #    Label it with THIS test's unique namespace so `finally` can always find and
+              #    uncordon exactly this node; then cordon it; then evict ONLY our own writer
+              #    pod, forcing the volume to detach from the victim. (No `drain` — unrelated
+              #    pods on the node are never touched.)
+              kubectl label node "$VICTIM" "conformance.home.k3s/cordoned-by=$NS" --overwrite
+              kubectl cordon "$VICTIM"
+              echo "cordoned + labelled victim node: $VICTIM"
+              kubectl -n "$NS" delete pod writer --wait=true --timeout=240s
+              echo "evicted writer off the victim node; volume detached from $VICTIM"
+
+              # -- THEN: a fresh workload re-binds the SAME volume. The victim is unschedulable,
+              #    so the scheduler MUST place the reader on a SURVIVING node; Longhorn re-attaches
+              #    the volume there from its replicas on the other nodes. --
+              apply_pod reader
+              kubectl -n "$NS" wait --for=condition=Ready pod/reader --timeout=360s
+              READER_NODE="$(kubectl -n "$NS" get pod reader -o jsonpath='{.spec.nodeName}')"
+              echo "reader scheduled on surviving node: $READER_NODE (victim was $VICTIM)"
+              if [ "$READER_NODE" = "$VICTIM" ]; then
+                echo "FAIL: reader landed back on the cordoned victim node — node was not removed from service" >&2
+                exit 1
+              fi
+
+              GOT="$(kubectl -n "$NS" exec reader -- cat /data/canary)"
+              echo "recovered='$GOT' expected='$CANARY'"
+              [ "$GOT" = "$CANARY" ] || { echo "FAIL: data not available after node loss" >&2; exit 1; }
+              echo "volume available + data intact from a surviving node"
+
+              # Best-effort reconvergence log: if the Longhorn Volume CR is readable, wait
+              # (bounded) for robustness to settle back to available (healthy|degraded — both
+              # serve I/O; a healthy 3-replica volume rebuilds a replica once headroom exists).
+              # Non-fatal for the same least-privilege reason as the pre-fault log.
+              if [ -n "$PVNAME" ]; then
+                for i in $(seq 1 30); do
+                  ROB="$(kubectl -n longhorn-system get volumes.longhorn.io "$PVNAME" \
+                        -o jsonpath='{.status.robustness}' 2>/dev/null || true)"
+                  [ -z "$ROB" ] && { echo "(Longhorn Volume CR not readable — skipping reconvergence log)"; break; }
+                  echo "post-fault robustness[$i]=$ROB"
+                  case "$ROB" in healthy|degraded) break ;; esac
+                  sleep 10
+                done
+              fi
+
+              echo NODE_LOSS_VOLUME_AVAILABLE_OK
+            check:
+              # green on either a clean skip (no longhorn / <3 nodes) or a proven survival.
+              (contains($stdout, 'NODE_LOSS_VOLUME_AVAILABLE_OK') || contains($stdout, 'NODE_LOSS_SKIPPED_OK')): true
+      finally:
+        # SAFETY FIRST: always restore the victim node to schedulable, even on failure/timeout.
+        # Keyed by this test's unique namespace label so we touch ONLY the node we cordoned.
+        - script:
+            content: |
+              set -uo pipefail
+              for n in $(kubectl get nodes -l "conformance.home.k3s/cordoned-by=$NAMESPACE" -o name 2>/dev/null); do
+                kubectl uncordon "${n#node/}" || true
+                kubectl label "$n" "conformance.home.k3s/cordoned-by-" || true
+                echo "restored node to service: ${n#node/}"
+              done
+        - script:
+            content: |
+              kubectl -n "$NAMESPACE" delete pod writer reader --ignore-not-found --wait=false || true
+              kubectl -n "$NAMESPACE" delete pvc survivor --ignore-not-found --wait=false || true
+              echo "cleanup best-effort; chainsaw namespace teardown finalizes"