home-network-k3s-impl/conformance/tests/l3-fault-injection/node-loss/chainsaw-test.yaml

# =============================================================================
# L3 FAULT-INJECTION — Volume survives node loss  (R-STATE-02)  [RUNNABLE / self-skips]
#                      *SAFETY-CRITICAL*
# Catalog: docs/14 "Node loss (R-STATE-02, L3)"
# Spec:    docs/04-storage-and-state.md ("Longhorn — primary persistent storage";
#          Default StorageClass = 3 replicas so any single node loss keeps data available)
#
# GIVEN a 3-replica Longhorn volume with written data, attached on some node N,
# WHEN  node N is REMOVED from service (cordoned + the attached workload evicted off it,
#       forcing the volume to detach from N),
# THEN  the volume stays AVAILABLE — a fresh workload scheduled on a SURVIVING node
#       re-attaches the same volume (served from its replicas on the other nodes) and
#       reads the data intact — and the cluster reconverges.
#
# R-TEST-05 (fault-injection, not happy-path): the availability guarantee is proven by
# actually taking a node out from under the attached volume and asserting the data is
# still served from a different node — not by re-reading the still-attached volume.
#
# SAFE-BY-CONSTRUCTION (small 3-node embedded-etcd dev cluster):
#   * Disruption is scoped to EXACTLY ONE node, and only at the SCHEDULING layer:
#     `kubectl cordon` (spec.unschedulable=true) + eviction of THIS test's own workload
#     pod. The node's kubelet/etcd member stay UP — etcd quorum is never touched, so this
#     cannot starve embedded-etcd I/O the way a real power-off / NotReady node can. That
#     harsher "node truly down" drill is an OPERATOR-run exercise for a maintenance window,
#     NOT this automated test (see report).
#   * We evict ONLY our own `writer` pod (delete), never `kubectl drain` — so no unrelated
#     workload or system pod on that node is disturbed.
#   * The victim node is labelled `conformance.home.k3s/cordoned-by=<this-test-namespace>`
#     at cordon time; the `finally` step ALWAYS uncordons every node carrying THAT unique
#     label and removes it — so the node is restored to schedulable even if the test fails,
#     panics, or times out, and we never uncordon a node an operator cordoned deliberately.
#
# RUNNABLE / SELF-SKIPPING: skips CLEANLY (green) until BOTH a `longhorn` StorageClass
# exists AND the cluster has >= 3 Ready nodes (a 3-replica volume + one-node-loss headroom
# is not meaningful otherwise); runs the full node-loss survival check the moment it does —
# no edit needed. Hermetic: PVC + pods live in the chainsaw per-test namespace and are torn
# down with it; the only out-of-namespace mutation (the node cordon/label) is reverted in
# `finally`.
# =============================================================================
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
  name: node-loss
  labels:
    conformance.home.k3s/R-STATE-02: "true"
  annotations:
    conformance.home.k3s/requirements: "R-STATE-02"
    conformance.home.k3s/layer: "L3"
    conformance.home.k3s/safety-critical: "true"
    # executable => runs against a cluster; self-skips (green) until longhorn + 3 nodes exist.
    conformance.home.k3s/executable: "true"
    conformance.home.k3s/intent: >-
      Take one node out of service (cordon + evict the attached workload) under a 3-replica
      Longhorn volume; a fresh workload on a surviving node re-attaches the volume and reads
      the data intact — the volume stays available and the cluster reconverges. Skips cleanly
      until a longhorn StorageClass and >= 3 Ready nodes exist.
spec:
  # Longhorn detach-from-victim + re-attach-on-survivor (rebuild/replica scheduling) is far
  # slower than a same-node pod restart; give the eventual-consistency waits real headroom.
  timeouts:
    exec: 20m
  steps:
    - name: volume-stays-available-across-node-loss
      try:
        - script:
            content: |
              set -euo pipefail
              NS="$NAMESPACE"

              # -- Gate 1: skip cleanly (green) if the longhorn tier is not present yet. --
              if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
                echo "SKIP: 'longhorn' StorageClass absent — node-loss not applicable on this cluster yet"
                echo NODE_LOSS_SKIPPED_OK
                exit 0
              fi

              # -- Gate 2: a 3-replica volume + single-node-loss headroom needs >= 3 Ready nodes. --
              READY="$(kubectl get nodes \
                -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \
                | grep -c '^True$' || true)"
              echo "ready nodes: ${READY}"
              if [ "${READY:-0}" -lt 3 ]; then
                echo "SKIP: only ${READY:-0} Ready node(s) (<3) — node-loss resilience not meaningful yet"
                echo NODE_LOSS_SKIPPED_OK
                exit 0
              fi

              CANARY="node-loss-canary-$(date +%s%N)"

              # -- GIVEN: a Longhorn PVC (platform default SC = 3 replicas across nodes) + a
              #    writer pod that writes the canary and flushes it to the replicated volume. --
              kubectl -n "$NS" apply -f - <<EOF
              apiVersion: v1
              kind: PersistentVolumeClaim
              metadata:
                name: survivor
              spec:
                accessModes: [ReadWriteOnce]
                storageClassName: longhorn
                resources:
                  requests:
                    storage: 1Gi
              EOF

              apply_pod() {
                kubectl -n "$NS" apply -f - <<EOF
              apiVersion: v1
              kind: Pod
              metadata:
                name: $1
              spec:
                restartPolicy: Never
                containers:
                  - name: app
                    image: busybox:1.36.1
                    command: ["sh", "-c", "sleep 3600"]
                    volumeMounts:
                      - name: data
                        mountPath: /data
                volumes:
                  - name: data
                    persistentVolumeClaim:
                      claimName: survivor
              EOF
              }

              apply_pod writer
              kubectl -n "$NS" wait --for=condition=Ready pod/writer --timeout=240s
              printf '%s' "$CANARY" | kubectl -n "$NS" exec -i writer -- sh -c 'cat > /data/canary && sync'
              echo "wrote canary: $(kubectl -n "$NS" exec writer -- cat /data/canary)"

              # The victim node is the one the attached volume currently lives on.
              VICTIM="$(kubectl -n "$NS" get pod writer -o jsonpath='{.spec.nodeName}')"
              [ -n "$VICTIM" ] || { echo "FAIL: could not determine writer's node" >&2; exit 1; }
              echo "victim node (currently hosts the attached volume): $VICTIM"

              # Best-effort precondition log: if the Longhorn Volume CR is readable, show the
              # volume is genuinely 3-replica/healthy before we take a node. Non-fatal (the
              # runner is least-privilege and may not have longhorn.io read rights) — the
              # data-availability assertion below is the authoritative proof, not this.
              PVNAME="$(kubectl -n "$NS" get pvc survivor -o jsonpath='{.spec.volumeName}' 2>/dev/null || true)"
              if [ -n "$PVNAME" ]; then
                kubectl -n longhorn-system get volumes.longhorn.io "$PVNAME" \
                  -o jsonpath='pre-fault robustness={.status.robustness} replicas={.spec.numberOfReplicas}{"\n"}' \
                  2>/dev/null || echo "(Longhorn Volume CR not readable — continuing; data read is the proof)"
              fi

              # -- WHEN: FAULT INJECTION — remove the victim node FROM SERVICE. --
              #    Label it with THIS test's unique namespace so `finally` can always find and
              #    uncordon exactly this node; then cordon it; then evict ONLY our own writer
              #    pod, forcing the volume to detach from the victim. (No `drain` — unrelated
              #    pods on the node are never touched.)
              kubectl label node "$VICTIM" "conformance.home.k3s/cordoned-by=$NS" --overwrite
              kubectl cordon "$VICTIM"
              echo "cordoned + labelled victim node: $VICTIM"
              kubectl -n "$NS" delete pod writer --wait=true --timeout=240s
              echo "evicted writer off the victim node; volume detached from $VICTIM"

              # -- THEN: a fresh workload re-binds the SAME volume. The victim is unschedulable,
              #    so the scheduler MUST place the reader on a SURVIVING node; Longhorn re-attaches
              #    the volume there from its replicas on the other nodes. --
              apply_pod reader
              kubectl -n "$NS" wait --for=condition=Ready pod/reader --timeout=360s
              READER_NODE="$(kubectl -n "$NS" get pod reader -o jsonpath='{.spec.nodeName}')"
              echo "reader scheduled on surviving node: $READER_NODE (victim was $VICTIM)"
              if [ "$READER_NODE" = "$VICTIM" ]; then
                echo "FAIL: reader landed back on the cordoned victim node — node was not removed from service" >&2
                exit 1
              fi

              GOT="$(kubectl -n "$NS" exec reader -- cat /data/canary)"
              echo "recovered='$GOT' expected='$CANARY'"
              [ "$GOT" = "$CANARY" ] || { echo "FAIL: data not available after node loss" >&2; exit 1; }
              echo "volume available + data intact from a surviving node"

              # Best-effort reconvergence log: if the Longhorn Volume CR is readable, wait
              # (bounded) for robustness to settle back to available (healthy|degraded — both
              # serve I/O; a healthy 3-replica volume rebuilds a replica once headroom exists).
              # Non-fatal for the same least-privilege reason as the pre-fault log.
              if [ -n "$PVNAME" ]; then
                for i in $(seq 1 30); do
                  ROB="$(kubectl -n longhorn-system get volumes.longhorn.io "$PVNAME" \
                        -o jsonpath='{.status.robustness}' 2>/dev/null || true)"
                  [ -z "$ROB" ] && { echo "(Longhorn Volume CR not readable — skipping reconvergence log)"; break; }
                  echo "post-fault robustness[$i]=$ROB"
                  case "$ROB" in healthy|degraded) break ;; esac
                  sleep 10
                done
              fi

              echo NODE_LOSS_VOLUME_AVAILABLE_OK
            check:
              # green on either a clean skip (no longhorn / <3 nodes) or a proven survival.
              (contains($stdout, 'NODE_LOSS_VOLUME_AVAILABLE_OK') || contains($stdout, 'NODE_LOSS_SKIPPED_OK')): true
      finally:
        # SAFETY FIRST: always restore the victim node to schedulable, even on failure/timeout.
        # Keyed by this test's unique namespace label so we touch ONLY the node we cordoned.
        - script:
            content: |
              set -uo pipefail
              for n in $(kubectl get nodes -l "conformance.home.k3s/cordoned-by=$NAMESPACE" -o name 2>/dev/null); do
                kubectl uncordon "${n#node/}" || true
                kubectl label "$n" "conformance.home.k3s/cordoned-by-" || true
                echo "restored node to service: ${n#node/}"
              done
        - script:
            content: |
              kubectl -n "$NAMESPACE" delete pod writer reader --ignore-not-found --wait=false || true
              kubectl -n "$NAMESPACE" delete pvc survivor --ignore-not-found --wait=false || true
              echo "cleanup best-effort; chainsaw namespace teardown finalizes"