jakman85andClaude Opus 4.8
cc89b4b830
Author-only Phase-2 components (platform suspended; cluster untouched): - platform/longhorn: 3-replica SC + bulk external-mount indirection (R-STATE) - platform/monitoring: kube-prometheus-stack + Loki + Alloy (dev-trimmed) + 8 alert rules (R-OBS) - platform/cilium: parity fix (k8sServiceHost=10.73.0.11 + masquerade) safe to adopt Phase 2 - conformance: partial L2 tests (worker hit session limit; incomplete — see DECISIONS-LOG K2) - bootstrap/gitea: fix rootless-podman perms (host bind-mount -> named volume) Flux installed on the cluster; host Gitea up + repo pushed; GitRepository BLOCKED by host firewalld (VM->10.73.0.1:3000) — needs one operator sudo command (DECISIONS-LOG). Known issue K1: cluster-vars substitution-var case mismatch (fix before un-suspend). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
113 lines
4.8 KiB
YAML
113 lines
4.8 KiB
YAML
# =============================================================================
|
|
# L3 FAULT-INJECTION — Data survival on rebuild (R-STATE-01) [RUNNABLE / self-skips]
|
|
# *SAFETY-CRITICAL*
|
|
# Catalog: docs/14 "Data survival (R-STATE-01, L3)"
|
|
# Spec: docs/04-storage-and-state.md
|
|
#
|
|
# GIVEN a stateful workload with written data on a longhorn-backed PVC,
|
|
# WHEN the workload is DESTROYED entirely and a fresh one re-bound to the PVC,
|
|
# THEN the data is intact.
|
|
#
|
|
# R-TEST-05: proves no-data-loss by actually destroying the workload (not a
|
|
# happy-path read). The PVC is kept; the pod is deleted and rebuilt.
|
|
#
|
|
# RUNNABLE / SELF-SKIPPING: Longhorn (T-2.1) is not on the base cluster yet.
|
|
# This test SKIPS CLEANLY (green) when no `longhorn` StorageClass exists, and
|
|
# runs the full destroy/rebuild survival check the moment one does — no edit
|
|
# needed. Hermetic: PVC + pods live in the chainsaw per-test namespace and are
|
|
# torn down with it.
|
|
# =============================================================================
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: data-survival
|
|
labels:
|
|
conformance.home.k3s/R-STATE-01: "true"
|
|
annotations:
|
|
conformance.home.k3s/requirements: "R-STATE-01"
|
|
conformance.home.k3s/layer: "L3"
|
|
conformance.home.k3s/safety-critical: "true"
|
|
# executable => runs against a cluster; self-skips (green) until longhorn exists.
|
|
conformance.home.k3s/executable: "true"
|
|
conformance.home.k3s/intent: >-
|
|
Destroy the workload entirely and re-bind a fresh one to the same longhorn
|
|
PVC; written data survives. Skips cleanly until a longhorn StorageClass exists.
|
|
spec:
|
|
steps:
|
|
- name: data-survives-destroy-and-rebuild
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -euo pipefail
|
|
NS="$NAMESPACE"
|
|
|
|
# -- Gate: skip cleanly (green) if the longhorn tier is not present yet. --
|
|
if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
|
|
echo "SKIP: 'longhorn' StorageClass absent — data-survival not applicable on this cluster yet"
|
|
echo DATA_SURVIVAL_SKIPPED_OK
|
|
exit 0
|
|
fi
|
|
|
|
CANARY="conformance-canary-$(date +%s%N)"
|
|
|
|
# -- GIVEN: a PVC on longhorn + a writer pod that writes the canary. --
|
|
kubectl -n "$NS" apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: survivor
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
EOF
|
|
|
|
apply_pod() {
|
|
kubectl -n "$NS" apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $1
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36.1
|
|
command: ["sh", "-c", "sleep 3600"]
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: survivor
|
|
EOF
|
|
}
|
|
|
|
apply_pod writer
|
|
kubectl -n "$NS" wait --for=condition=Ready pod/writer --timeout=180s
|
|
printf '%s' "$CANARY" | kubectl -n "$NS" exec -i writer -- sh -c 'cat > /data/canary && sync'
|
|
echo "wrote canary: $(kubectl -n "$NS" exec writer -- cat /data/canary)"
|
|
|
|
# -- WHEN: FAULT INJECTION — destroy the workload ENTIRELY (keep the PVC). --
|
|
kubectl -n "$NS" delete pod writer --wait=true --timeout=180s
|
|
kubectl -n "$NS" get pvc survivor # PVC must still exist
|
|
|
|
# -- THEN: a fresh workload re-bound to the same PVC still sees the data. --
|
|
apply_pod reader
|
|
kubectl -n "$NS" wait --for=condition=Ready pod/reader --timeout=240s
|
|
GOT="$(kubectl -n "$NS" exec reader -- cat /data/canary)"
|
|
echo "recovered=$GOT expected=$CANARY"
|
|
[ "$GOT" = "$CANARY" ] && echo DATA_SURVIVED_OK
|
|
check:
|
|
# green on either a clean skip (no longhorn) or a proven survival.
|
|
($stdout =~ '(DATA_SURVIVED_OK|DATA_SURVIVAL_SKIPPED_OK)'): true
|
|
finally:
|
|
- script:
|
|
content: |
|
|
kubectl -n "$NAMESPACE" delete pod writer reader --ignore-not-found --wait=false || true
|
|
kubectl -n "$NAMESPACE" delete pvc survivor --ignore-not-found --wait=false || true
|
|
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|