From bd17b65db9fa906f336528e90b429ec1e2377963 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sun, 13 Sep 2026 21:18:11 -0500 Subject: [PATCH] =?UTF-8?q?test(conformance):=20R-STATE-03=20PROVEN=20on?= =?UTF-8?q?=20prod=20=E2=80=94=20DR=20restore=20from=20R2,=20off-cluster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 7's outstanding exit criterion. The only previous dr-restore proof was recorded on the dev cluster (decommissioned 2026-09-12) against MinIO, so it could not speak for prod or for today's Cloudflare R2 target. Run 20260914T021505Z-24b8111, env=prod, kube v1.36.4+k3s1, chainsaw v0.2.12, 150.70s, 1 passed / 0 failed / 0 skipped. Sentinel DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK. What makes it a real proof rather than a green check: the test seeds a canary on the replicated longhorn class, backs the namespace AND volume data up to the off-cluster store, then DESTROYS every in-cluster copy — namespace, PVC and PV — before restoring into a clean, previously-nonexistent namespace. It asserts the restored PVC binds a DIFFERENT PersistentVolume (pvc-f0c061e1 != pvc-4107d59d) and fails explicitly if it rebinds the original, because that would be an in-cluster survival masquerading as a restore. The canary came back byte-identical. It touches no real service's namespace, PVC or PV — the scratch namespaces are named after its own Chainsaw namespace and are destroyed by design. This is the run whose first attempt was discarded for recording env=dev; the recorder fix is 24b8111, which this proof is recorded against. Co-Authored-By: Claude Opus 5 --- .../20260914T021505Z-24b8111/report.junit.xml | 4 + .../runs/20260914T021505Z-24b8111/run.log | 672 ++++++++++++++++++ conformance/runs/ledger.jsonl | 1 + 3 files changed, 677 insertions(+) create mode 100644 conformance/runs/20260914T021505Z-24b8111/report.junit.xml create mode 100644 conformance/runs/20260914T021505Z-24b8111/run.log diff --git a/conformance/runs/20260914T021505Z-24b8111/report.junit.xml b/conformance/runs/20260914T021505Z-24b8111/report.junit.xml new file mode 100644 index 0000000..a842240 --- /dev/null +++ b/conformance/runs/20260914T021505Z-24b8111/report.junit.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/conformance/runs/20260914T021505Z-24b8111/run.log b/conformance/runs/20260914T021505Z-24b8111/run.log new file mode 100644 index 0000000..aefbcbe --- /dev/null +++ b/conformance/runs/20260914T021505Z-24b8111/run.log @@ -0,0 +1,672 @@ +[run.sh] Coverage report first (always) +[coverage] 51/51 requirements have a scaffolded test (100.0%); 0 gap(s), 0 safety-critical gap(s). +[coverage] wrote /work/conformance/_artifacts/coverage/coverage.md and /work/conformance/_artifacts/coverage/coverage.json +[run.sh] No L0/L1 offline checks match the selection (layer=l3 reqs='R-STATE-03'). +[run.sh] LIGHT pass: no tests selected. +[run.sh] HEAVY pass: 1 test(s) SERIAL (parallel=1) — dr-restore +Version: 0.2.12 +Loading config (/work/conformance/config/chainsaw-config.yaml)... +- Using test file: chainsaw-test +- TestDirs [/work/conformance/tests/l3-fault-injection/dr-restore] +- SkipDelete false +- FailFast false +- ReportFormat 'JUNIT-TEST' +- ReportName 'chainsaw-report-heavy' +- ReportPath '/work/conformance/_artifacts' +- Namespace '' +- FullName false +- IncludeTestRegex '' +- ExcludeTestRegex '' +- ApplyTimeout 1m30s +- AssertTimeout 3m0s +- CleanupTimeout 2m0s +- DeleteTimeout 2m0s +- ErrorTimeout 1m0s +- ExecTimeout 3m0s +- DeletionPropagationPolicy Background +- Parallel 1 +- Template true +- NoCluster false +- PauseOnFailure false +Loading tests... +- dr-restore (/work/conformance/tests/l3-fault-injection/dr-restore) +Loading values... +Running tests... +=== RUN chainsaw +=== PAUSE chainsaw +=== CONT chainsaw +=== RUN chainsaw/dr-restore +=== PAUSE chainsaw/dr-restore +=== CONT chainsaw/dr-restore + l.go:52: | 02:15:05 | dr-restore | @chainsaw  | CREATE | OK | v1/Namespace @ chainsaw-glorious-sheep + l.go:52: | 02:15:05 | dr-restore | assert-backup-target-is-off-cluster  | TRY | BEGIN | + l.go:52: | 02:15:05 | dr-restore | assert-backup-target-is-off-cluster  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + + # ---- helpers ------------------------------------------------------- + # Classify a kubectl failure: an RBAC denial is a HARNESS DEFECT and must + # fail loudly; an absent API is a genuine precondition gap and may skip. + die_or_skip() { + case "$2" in + *orbidden*|*Unauthorized*|*cannot\ list*|*cannot\ get*) + echo "FAIL: conformance runner lacks RBAC for $1 -- fix conformance/incluster/rbac.yaml" >&2 + echo "$2" >&2 + exit 1 ;; + *) + echo "SKIP: $1 not available on this cluster -- $2" + echo DR_RESTORE_SKIPPED_OK + exit 0 ;; + esac + } + + ip2int() { + _q="$(echo "$1" | tr '.' ' ')" + set -- $_q + [ $# -eq 4 ] || { echo -1; return 0; } + echo $(( ($1 * 16777216) + ($2 * 65536) + ($3 * 256) + $4 )) + } + + # in_cidr -> 0 when the ip is inside the prefix + in_cidr() { + case "$2" in *"/"*) ;; *) return 1 ;; esac + _net="${2%/*}"; _len="${2#*/}" + case "$_len" in ''|*[!0-9]*) return 1 ;; esac + [ "$_len" -ge 0 ] && [ "$_len" -le 32 ] || return 1 + _i="$(ip2int "$1")"; _n="$(ip2int "$_net")" + [ "$_i" -ge 0 ] && [ "$_n" -ge 0 ] || return 1 + [ "$_len" -eq 0 ] && return 0 + _d=1; _s=0 + while [ "$_s" -lt $((32 - _len)) ]; do _d=$((_d * 2)); _s=$((_s + 1)); done + [ $(( _i / _d )) -eq $(( _n / _d )) ] + } + + # ---- Gate 1: the replicated storage tier the DR-critical data lives on -- + if ! kubectl get storageclass longhorn >/dev/null 2>&1; then + echo "SKIP: 'longhorn' StorageClass absent -- no DR-critical data tier on this cluster yet" + echo DR_RESTORE_SKIPPED_OK + exit 0 + fi + + # ---- Gate 2: the Velero API + a configured BackupStorageLocation -------- + BSL_LINES="$(kubectl get backupstoragelocations.velero.io -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.status.phase}{" "}{.spec.provider}{" "}{.spec.default}{" "}{.spec.config.s3Url}{" "}{.spec.objectStorage.bucket}{"\n"}{end}' \ + 2>&1)" || die_or_skip "velero.io/backupstoragelocations" "$BSL_LINES" + if [ -z "$(echo "$BSL_LINES" | tr -d '[:space:]')" ]; then + echo "SKIP: Velero API present but NO BackupStorageLocation is configured -- no off-cluster backup target yet" + echo DR_RESTORE_SKIPPED_OK + exit 0 + fi + echo "backup storage locations:" + echo "$BSL_LINES" + + VNS="$(echo "$BSL_LINES" | head -1 | cut -d' ' -f1)" + + # ---- Gate 3: Velero server + node-agent (kopia FSB) actually ready ------- + # File System Backup is the volume-DATA path (platform/velero/helmrelease.yaml: + # deployNodeAgent=true, uploaderType=kopia, snapshotsEnabled=false). Without a + # ready node-agent no volume data can be captured, so the drill is not runnable. + VREADY=0; NAREADY=0 + for i in $(seq 1 12); do + VREADY="$(kubectl -n "$VNS" get deploy velero -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" + NAREADY="$(kubectl -n "$VNS" get daemonset node-agent -o jsonpath='{.status.numberReady}' 2>/dev/null || true)" + echo "poll $i: velero readyReplicas=${VREADY:-0} node-agent numberReady=${NAREADY:-0}" + if [ "${VREADY:-0}" -ge 1 ] && [ "${NAREADY:-0}" -ge 1 ]; then break; fi + sleep 10 + done + if [ "${VREADY:-0}" -lt 1 ] || [ "${NAREADY:-0}" -lt 1 ]; then + echo "SKIP: Velero server and/or its node-agent DaemonSet are not ready in ns '$VNS' -- backup/restore path not available" + echo DR_RESTORE_SKIPPED_OK + exit 0 + fi + + # ---- Pick the BSL this drill will use: prefer the default, and it MUST be + # Available. A CONFIGURED-BUT-UNAVAILABLE target is NOT a precondition + # gap -- it is a live R-STATE-03 defect, so it FAILS (after a bounded + # wait for first-validation on a cold start), it does not skip. -------- + BSL_NS=""; BSL_NAME=""; BSL_URL=""; BSL_BUCKET="" + for i in $(seq 1 12); do + BSL_LINES="$(kubectl get backupstoragelocations.velero.io -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.status.phase}{" "}{.spec.provider}{" "}{.spec.default}{" "}{.spec.config.s3Url}{" "}{.spec.objectStorage.bucket}{"\n"}{end}' 2>/dev/null || true)" + CAND="$(echo "$BSL_LINES" | grep ' Available ' | grep ' true ' | head -1 || true)" + [ -n "$CAND" ] || CAND="$(echo "$BSL_LINES" | grep ' Available ' | head -1 || true)" + if [ -n "$CAND" ]; then + BSL_NS="$(echo "$CAND" | cut -d' ' -f1)" + BSL_NAME="$(echo "$CAND" | cut -d' ' -f2)" + BSL_URL="$(echo "$CAND" | cut -d' ' -f6)" + BSL_BUCKET="$(echo "$CAND" | cut -d' ' -f7)" + break + fi + echo "poll $i: no Available BackupStorageLocation yet" + sleep 10 + done + if [ -z "$BSL_NAME" ]; then + echo "FAIL: a BackupStorageLocation is configured but NONE reached phase 'Available'." >&2 + echo " The off-cluster backup target is unreachable -- R-STATE-03 is BROKEN, not merely untested." >&2 + echo "$BSL_LINES" >&2 + exit 1 + fi + echo "using BSL ${BSL_NS}/${BSL_NAME} bucket=${BSL_BUCKET} endpoint=${BSL_URL}" + + # ---- OFF-CLUSTER PROOF (1): an endpoint we can evaluate ------------------ + if [ -z "$BSL_URL" ]; then + echo "FAIL: BackupStorageLocation ${BSL_NS}/${BSL_NAME} exposes no spec.config.s3Url;" >&2 + echo " the OFF-CLUSTER property of the backup target cannot be established, so this" >&2 + echo " test refuses to claim it. (Extend this check for a non-S3 provider.)" >&2 + exit 1 + fi + HOST="${BSL_URL#*://}" # strip scheme + HOST="${HOST%%/*}" # strip path + HOST="${HOST%%:*}" # strip port + echo "backup endpoint host: $HOST" + + case "$HOST" in + localhost|*.svc|*.svc.cluster.local|*.cluster.local) + echo "FAIL: backup endpoint '$HOST' is a CLUSTER-INTERNAL name -- the backup target is IN the cluster." >&2 + echo " An in-cluster backup store cannot satisfy R-STATE-03 (off-cluster backup)." >&2 + exit 1 ;; + esac + + # Resolve to an IPv4 address. If it cannot be resolved we CANNOT establish the + # off-cluster property -- that is a failure, never a silent pass. + case "$HOST" in + *[!0-9.]*) + # Endpoint given as a name. Try the resolvers that may exist in the + # runner image, in order; if none is available or none resolves, the + # test FAILS below rather than guessing. + EPIP="$(getent ahostsv4 "$HOST" 2>/dev/null | head -1 | cut -d' ' -f1 || true)" + [ -n "$EPIP" ] || EPIP="$(getent hosts "$HOST" 2>/dev/null | head -1 | cut -d' ' -f1 || true)" + [ -n "$EPIP" ] || EPIP="$(nslookup "$HOST" 2>/dev/null | grep -A2 '^Name:' | grep '^Address' | head -1 | sed 's/.*: *//' || true)" ;; + *) EPIP="$HOST" ;; + esac + if [ -z "$EPIP" ]; then + echo "FAIL: could not resolve backup endpoint host '$HOST' to an IPv4 address;" >&2 + echo " the OFF-CLUSTER property cannot be asserted, so this test does not pass." >&2 + exit 1 + fi + echo "backup endpoint resolves to: $EPIP" + + # ---- OFF-CLUSTER PROOF (2): the endpoint is outside every cluster address -- + case "$EPIP" in 127.*|0.0.0.0) echo "FAIL: backup endpoint $EPIP is loopback -- not off-cluster" >&2; exit 1 ;; esac + + NODE_IPS="$(kubectl get nodes -o jsonpath='{range .items[*]}{range .status.addresses[*]}{.address}{"\n"}{end}{end}' 2>/dev/null || true)" + for a in $NODE_IPS; do + if [ "$a" = "$EPIP" ]; then + echo "FAIL: backup endpoint $EPIP is a NODE address of this cluster -- the backup is not off-cluster." >&2 + exit 1 + fi + done + echo "endpoint is not any node address (checked $(echo "$NODE_IPS" | grep -c . || true) node addresses)" + + SVC_IPS="$(kubectl get svc -A -o jsonpath='{range .items[*]}{.spec.clusterIP}{"\n"}{end}' 2>/dev/null || true)" + for a in $SVC_IPS; do + if [ "$a" = "$EPIP" ]; then + echo "FAIL: backup endpoint $EPIP is a Service ClusterIP -- the backup target lives INSIDE the cluster." >&2 + exit 1 + fi + done + echo "endpoint is not any Service ClusterIP (checked $(echo "$SVC_IPS" | grep -c . || true))" + + POD_IPS="$(kubectl get pods -A -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' 2>/dev/null || true)" + for a in $POD_IPS; do + if [ "$a" = "$EPIP" ]; then + echo "FAIL: backup endpoint $EPIP is a POD IP -- the backup target is an in-cluster pod." >&2 + exit 1 + fi + done + echo "endpoint is not any Pod IP (checked $(echo "$POD_IPS" | grep -c . || true))" + + POD_CIDRS="$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.podCIDR}{"\n"}{range .spec.podCIDRs[*]}{@}{"\n"}{end}{end}' 2>/dev/null || true)" + for c in $POD_CIDRS; do + case "$c" in *:*) continue ;; esac # IPv6 prefixes are out of scope here + if in_cidr "$EPIP" "$c"; then + echo "FAIL: backup endpoint $EPIP falls inside node podCIDR $c -- it is in the cluster pod network." >&2 + exit 1 + fi + done + echo "endpoint is outside every node podCIDR (checked: $(echo "$POD_CIDRS" | tr '\n' ' '))" + + # Best-effort (needs networking.k8s.io/ServiceCIDR, which the least-privilege + # runner may not read): the checks above are the authoritative ones. + SVC_CIDRS="$(kubectl get servicecidrs.networking.k8s.io -o jsonpath='{range .items[*]}{range .spec.cidrs[*]}{@}{"\n"}{end}{end}' 2>/dev/null || true)" + if [ -n "$SVC_CIDRS" ]; then + for c in $SVC_CIDRS; do + case "$c" in *:*) continue ;; esac + if in_cidr "$EPIP" "$c"; then + echo "FAIL: backup endpoint $EPIP falls inside the Service CIDR $c -- it is in the cluster." >&2 + exit 1 + fi + done + echo "endpoint is outside every ServiceCIDR ($(echo "$SVC_CIDRS" | tr '\n' ' '))" + else + echo "(ServiceCIDR objects unreadable -- skipping that best-effort check; ClusterIP/PodIP/podCIDR checks stand)" + fi + + echo "OFF_CLUSTER_TARGET_OK bsl=${BSL_NS}/${BSL_NAME} endpoint=${EPIP} bucket=${BSL_BUCKET}" + l.go:52: | 02:15:08 | dr-restore | assert-backup-target-is-off-cluster  | SCRIPT | LOG | + === STDOUT + backup storage locations: + velero default Available aws true https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com k3s-prod-backup + poll 1: velero readyReplicas=1 node-agent numberReady=3 + using BSL velero/default bucket=k3s-prod-backup endpoint=https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com + backup endpoint host: 7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com + backup endpoint resolves to: 172.64.66.1 + endpoint is not any node address (checked 6 node addresses) + endpoint is not any Service ClusterIP (checked 82) + endpoint is not any Pod IP (checked 197) + endpoint is outside every node podCIDR (checked: 10.42.0.0/24 10.42.0.0/24 10.42.1.0/24 10.42.1.0/24 10.42.2.0/24 10.42.2.0/24 ) + endpoint is outside every ServiceCIDR (10.43.0.0/16 ) + OFF_CLUSTER_TARGET_OK bsl=velero/default endpoint=172.64.66.1 bucket=k3s-prod-backup + l.go:52: | 02:15:08 | dr-restore | assert-backup-target-is-off-cluster  | SCRIPT | DONE | + l.go:52: | 02:15:08 | dr-restore | assert-backup-target-is-off-cluster  | TRY | END | + l.go:52: | 02:15:08 | dr-restore | assert-backups-are-scheduled  | TRY | BEGIN | + l.go:52: | 02:15:08 | dr-restore | assert-backups-are-scheduled  | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + SCHED="$(kubectl get schedules.velero.io -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" phase="}{.status.phase}{" cron="}{.spec.schedule}{" lastBackup="}{.status.lastBackup}{"\n"}{end}' 2>/dev/null || true)" + if [ -z "$(echo "$SCHED" | tr -d '[:space:]')" ]; then + # Distinguish "Velero absent" (skip, in lock-step with step 1) from + # "Velero present but nothing scheduled" (a real R-STATE-03 defect). + if kubectl get backupstoragelocations.velero.io -A >/dev/null 2>&1; then + if [ -z "$(kubectl get backupstoragelocations.velero.io -A -o name 2>/dev/null || true)" ]; then + echo "SKIP: no BackupStorageLocation configured -- scheduling not applicable yet" + echo DR_SCHEDULE_SKIPPED_OK + exit 0 + fi + echo "FAIL: Velero is configured with a backup target but declares NO Schedule --" >&2 + echo " R-STATE-03 requires SCHEDULED off-cluster backups, not manual ones." >&2 + exit 1 + fi + echo "SKIP: Velero API absent -- backup scheduling not applicable on this cluster yet" + echo DR_SCHEDULE_SKIPPED_OK + exit 0 + fi + echo "velero schedules:" + echo "$SCHED" + # A Schedule that exists but is paused/disabled backs nothing up. + if echo "$SCHED" | grep -q 'phase=Enabled'; then + echo BACKUPS_ARE_SCHEDULED_OK + else + echo "FAIL: no Velero Schedule is in phase 'Enabled' -- scheduled off-cluster backups are not running." >&2 + exit 1 + fi + l.go:52: | 02:15:08 | dr-restore | assert-backups-are-scheduled  | SCRIPT | LOG | + === STDOUT + velero schedules: + velero/velero-daily phase=Enabled cron=0 3 * * * lastBackup=2026-09-13T04:01:06Z + velero/velero-daily-resources-only phase=Enabled cron=0 3 * * * lastBackup=2026-09-13T03:00:03Z + BACKUPS_ARE_SCHEDULED_OK + l.go:52: | 02:15:08 | dr-restore | assert-backups-are-scheduled  | SCRIPT | DONE | + l.go:52: | 02:15:08 | dr-restore | assert-backups-are-scheduled  | TRY | END | + l.go:52: | 02:15:08 | dr-restore | data-restores-from-the-off-cluster-backup | TRY | BEGIN | + l.go:52: | 02:15:08 | dr-restore | data-restores-from-the-off-cluster-backup | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -eu + NS="$NAMESPACE" + SRC="${NS}-dr-src" # scratch source namespace (destroyed by design) + DST="${NS}-dr-dst" # clean restore target (must not pre-exist) + BK="${NS}-dr-backup" + RS="${NS}-dr-restore" + OWNER_LABEL="conformance.home.k3s/created-by=${NS}" + + # ---- Re-derive the preconditions (steps do not share shell state). Every + # skip below mirrors step 1 exactly, so the whole test skips or runs + # as a unit -- it can never half-run. ----------------------------- + if ! kubectl get storageclass longhorn >/dev/null 2>&1; then + echo "SKIP: 'longhorn' StorageClass absent"; echo DR_RESTORE_SKIPPED_OK; exit 0 + fi + BSL_LINES="$(kubectl get backupstoragelocations.velero.io -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.status.phase}{" "}{.spec.default}{"\n"}{end}' 2>/dev/null || true)" + CAND="$(echo "$BSL_LINES" | grep ' Available ' | grep ' true$' | head -1 || true)" + [ -n "$CAND" ] || CAND="$(echo "$BSL_LINES" | grep ' Available ' | head -1 || true)" + if [ -z "$CAND" ]; then + echo "SKIP: no Available BackupStorageLocation (see the off-cluster step for the authoritative verdict)" + echo DR_RESTORE_SKIPPED_OK; exit 0 + fi + VNS="$(echo "$CAND" | cut -d' ' -f1)" + BSL_NAME="$(echo "$CAND" | cut -d' ' -f2)" + if [ "$(kubectl -n "$VNS" get daemonset node-agent -o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0)" -lt 1 ]; then + echo "SKIP: Velero node-agent (kopia FSB) not ready -- volume data cannot be backed up" + echo DR_RESTORE_SKIPPED_OK; exit 0 + fi + echo "velero ns=$VNS bsl=$BSL_NAME src=$SRC dst=$DST" + + # A clean restore target means PREVIOUSLY NONEXISTENT, and the source name must + # be free. If either is lying around from an aborted run, remove it before we + # claim "clean" -- and fail loudly if it will not go, because a half-deleted + # namespace would silently invalidate the destroy proof below. + kubectl delete namespace "$DST" "$SRC" --ignore-not-found --wait=true --timeout=300s + + # ============================ GIVEN ============================== + # A DR-critical stateful workload with known data on the replicated tier. + CANARY="dr-canary-$(date +%s%N)" + kubectl create namespace "$SRC" + kubectl label namespace "$SRC" "$OWNER_LABEL" --overwrite + + kubectl -n "$SRC" apply -f - < /data/canary && sync' + echo "seeded canary: $(kubectl -n "$SRC" exec dr-workload -- cat /data/canary)" + + ORIG_PV="$(kubectl -n "$SRC" get pvc dr-data -o jsonpath='{.spec.volumeName}')" + [ -n "$ORIG_PV" ] || { echo "FAIL: source PVC never bound to a PV" >&2; exit 1; } + echo "original in-cluster PersistentVolume: $ORIG_PV" + + # ======================= WHEN (part 1): back up off-cluster ======= + # Created as a Velero Backup CR -- exactly what `velero backup create` does. + # defaultVolumesToFsBackup:true selects the kopia File System Backup path, so + # the VOLUME DATA (not just the resource JSON) is uploaded to the object store. + # ttl is short: if the finally-step DeleteBackupRequest is ever lost, Velero's + # own GC still removes this drill's objects from the bucket. + kubectl apply -f - </dev/null || true)" + echo "poll $i: backup phase=${BPHASE:-}" + case "${BPHASE:-}" in + Completed|Failed|PartiallyFailed|FailedValidation) break ;; + esac + sleep 10 + done + if [ "${BPHASE:-}" != "Completed" ]; then + echo "FAIL: off-cluster backup did not complete (phase=${BPHASE:-})" >&2 + kubectl -n "$VNS" get backups.velero.io "$BK" -o yaml >&2 || true + exit 1 + fi + echo "backup $BK Completed to ${BSL_NAME}" + + # ---- OFF-CLUSTER PROOF (3): the VOLUME DATA really landed in the remote + # kopia repository -- a PodVolumeBackup Completed with a snapshot ID. + PVB="$(kubectl -n "$VNS" get podvolumebackups.velero.io \ + -l velero.io/backup-name="$BK" \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.snapshotID}{"\n"}{end}' 2>/dev/null || true)" + echo "pod volume backups:" + echo "$PVB" + if [ -z "$(echo "$PVB" | tr -d '[:space:]')" ]; then + echo "FAIL: the backup captured NO pod volume data (no PodVolumeBackup)." >&2 + echo " Resource-only backups cannot restore DR-critical DATA." >&2 + exit 1 + fi + # A line is " "; require Completed AND a + # non-empty snapshot ID (an empty ID means nothing reached the repository). + GOOD_PVB="$(echo "$PVB" | grep -cE ' Completed [^[:space:]]+$' || true)" + if [ "${GOOD_PVB:-0}" -lt 1 ]; then + echo "FAIL: no PodVolumeBackup reached phase Completed with a snapshot ID --" >&2 + echo " the volume data did not reach the off-cluster repository." >&2 + exit 1 + fi + echo "volume data uploaded off-cluster (PodVolumeBackup Completed with a kopia snapshotID)" + + # ============ WHEN (part 2): FAULT INJECTION -- annihilate the source ==== + # Destroy EVERY in-cluster copy: the pod, the PVC, the namespace, and the + # underlying PersistentVolume (and with it the Longhorn replicas). After this + # there is no in-cluster source the restore could quietly read from. + kubectl delete namespace "$SRC" --wait=true --timeout=420s + for i in $(seq 1 30); do + if ! kubectl get namespace "$SRC" >/dev/null 2>&1; then break; fi + echo "poll $i: waiting for namespace $SRC to disappear" + sleep 10 + done + if kubectl get namespace "$SRC" >/dev/null 2>&1; then + echo "FAIL: source namespace $SRC did not terminate; the destroy step is not convincing" >&2 + exit 1 + fi + echo "source namespace destroyed" + + # The longhorn StorageClass reclaims Delete, so the PV should already be gone; + # force it if a Retain policy (or a stuck finalizer) left it behind, because a + # surviving PV would make the restore prove nothing about off-cluster recovery. + kubectl delete pv "$ORIG_PV" --ignore-not-found --wait=false || true + for i in $(seq 1 30); do + if ! kubectl get pv "$ORIG_PV" >/dev/null 2>&1; then break; fi + echo "poll $i: waiting for PersistentVolume $ORIG_PV to disappear" + sleep 10 + done + if kubectl get pv "$ORIG_PV" >/dev/null 2>&1; then + echo "FAIL: the original PersistentVolume $ORIG_PV SURVIVED the destroy step." >&2 + echo " An in-cluster copy of the data still exists, so a successful restore" >&2 + echo " would NOT prove recovery from the off-cluster backup. Refusing to pass." >&2 + exit 1 + fi + echo "original PersistentVolume $ORIG_PV is GONE -- no in-cluster copy of the data remains" + + # Best-effort corroboration (the least-privilege runner may not read longhorn.io): + # the Longhorn Volume CR and its replicas should be gone too. + if kubectl -n longhorn-system get volumes.longhorn.io "$ORIG_PV" >/dev/null 2>&1; then + echo "FAIL: Longhorn Volume $ORIG_PV still exists after PV deletion -- replicas survive in-cluster" >&2 + exit 1 + fi + echo "(longhorn Volume CR $ORIG_PV not present -- either deleted or not readable by this identity)" + + # ============================ RESTORE ============================ + # Into a CLEAN, previously-nonexistent namespace (Velero creates it), which is + # the catalog's "restored into a clean state". + kubectl apply -f - </dev/null || true)" + echo "poll $i: restore phase=${RPHASE:-}" + case "${RPHASE:-}" in + Completed|Failed|PartiallyFailed|FailedValidation) break ;; + esac + sleep 10 + done + if [ "${RPHASE:-}" != "Completed" ]; then + echo "FAIL: restore did not complete (phase=${RPHASE:-})" >&2 + kubectl -n "$VNS" get restores.velero.io "$RS" -o yaml >&2 || true + kubectl -n "$VNS" get podvolumerestores.velero.io -l velero.io/restore-name="$RS" -o wide >&2 || true + exit 1 + fi + echo "restore $RS Completed into $DST" + + # ============================= THEN ============================== + # The service returns WITH ITS DATA. With kopia FSB the restored pod carries a + # `restore-wait` init container that blocks until the node-agent has written the + # volume contents back, so Ready here already implies the data landed. + if ! kubectl -n "$DST" wait --for=condition=Ready pod/dr-workload --timeout=600s; then + echo "FAIL: restored workload never became Ready in the clean namespace $DST" >&2 + kubectl -n "$DST" describe pod dr-workload >&2 || true + kubectl -n "$VNS" get podvolumerestores.velero.io -l velero.io/restore-name="$RS" -o wide >&2 || true + exit 1 + fi + + NEW_PV="$(kubectl -n "$DST" get pvc dr-data -o jsonpath='{.spec.volumeName}' 2>/dev/null || true)" + echo "restored PVC bound to PersistentVolume: ${NEW_PV:-}" + [ -n "$NEW_PV" ] || { echo "FAIL: restored PVC did not bind a PV" >&2; exit 1; } + if [ "$NEW_PV" = "$ORIG_PV" ]; then + echo "FAIL: the restored PVC rebound the ORIGINAL PersistentVolume ($ORIG_PV)." >&2 + echo " That is an in-cluster survival, not an off-cluster restore." >&2 + exit 1 + fi + echo "restored onto a NEW volume ($NEW_PV != $ORIG_PV) -- the data came from outside the cluster" + + GOT="$(kubectl -n "$DST" exec dr-workload -- cat /data/canary)" + echo "recovered='$GOT' expected='$CANARY'" + if [ "$GOT" != "$CANARY" ]; then + echo "FAIL: restored data does not match what was backed up -- DR restore lost/corrupted data" >&2 + exit 1 + fi + + echo DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK + l.go:52: | 02:17:18 | dr-restore | data-restores-from-the-off-cluster-backup | SCRIPT | LOG | + === STDOUT + velero ns=velero bsl=default src=chainsaw-glorious-sheep-dr-src dst=chainsaw-glorious-sheep-dr-dst + namespace/chainsaw-glorious-sheep-dr-src created + namespace/chainsaw-glorious-sheep-dr-src labeled + persistentvolumeclaim/dr-data created + pod/dr-workload created + pod/dr-workload condition met + seeded canary: dr-canary-1789352108935754215 + original in-cluster PersistentVolume: pvc-4107d59d-a34f-4cba-bd2a-c5b2446156d9 + backup.velero.io/chainsaw-glorious-sheep-dr-backup created + poll 1: backup phase=InProgress + poll 2: backup phase=InProgress + poll 3: backup phase=InProgress + poll 4: backup phase=Completed + backup chainsaw-glorious-sheep-dr-backup Completed to default + pod volume backups: + chainsaw-glorious-sheep-dr-backup-2gjrq Completed 979eed437233b45c7f85ba80074ab56b + volume data uploaded off-cluster (PodVolumeBackup Completed with a kopia snapshotID) + namespace "chainsaw-glorious-sheep-dr-src" deleted + source namespace destroyed + original PersistentVolume pvc-4107d59d-a34f-4cba-bd2a-c5b2446156d9 is GONE -- no in-cluster copy of the data remains + (longhorn Volume CR pvc-4107d59d-a34f-4cba-bd2a-c5b2446156d9 not present -- either deleted or not readable by this identity) + restore.velero.io/chainsaw-glorious-sheep-dr-restore created + poll 1: restore phase=InProgress + poll 2: restore phase=InProgress + poll 3: restore phase=InProgress + poll 4: restore phase=Completed + restore chainsaw-glorious-sheep-dr-restore Completed into chainsaw-glorious-sheep-dr-dst + pod/dr-workload condition met + restored PVC bound to PersistentVolume: pvc-f0c061e1-c080-49f2-a381-23da5c886df2 + restored onto a NEW volume (pvc-f0c061e1-c080-49f2-a381-23da5c886df2 != pvc-4107d59d-a34f-4cba-bd2a-c5b2446156d9) -- the data came from outside the cluster + recovered='dr-canary-1789352108935754215' expected='dr-canary-1789352108935754215' + DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK + === STDERR + Defaulted container "app" out of: app, restore-wait (init) + l.go:52: | 02:17:18 | dr-restore | data-restores-from-the-off-cluster-backup | SCRIPT | DONE | + l.go:52: | 02:17:18 | dr-restore | data-restores-from-the-off-cluster-backup | TRY | END | + l.go:52: | 02:17:18 | dr-restore | data-restores-from-the-off-cluster-backup | FINALLY | BEGIN | + l.go:52: | 02:17:18 | dr-restore | data-restores-from-the-off-cluster-backup | CMD | RUN | + === COMMAND + /usr/bin/sh -c set -u + NS="$NAMESPACE" + SRC="${NS}-dr-src"; DST="${NS}-dr-dst"; BK="${NS}-dr-backup"; RS="${NS}-dr-restore" + + VNS="$(kubectl get backupstoragelocations.velero.io -A -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null || true)" + + # Velero objects first: a DeleteBackupRequest makes Velero remove the backup + # DATA from the off-cluster bucket, not just the CR (self-cleaning contract). + if [ -n "$VNS" ]; then + kubectl -n "$VNS" delete restores.velero.io "$RS" --ignore-not-found --wait=false || true + if kubectl -n "$VNS" get backups.velero.io "$BK" >/dev/null 2>&1; then + kubectl apply -f - </dev/null 2>&1 || break + sleep 10 + done + # Backstop if Velero could not process the request (the 2h ttl also GCs it). + kubectl -n "$VNS" delete backups.velero.io "$BK" --ignore-not-found --wait=false || true + kubectl -n "$VNS" delete deletebackuprequests.velero.io "${BK}-delete" --ignore-not-found --wait=false || true + fi + echo "velero objects for $NS cleaned up (ns $VNS)" + fi + + # Scratch namespaces (both, in case the run aborted at any point). + for n in "$DST" "$SRC"; do + kubectl delete namespace "$n" --ignore-not-found --wait=false || true + done + # Anything left labelled as ours (defensive; the two names above are the only + # namespaces this test ever creates). + kubectl delete namespace -l "conformance.home.k3s/created-by=${NS}" --ignore-not-found --wait=false || true + + # Released PVs from either namespace's PVCs would otherwise leak storage. + for pv in $(kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.claimRef.namespace}{"\n"}{end}' 2>/dev/null | grep -E " (${SRC}|${DST})$" | cut -d' ' -f1 || true); do + kubectl delete pv "$pv" --ignore-not-found --wait=false || true + echo "released leftover PersistentVolume: $pv" + done + + echo "dr-restore teardown complete (best-effort; every object was created by this test)" + l.go:52: | 02:17:30 | dr-restore | data-restores-from-the-off-cluster-backup | SCRIPT | LOG | + === STDOUT + restore.velero.io "chainsaw-glorious-sheep-dr-restore" deleted from velero namespace + deletebackuprequest.velero.io/chainsaw-glorious-sheep-dr-backup-delete created + velero objects for chainsaw-glorious-sheep cleaned up (ns velero) + namespace "chainsaw-glorious-sheep-dr-dst" deleted + namespace "chainsaw-glorious-sheep-dr-dst" deleted + persistentvolume "pvc-f0c061e1-c080-49f2-a381-23da5c886df2" deleted + released leftover PersistentVolume: pvc-f0c061e1-c080-49f2-a381-23da5c886df2 + dr-restore teardown complete (best-effort; every object was created by this test) + l.go:52: | 02:17:30 | dr-restore | data-restores-from-the-off-cluster-backup | SCRIPT | DONE | + l.go:52: | 02:17:30 | dr-restore | data-restores-from-the-off-cluster-backup | FINALLY | END | + l.go:52: | 02:17:30 | dr-restore | @chainsaw  | CLEANUP | BEGIN | + l.go:52: | 02:17:30 | dr-restore | @chainsaw  | DELETE | OK | v1/Namespace @ chainsaw-glorious-sheep + l.go:52: | 02:17:36 | dr-restore | @chainsaw  | CLEANUP | END | +--- PASS: chainsaw (0.00s) + --- PASS: chainsaw/dr-restore (150.70s) +PASS +Tests Summary... +- Passed tests 1 +- Failed tests 0 +- Skipped tests 0 +Done. +merge_junit: merged 1 report(s) -> /work/conformance/_artifacts/report.junit.xml (tests=1 failures=0 errors=0 skipped=0) +[run.sh] Done. JUnit: /work/conformance/_artifacts/report.junit.xml | Coverage: /work/conformance/_artifacts/coverage/coverage.md diff --git a/conformance/runs/ledger.jsonl b/conformance/runs/ledger.jsonl index 1848693..9f9a864 100644 --- a/conformance/runs/ledger.jsonl +++ b/conformance/runs/ledger.jsonl @@ -3,3 +3,4 @@ {"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L1", "note": "L1 offline check exited 0; run_l1_offline.py renders its skips as JUnit , so a green JUnit is truthful without a sentinel", "proof": "proven", "requirements": ["R-NET-05", "R-NET-06", "R-NET-07"], "run_id": "20260913T014321Z-19fc8bc", "sentinel": null, "test": "egress-class-render", "ts_utc": "2026-09-13T01:43:21Z"} {"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6-dirty", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014437Z-19fc8bc", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:44:37Z"} {"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "9d0814ae61e6a0ba64d3789a71f1cd9c985efbf0", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014923Z-9d0814a", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:49:23Z"} +{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "24b8111d0c39e9417e9f4d782f333f683540cfa7", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (OFF_CLUSTER_TARGET_OK, BACKUPS_ARE_SCHEDULED_OK, DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-STATE-03"], "run_id": "20260914T021505Z-24b8111", "sentinel": "DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK", "test": "dr-restore", "ts_utc": "2026-09-14T02:15:05Z"}