conformance: serialize heavy L2/L3 tests + author alerting SMTP sink

TASK 1 — reliably single-pass green L2/L3: Split run_chainsaw into a LIGHT pass (concurrent, config parallelism) and a HEAVY pass (serial, parallel=1). The KubeVirt-CDI/Longhorn/DR tests (vm-lifecycle, vm-data-survival, vm-default-deny, data-survival, dr-restore) spike embedded-etcd on the small dev VMs; running them serially in their own pass keeps their etcd spikes from ever coinciding with another test's setup — the coincidence that let one heavy test's apiserver blip fail-close Kyverno admission and abort every in-flight test at 0.01s. Heavy set is keyed by test dir basename and overridable via CONF_HEAVY_TESTS. Each pass writes its own JUnit; lib/merge_junit.py stitches them into the one stable report.junit.xml CI collects, so the split is invisible downstream. Overall rc is the union of both passes (no masking — real failures still surface). Proof: full --layer l2,l3 run now completes single-pass green: 24/24 passed, 0 failures/errors/skipped. vpn-killswitch, which previously failed with an apiserver EOF (etcd blip), passed; no mass environmental setup-aborts. TASK 2 (T-4.1) — author conformance/tests/l3-fault-injection/alerting/ fixtures/smtp-sink.yaml: a hermetic Mailpit SMTP capture sink (Service smtp-sink, SMTP 1025, HTTP API 8025) the alerting test applies as its mail sink. restricted-PodSecurity compliant, in-memory (no PVC). Unblocks the previously-aborting alerting scaffold; it now passes. Touches conformance/ + ci-read content only — no apps/ or platform/ workload. Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-08-26 02:56:56 -05:00
1 parent a02c0e2d1a
commit 948c390272
@@ -137,7 +137,56 @@ select_dirs() {
  fi
 }

+# --- HEAVY tests: the storage/VM/DR tests whose KubeVirt-CDI imports + Longhorn volume ops
+#     spike embedded-etcd I/O on the small dev VMs. Run SERIALLY (parallel=1) in their own
+#     pass so their etcd spikes never coincide with ANOTHER test's setup — the coincidence
+#     that made a single heavy test's apiserver blip fail-close Kyverno admission and abort
+#     every in-flight test's 0.01s setup (see config/chainsaw-config.yaml). Keyed by the test
+#     DIRECTORY basename (stable); override via CONF_HEAVY_TESTS to add/remove without code
+#     changes. Listed here even for tests still scaffolded (vm-lifecycle, dr-restore, ...) so
+#     the serialization is already correct the moment they grow real KubeVirt/Longhorn bodies.
+CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore}"
+
+conf_is_heavy_dir() {
+  # $1 = test dir; true if its basename is in the heavy set.
+  local base t
+  base="$(basename "$1")"
+  for t in ${CONF_HEAVY_TESTS}; do [ "$base" = "$t" ] && return 0; done
+  return 1
+}
+
+# One Chainsaw invocation over a specific set of dirs, at a given parallelism, writing to a
+# named report file. Returns chainsaw's rc (0=all passed) WITHOUT aborting run.sh (callers
+# collect both passes' rc so a light-pass failure still lets the heavy pass run, and vice
+# versa). failFast stays false (config) so every test in the pass runs regardless.
+run_chainsaw_pass() {
+  local report_name="$1" parallel="$2"; shift 2
+  local -a dirs=("$@")
+  local -a test_dir_args=()
+  local d
+  for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done
+  # Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts).
+  # --report-path is passed ABSOLUTE (overriding the config's relative `_artifacts`) so the
+  # report write succeeds regardless of the caller's CWD. --report-name/--parallel override
+  # the config per-pass so the two passes don't clobber each other's report and the heavy
+  # pass forces serial execution.
+  local rc=0
+  chainsaw test \
+    --config "${CONF_ROOT}/config/chainsaw-config.yaml" \
+    --report-path "${ARTIFACTS}" \
+    --report-name "${report_name}" \
+    --parallel "${parallel}" \
+    "${test_dir_args[@]}" || rc=$?
+  return $rc
+}
+
 # --- Execute the Chainsaw suite against a live cluster ---------------------------------
+# Split into a LIGHT pass (concurrent, config parallelism) and a HEAVY pass (serial,
+# parallel=1). Robust single-pass green: the heavy etcd spikes are isolated in time from the
+# light tests' setup, so a heavy test's transient apiserver blip can no longer cascade into
+# mass fail-closed setup-aborts of unrelated tests. The two JUnit reports are stitched back
+# into the one stable artifact CI collects (report.junit.xml) via lib/merge_junit.py, so the
+# split is invisible to everything downstream.
 run_chainsaw() {
  local -a dirs
  mapfile -t dirs < <(select_dirs)
@@ -145,28 +194,52 @@ run_chainsaw() {
    log "No tests match the selection (layer=${LAYER} reqs='${REQS}')."
    return 0
  fi
-  local -a test_dir_args=()
-  for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done

-  log "Running Chainsaw over ${#dirs[@]} test dir(s) with pinned ${CHAINSAW_VERSION}"
-  # Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts).
-  # --report-path is passed ABSOLUTE (overriding the config's relative `_artifacts`) so the report
-  # write succeeds regardless of the caller's CWD — otherwise chainsaw's relative path resolves
-  # against $PWD and fails ("open _artifacts/chainsaw-report.xml: no such file") when run.sh is
-  # invoked from the repo root instead of conformance/.
+  local -a light=() heavy=()
+  local d
+  for d in "${dirs[@]}"; do
+    if conf_is_heavy_dir "$d"; then heavy+=("$d"); else light+=("$d"); fi
+  done
+
  mkdir -p "${ARTIFACTS}"
-  chainsaw test \
-    --config "${CONF_ROOT}/config/chainsaw-config.yaml" \
-    --report-path "${ARTIFACTS}" \
-    "${test_dir_args[@]}"
+  local light_report="${ARTIFACTS}/chainsaw-report-light.xml"
+  local heavy_report="${ARTIFACTS}/chainsaw-report-heavy.xml"
+  rm -f "${light_report}" "${heavy_report}"
+  local light_rc=0 heavy_rc=0

-  # Normalize whatever chainsaw named the report to a stable path CI collects.
-  local produced
-  produced="$(find "${ARTIFACTS}" -maxdepth 1 -name 'chainsaw-report*.xml' -o \
-                   -name 'report*.xml' 2>/dev/null | grep -v report.junit.xml | head -1 || true)"
-  if [ -n "${produced}" ] && [ "${produced}" != "${JUNIT_OUT}" ]; then
-    cp "${produced}" "${JUNIT_OUT}"
+  # Determine the light pass's parallelism from the shared config (single source of truth),
+  # falling back to 2 if unreadable — so bumping the config on prod-class hw needs no run.sh edit.
+  local light_parallel
+  light_parallel="$(sed -nE 's/^[[:space:]]*parallel:[[:space:]]*([0-9]+).*/\1/p' \
+                      "${CONF_ROOT}/config/chainsaw-config.yaml" | head -1)"
+  [ -n "${light_parallel}" ] || light_parallel=2
+
+  if [ "${#light[@]}" -gt 0 ]; then
+    log "LIGHT pass: ${#light[@]} test(s), parallel=${light_parallel} (pinned ${CHAINSAW_VERSION})"
+    run_chainsaw_pass "chainsaw-report-light" "${light_parallel}" "${light[@]}" || light_rc=$?
+  else
+    log "LIGHT pass: no tests selected."
  fi
+
+  if [ "${#heavy[@]}" -gt 0 ]; then
+    log "HEAVY pass: ${#heavy[@]} test(s) SERIAL (parallel=1) — ${heavy[*]##*/}"
+    run_chainsaw_pass "chainsaw-report-heavy" 1 "${heavy[@]}" || heavy_rc=$?
+  else
+    log "HEAVY pass: no heavy tests selected."
+  fi
+
+  # Stitch both passes' JUnit into the one stable artifact CI collects.
+  "${PYTHON}" "${CONF_ROOT}/lib/merge_junit.py" \
+    --out "${JUNIT_OUT}" --name chainsaw-report \
+    "${light_report}" "${heavy_report}"
+
+  # Overall result is the union: fail if EITHER pass had a failing/erroring test. Real
+  # failures still surface (nothing is masked); the split only changes WHEN heavy tests run.
+  if [ "${light_rc}" -ne 0 ] || [ "${heavy_rc}" -ne 0 ]; then
+    log "Chainsaw reported failures (light rc=${light_rc}, heavy rc=${heavy_rc})."
+    return 1
+  fi
+  return 0
 }

 # --- L1 rendering: cluster-INDEPENDENT render checks (docs/14) --------------------------