diff --git a/conformance/lib/merge_junit.py b/conformance/lib/merge_junit.py new file mode 100644 index 0000000..d488d0d --- /dev/null +++ b/conformance/lib/merge_junit.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Merge several Chainsaw JUnit reports into one. + +run.sh splits the L2/L3 run into a LIGHT pass (concurrent) and a HEAVY pass +(serial, parallel=1) so the storage/VM-heavy tests never spike embedded-etcd +while other tests are mid-setup (see run.sh `run_chainsaw`). Each pass writes its +own JUnit file; this stitches them back into the single stable artifact CI +collects (`_artifacts/report.junit.xml`) so the split is invisible downstream. + +Chainsaw emits `` with one `` per test. Merging is a +concatenation of the child `` elements plus a recomputed roll-up of +tests/failures/errors/time on the root, so the combined report totals exactly the +sum of the passes (no double counting, no dropped cases). Inputs that don't exist +are skipped (a pass may legitimately select zero tests); at least one must exist. +""" +from __future__ import annotations + +import argparse +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +def _int(el: ET.Element, attr: str) -> int: + try: + return int(el.get(attr, "0") or "0") + except ValueError: + return 0 + + +def _float(el: ET.Element, attr: str) -> float: + try: + return float(el.get(attr, "0") or "0") + except ValueError: + return 0.0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--out", type=Path, required=True, help="merged JUnit path") + ap.add_argument("--name", default="chainsaw-report", help="root testsuites name") + ap.add_argument("inputs", nargs="+", type=Path, help="JUnit files to merge (missing ones skipped)") + args = ap.parse_args() + + merged = ET.Element("testsuites", {"name": args.name}) + tests = failures = errors = skipped = 0 + total_time = 0.0 + found = 0 + + for path in args.inputs: + if not path.exists() or path.stat().st_size == 0: + continue + found += 1 + root = ET.parse(path).getroot() + # Accept either a wrapper or a bare . + suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) + for suite in suites: + merged.append(suite) + tests += _int(suite, "tests") + failures += _int(suite, "failures") + errors += _int(suite, "errors") + skipped += _int(suite, "skipped") + total_time += _float(suite, "time") + + if found == 0: + print(f"merge_junit: none of the input reports exist: {args.inputs}", file=sys.stderr) + return 1 + + merged.set("tests", str(tests)) + merged.set("failures", str(failures)) + merged.set("errors", str(errors)) + if skipped: + merged.set("skipped", str(skipped)) + merged.set("time", f"{total_time:.6f}") + + args.out.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(merged).write(args.out, encoding="unicode", xml_declaration=False) + print(f"merge_junit: merged {found} report(s) -> {args.out} " + f"(tests={tests} failures={failures} errors={errors} skipped={skipped})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/conformance/run.sh b/conformance/run.sh index bea1715..8a4d84e 100755 --- a/conformance/run.sh +++ b/conformance/run.sh @@ -137,7 +137,56 @@ select_dirs() { fi } +# --- HEAVY tests: the storage/VM/DR tests whose KubeVirt-CDI imports + Longhorn volume ops +# spike embedded-etcd I/O on the small dev VMs. Run SERIALLY (parallel=1) in their own +# pass so their etcd spikes never coincide with ANOTHER test's setup — the coincidence +# that made a single heavy test's apiserver blip fail-close Kyverno admission and abort +# every in-flight test's 0.01s setup (see config/chainsaw-config.yaml). Keyed by the test +# DIRECTORY basename (stable); override via CONF_HEAVY_TESTS to add/remove without code +# changes. Listed here even for tests still scaffolded (vm-lifecycle, dr-restore, ...) so +# the serialization is already correct the moment they grow real KubeVirt/Longhorn bodies. +CONF_HEAVY_TESTS="${CONF_HEAVY_TESTS:-vm-lifecycle vm-data-survival vm-default-deny data-survival dr-restore}" + +conf_is_heavy_dir() { + # $1 = test dir; true if its basename is in the heavy set. + local base t + base="$(basename "$1")" + for t in ${CONF_HEAVY_TESTS}; do [ "$base" = "$t" ] && return 0; done + return 1 +} + +# One Chainsaw invocation over a specific set of dirs, at a given parallelism, writing to a +# named report file. Returns chainsaw's rc (0=all passed) WITHOUT aborting run.sh (callers +# collect both passes' rc so a light-pass failure still lets the heavy pass run, and vice +# versa). failFast stays false (config) so every test in the pass runs regardless. +run_chainsaw_pass() { + local report_name="$1" parallel="$2"; shift 2 + local -a dirs=("$@") + local -a test_dir_args=() + local d + for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done + # Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts). + # --report-path is passed ABSOLUTE (overriding the config's relative `_artifacts`) so the + # report write succeeds regardless of the caller's CWD. --report-name/--parallel override + # the config per-pass so the two passes don't clobber each other's report and the heavy + # pass forces serial execution. + local rc=0 + chainsaw test \ + --config "${CONF_ROOT}/config/chainsaw-config.yaml" \ + --report-path "${ARTIFACTS}" \ + --report-name "${report_name}" \ + --parallel "${parallel}" \ + "${test_dir_args[@]}" || rc=$? + return $rc +} + # --- Execute the Chainsaw suite against a live cluster --------------------------------- +# Split into a LIGHT pass (concurrent, config parallelism) and a HEAVY pass (serial, +# parallel=1). Robust single-pass green: the heavy etcd spikes are isolated in time from the +# light tests' setup, so a heavy test's transient apiserver blip can no longer cascade into +# mass fail-closed setup-aborts of unrelated tests. The two JUnit reports are stitched back +# into the one stable artifact CI collects (report.junit.xml) via lib/merge_junit.py, so the +# split is invisible to everything downstream. run_chainsaw() { local -a dirs mapfile -t dirs < <(select_dirs) @@ -145,28 +194,52 @@ run_chainsaw() { log "No tests match the selection (layer=${LAYER} reqs='${REQS}')." return 0 fi - local -a test_dir_args=() - for d in "${dirs[@]}"; do test_dir_args+=(--test-dir "$d"); done - log "Running Chainsaw over ${#dirs[@]} test dir(s) with pinned ${CHAINSAW_VERSION}" - # Chainsaw writes JUnit per config/chainsaw-config.yaml (report.* under _artifacts). - # --report-path is passed ABSOLUTE (overriding the config's relative `_artifacts`) so the report - # write succeeds regardless of the caller's CWD — otherwise chainsaw's relative path resolves - # against $PWD and fails ("open _artifacts/chainsaw-report.xml: no such file") when run.sh is - # invoked from the repo root instead of conformance/. + local -a light=() heavy=() + local d + for d in "${dirs[@]}"; do + if conf_is_heavy_dir "$d"; then heavy+=("$d"); else light+=("$d"); fi + done + mkdir -p "${ARTIFACTS}" - chainsaw test \ - --config "${CONF_ROOT}/config/chainsaw-config.yaml" \ - --report-path "${ARTIFACTS}" \ - "${test_dir_args[@]}" + local light_report="${ARTIFACTS}/chainsaw-report-light.xml" + local heavy_report="${ARTIFACTS}/chainsaw-report-heavy.xml" + rm -f "${light_report}" "${heavy_report}" + local light_rc=0 heavy_rc=0 - # Normalize whatever chainsaw named the report to a stable path CI collects. - local produced - produced="$(find "${ARTIFACTS}" -maxdepth 1 -name 'chainsaw-report*.xml' -o \ - -name 'report*.xml' 2>/dev/null | grep -v report.junit.xml | head -1 || true)" - if [ -n "${produced}" ] && [ "${produced}" != "${JUNIT_OUT}" ]; then - cp "${produced}" "${JUNIT_OUT}" + # Determine the light pass's parallelism from the shared config (single source of truth), + # falling back to 2 if unreadable — so bumping the config on prod-class hw needs no run.sh edit. + local light_parallel + light_parallel="$(sed -nE 's/^[[:space:]]*parallel:[[:space:]]*([0-9]+).*/\1/p' \ + "${CONF_ROOT}/config/chainsaw-config.yaml" | head -1)" + [ -n "${light_parallel}" ] || light_parallel=2 + + if [ "${#light[@]}" -gt 0 ]; then + log "LIGHT pass: ${#light[@]} test(s), parallel=${light_parallel} (pinned ${CHAINSAW_VERSION})" + run_chainsaw_pass "chainsaw-report-light" "${light_parallel}" "${light[@]}" || light_rc=$? + else + log "LIGHT pass: no tests selected." fi + + if [ "${#heavy[@]}" -gt 0 ]; then + log "HEAVY pass: ${#heavy[@]} test(s) SERIAL (parallel=1) — ${heavy[*]##*/}" + run_chainsaw_pass "chainsaw-report-heavy" 1 "${heavy[@]}" || heavy_rc=$? + else + log "HEAVY pass: no heavy tests selected." + fi + + # Stitch both passes' JUnit into the one stable artifact CI collects. + "${PYTHON}" "${CONF_ROOT}/lib/merge_junit.py" \ + --out "${JUNIT_OUT}" --name chainsaw-report \ + "${light_report}" "${heavy_report}" + + # Overall result is the union: fail if EITHER pass had a failing/erroring test. Real + # failures still surface (nothing is masked); the split only changes WHEN heavy tests run. + if [ "${light_rc}" -ne 0 ] || [ "${heavy_rc}" -ne 0 ]; then + log "Chainsaw reported failures (light rc=${light_rc}, heavy rc=${heavy_rc})." + return 1 + fi + return 0 } # --- L1 rendering: cluster-INDEPENDENT render checks (docs/14) -------------------------- diff --git a/conformance/tests/l3-fault-injection/alerting/fixtures/smtp-sink.yaml b/conformance/tests/l3-fault-injection/alerting/fixtures/smtp-sink.yaml new file mode 100644 index 0000000..f500465 --- /dev/null +++ b/conformance/tests/l3-fault-injection/alerting/fixtures/smtp-sink.yaml @@ -0,0 +1,121 @@ +# ============================================================================= +# T-4.1 — Hermetic SMTP capture sink for the L3 "alerting" conformance test +# (R-OBS-02, R-OBS-04). +# +# The alerting test injects a fault, Alertmanager fires and routes an email, and +# the test asserts the email arrived. To keep the test hermetic (no real inbox, +# ADR-0011 reproducibility), Alertmanager's email route is pointed at THIS +# in-namespace sink instead of a real MTA. The sink is Mailpit: it speaks SMTP +# for delivery and exposes an HTTP API the test polls for the captured message. +# +# Contract the alerting test relies on (chainsaw injects the ephemeral namespace, +# so these are intentionally namespace-less): +# * Service name .............. smtp-sink (stable DNS the test/AM target) +# * SMTP port (delivery) ...... 1025 (Alertmanager `smarthost`) +# * HTTP port (query/API) ..... 8025 (test polls for the message) +# * Query endpoint ............ GET http://smtp-sink:8025/api/v1/messages +# -> JSON: .messages[].Subject / .To / .From +# (health: GET /api/v1/messages returns 200 once the sink is up; a delivered +# alert appears as a message whose Subject matches the fired alertname.) +# +# Mailpit stores messages in memory (no PVC, no persistence) — a leaked namespace +# leaves nothing behind, and chainsaw's per-test namespace teardown removes it. +# ============================================================================= +apiVersion: apps/v1 +kind: Deployment +metadata: + name: smtp-sink + labels: + app.kubernetes.io/name: smtp-sink + app.kubernetes.io/part-of: conformance-alerting + conformance.home.k3s/fixture: smtp-sink +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: smtp-sink + template: + metadata: + labels: + app.kubernetes.io/name: smtp-sink + app.kubernetes.io/part-of: conformance-alerting + spec: + # Hermetic, disposable capture sink — no state worth protecting. + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: mailpit + image: axllent/mailpit:v1.21.8 + imagePullPolicy: IfNotPresent + env: + # Bind SMTP + HTTP UI/API on the well-known conformance ports. + - name: MP_SMTP_BIND_ADDR + value: "0.0.0.0:1025" + - name: MP_UI_BIND_ADDR + value: "0.0.0.0:8025" + # In-memory store: no disk, cleared on pod restart (hermetic per-run). + - name: MP_MAX_MESSAGES + value: "500" + ports: + - name: smtp + containerPort: 1025 + protocol: TCP + - name: http + containerPort: 8025 + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 2 + periodSeconds: 3 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /livez + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: smtp-sink + labels: + app.kubernetes.io/name: smtp-sink + app.kubernetes.io/part-of: conformance-alerting +spec: + selector: + app.kubernetes.io/name: smtp-sink + ports: + - name: smtp + port: 1025 + targetPort: smtp + protocol: TCP + - name: http + port: 8025 + targetPort: http + protocol: TCP