#!/usr/bin/env bash # platform/monitoring/tests/run.sh — Prometheus RULE-UNIT-TEST harness (no cluster). # # WHY THIS EXISTS # This repo has now shipped both halves of the alerting failure space to prod: # * a PERMANENTLY-FIRING rule (the CNPG alert fixed 2026-09-12), and # * a STRUCTURALLY INERT rule (VeleroNoRecentBackup: `time() - ` yields no # series, so a schedule that had NEVER backed up could not raise it — prod ran with no # volume backups at all and nothing alerted). # Neither is visible by reading the expression, by `kubectl apply`, or by looking at a healthy # live cluster: an inert rule and a correct rule are indistinguishable when everything is fine. # `promtool test rules` is the tool that separates them, because it lets the BAD state be # constructed on purpose and asserted on, offline and in CI. Every rule here is therefore # pinned twice: quiet on the good state, firing on the bad one. # # The rules under test are the COMMITTED manifests (extract_rules.py flattens the # PrometheusRule CRs into a rule file) — never a copy, so a fixture cannot drift green while # the deployed rule rots. # # Usage: ./run.sh # run every *.test.yaml here # ./run.sh foo.test.yaml # # Exit codes follow ci/lib.sh: 0 pass, 1 fail, 3 SKIPPED (toolchain unavailable — never green). # promtool is taken from $PATH if present, otherwise run from the pinned Prometheus image via # podman/docker. The image tag matches the version prod runs, so "passes here" means "passes # with the engine that will evaluate it". set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MONITORING_DIR="$(cd "${HERE}/.." && pwd)" PROM_IMAGE="${PROM_IMAGE:-quay.io/prometheus/prometheus:v3.14.0}" MANIFESTS=( "${MONITORING_DIR}/alerts.yaml" "${MONITORING_DIR}/pvc-capacity-alerts.yaml" ) if [ "$#" -gt 0 ]; then TESTS=("$@") else mapfile -t TESTS < <(find "${HERE}" -maxdepth 1 -name '*.test.yaml' | sort) fi if [ "${#TESTS[@]}" -eq 0 ]; then echo "run.sh: no *.test.yaml found in ${HERE}" >&2 exit 3 fi # ---------------------------------------------------------------- toolchain resolution RUNNER="" if command -v promtool >/dev/null 2>&1; then RUNNER="native" elif command -v podman >/dev/null 2>&1; then RUNNER="podman" elif command -v docker >/dev/null 2>&1; then RUNNER="docker" else echo "run.sh: SKIP — no promtool on PATH and no podman/docker to run ${PROM_IMAGE}" >&2 exit 3 fi WORK="$(mktemp -d)" trap 'rm -rf "${WORK}"' EXIT # The test files reference the generated rule file by bare name, so everything must sit in one # directory (that is also the only directory the container needs mounted). rc=0 python3 "${HERE}/extract_rules.py" --out "${WORK}/generated.rules.yaml" "${MANIFESTS[@]}" || rc=$? if [ "$rc" -eq 4 ]; then echo "run.sh: SKIP — PyYAML unavailable, cannot flatten the PrometheusRule manifests" >&2 exit 3 elif [ "$rc" -ne 0 ]; then exit "$rc" fi names=() for t in "${TESTS[@]}"; do cp "$t" "${WORK}/" names+=("$(basename "$t")") done chmod -R a+rX "${WORK}" echo "run.sh: promtool test rules (${RUNNER}) ${names[*]}" >&2 case "$RUNNER" in native) ( cd "${WORK}" && promtool test rules "${names[@]}" ) ;; podman) podman run --rm --user 0 -v "${WORK}:/rules:z" -w /rules \ --entrypoint /bin/promtool "${PROM_IMAGE}" test rules "${names[@]}" ;; docker) docker run --rm --user 0 -v "${WORK}:/rules" -w /rules \ --entrypoint /bin/promtool "${PROM_IMAGE}" test rules "${names[@]}" ;; esac