#!/usr/bin/env bash ############################################################################### # platform/cnpg/run-alert-tests.sh — run the promtool unit tests for alerts.yaml. # # WHY A SCRIPT AND NOT JUST `promtool test rules`. Two impedance mismatches: # 1. alerts.yaml is a Kubernetes PrometheusRule CR; promtool wants a bare Prometheus # rules file (`groups:` at the top level). So `.spec` is extracted first. # 2. runbook_url carries `${BASE_DOMAIN}`, a Flux postBuild substitution that is not # resolved in git. promtool does not care what the string says, but leaving a raw # `${...}` in a rendered rules file is exactly the kind of thing that gets copied # somewhere it DOES matter, so it is substituted with a placeholder here. # Both derived files go to a temp dir — nothing generated is left in the worktree, because # a generated file sitting next to its source is how a stale copy becomes the thing under # test. # # promtool is NOT in the baked devcontainer toolchain (infra/devcontainer/versions.env). # If it is not on PATH this fetches the version that matches the Prometheus actually # running on prod, so the unit tests are evaluated by the same engine that will evaluate # the rules for real. Override with PROMTOOL=/path/to/promtool. # # ./run-alert-tests.sh ############################################################################### set -euo pipefail # Match the Prometheus running on prod: quay.io/prometheus/prometheus:v3.14.0-distroless # (`kubectl get prometheus -n monitoring -o jsonpath='{.items[*].spec.version}'`). PROMETHEUS_VERSION="${PROMETHEUS_VERSION:-3.14.0}" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT promtool="${PROMTOOL:-$(command -v promtool || true)}" if [[ -z "$promtool" ]]; then echo "==> promtool not on PATH; fetching v${PROMETHEUS_VERSION} (matches prod Prometheus)" tarball="prometheus-${PROMETHEUS_VERSION}.linux-amd64" curl -fsSL "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${tarball}.tar.gz" \ | tar -xz -C "$tmp" --strip-components=1 "${tarball}/promtool" promtool="$tmp/promtool" fi # Absolutised because step 3 runs promtool from inside $tmp — a relative PROMTOOL= would # break there, and it did during development. promtool="$(cd "$(dirname "$promtool")" && pwd)/$(basename "$promtool")" echo "==> $("$promtool" --version 2>&1 | head -1)" # 1. PrometheusRule CR -> bare Prometheus rules file, with ${BASE_DOMAIN} resolved. sed 's/\${BASE_DOMAIN}/base-domain.invalid/g' "$here/alerts.yaml" \ | python3 -c 'import sys,yaml; yaml.safe_dump(yaml.safe_load(sys.stdin)["spec"], sys.stdout, sort_keys=False, width=10**6)' \ > "$tmp/alerts.rules.yaml" cp "$here/alerts.promtool-test.yaml" "$tmp/" # 2. Syntax gate first — a rules file that does not parse would make every unit test # "pass" vacuously by finding no rules at all. echo "==> promtool check rules" "$promtool" check rules "$tmp/alerts.rules.yaml" # 3. The unit tests. echo "==> promtool test rules" cd "$tmp" "$promtool" test rules alerts.promtool-test.yaml