--- # promtool unit tests for the CNPGPrimariesCoLocated rule in platform/cnpg/alerts.yaml. # # WHY THIS FILE EXISTS. This repo has been burned in both directions by alerts nobody tested: # a permanently-firing `critical` (the VPNTunnelDown post-mortem in platform/monitoring/alerts.yaml) # and, worse, silence that looked like health (the 2026-09-06 CNPG archiving incident — 38h of # ContinuousArchiving=False with no alert of any kind). CNPGPrimariesCoLocated is a JOIN across two # metric families with a label_replace on each side and a node-readiness gate; "it looks right" is # not evidence that it can fire, and it is certainly not evidence that it can stop. These tests pin # BOTH directions plus the two ways the rule is meant to stay quiet. # # HOW TO RUN. `platform/monitoring/tests/run.sh` — the ONE repo-wide rule-unit harness. It finds # this file by the companion convention (`.promtool-test.yaml` sits beside # `.yaml`), flattens the PrometheusRule CR's `.spec.groups` into the bare Prometheus # rules file promtool wants (a CR is not a rules file), writes it under the name `rule_files:` # below declares, `promtool check rules` it first so a non-parsing file cannot make these tests # pass vacuously, and runs them — all in a temp dir, because a generated file sitting next to its # source is how a stale copy becomes the thing under test. Run just this file with # `platform/monitoring/tests/run.sh platform/cnpg/alerts.promtool-test.yaml`. # (There used to be a cnpg-only `run-alert-tests.sh` here; it was folded into that harness on # 2026-09-12 — two harnesses means two behaviours, and the one nothing runs rots.) # # promtool IS in the baked devcontainer toolchain (PROMTOOL_VERSION in # infra/devcontainer/versions.env, mirrored in ci/versions.env), pinned to the Prometheus version # prod actually runs (v3.14.0) because promtool is the rule ENGINE. A missing promtool is exit 3 # (SKIPPED, never green), not a silent download. # # WHO RUNS IT IN CI. conformance/tests/l0-static/alert-rules-unit-tested — the L0 static check # that runs the harness over every *.promtool-test.yaml in the repo and reports which alert files # still have none. Before that check existed these tests were never evaluated by anything. # # WHY THE ASSERTIONS USE `ALERTS` AND NOT `exp_alerts`. promtool's alert_rule_test compares # annotations VERBATIM, which would mean pasting this rule's ~1200-character runbook description in # here and re-pasting it on every wording change. Querying the synthetic `ALERTS` series instead # proves exactly what matters — the alert reaches `pending` and then `firing` across the `for: 30m` # boundary, with the right labels — and stays decoupled from annotation prose. # # THE SERIES BELOW ARE REAL SHAPES, not invented ones: every label on # kube_endpointslice_endpoints / cnpg_collector_up / kube_node_status_condition / kube_node_info was # read off live prod Prometheus on 2026-09-12 before being copied here. rule_files: - alerts.rules.yaml evaluation_interval: 1m tests: # ------------------------------------------------------------------------------------------- # 1. THE BAD STATE THAT PROMPTED THIS RULE, exactly as prod looked on 2026-09-12: all three # primaries (keycloak-db-2, nextcloud-db-2, semaphore-db-2) on k3s-prod-1, all nodes Ready. # Asserts the rule CAN fire, that it counts 3 (not 1, which a broken join would give), and # that `for: 30m` is honoured — pending at 29m, firing at 31m. - interval: 1m name: three primaries on one node fires after 30m input_series: - series: 'kube_endpointslice_endpoints{namespace="keycloak",endpointslice="keycloak-db-rw-k9phd",targetref_kind="Pod",targetref_name="keycloak-db-2",targetref_namespace="keycloak",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",address="10.0.0.142",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'kube_endpointslice_endpoints{namespace="nextcloud-db",endpointslice="nextcloud-db-rw-dxhx7",targetref_kind="Pod",targetref_name="nextcloud-db-2",targetref_namespace="nextcloud-db",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",address="10.0.0.126",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'kube_endpointslice_endpoints{namespace="semaphore-db",endpointslice="semaphore-db-rw-2xl8g",targetref_kind="Pod",targetref_name="semaphore-db-2",targetref_namespace="semaphore-db",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",address="10.0.0.201",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' # All nine instances publish cnpg_collector_up — the join must pick out only the three that # the -rw EndpointSlices point at, so the replicas are present here on purpose. - series: 'cnpg_collector_up{namespace="keycloak",pod="keycloak-db-1",cluster="keycloak-db",container="postgres",job="keycloak/keycloak-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="keycloak",pod="keycloak-db-2",cluster="keycloak-db",container="postgres",job="keycloak/keycloak-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="keycloak",pod="keycloak-db-3",cluster="keycloak-db",container="postgres",job="keycloak/keycloak-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="nextcloud-db",pod="nextcloud-db-1",cluster="nextcloud-db",container="postgres",job="nextcloud-db/nextcloud-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="nextcloud-db",pod="nextcloud-db-2",cluster="nextcloud-db",container="postgres",job="nextcloud-db/nextcloud-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="nextcloud-db",pod="nextcloud-db-3",cluster="nextcloud-db",container="postgres",job="nextcloud-db/nextcloud-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="semaphore-db",pod="semaphore-db-1",cluster="semaphore-db",container="postgres",job="semaphore-db/semaphore-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="semaphore-db",pod="semaphore-db-2",cluster="semaphore-db",container="postgres",job="semaphore-db/semaphore-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="semaphore-db",pod="semaphore-db-3",cluster="semaphore-db",container="postgres",job="semaphore-db/semaphore-db"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-1",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-2",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-3",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-1"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-2"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-3"}' values: '1x60' promql_expr_test: # The join resolves each -rw endpoint to (namespace, primary pod, node) — the drill-down an # operator actually wants. All three land on k3s-prod-1. - expr: | max by (namespace, pod, node) ( label_replace( label_replace( kube_endpointslice_endpoints{endpointslice=~".+-rw-.+", targetref_kind="Pod", ready="true"}, "pod", "$1", "targetref_name", "(.+)" ), "node", "$1", "endpoint_nodename", "(.+)" ) ) * on (namespace, pod) group_left () max by (namespace, pod) (cnpg_collector_up) eval_time: 10m exp_samples: - labels: '{namespace="keycloak", node="k3s-prod-1", pod="keycloak-db-2"}' value: 1 - labels: '{namespace="nextcloud-db", node="k3s-prod-1", pod="nextcloud-db-2"}' value: 1 - labels: '{namespace="semaphore-db", node="k3s-prod-1", pod="semaphore-db-2"}' value: 1 # ... and the alert's own expression counts 3 of them on that one node. - expr: | count by (node) ( max by (namespace, pod, node) ( label_replace( label_replace( kube_endpointslice_endpoints{endpointslice=~".+-rw-.+", targetref_kind="Pod", ready="true"}, "pod", "$1", "targetref_name", "(.+)" ), "node", "$1", "endpoint_nodename", "(.+)" ) ) * on (namespace, pod) group_left () max by (namespace, pod) (cnpg_collector_up) ) > 1 and on () count(kube_node_status_condition{condition="Ready", status="true"} == 1) == count(kube_node_info) eval_time: 10m exp_samples: - labels: '{node="k3s-prod-1"}' value: 3 # NOT PERMANENTLY FIRING, part 1: `for: 30m` really is waited out. - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 29m exp_samples: - labels: 'ALERTS{alertname="CNPGPrimariesCoLocated",alertstate="pending",node="k3s-prod-1",severity="warning"}' value: 1 # NOT PERMANENTLY INERT: it does reach `firing`. - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 31m exp_samples: - labels: 'ALERTS{alertname="CNPGPrimariesCoLocated",alertstate="firing",node="k3s-prod-1",severity="warning"}' value: 1 # ------------------------------------------------------------------------------------------- # 2. THE GOOD STATE — one primary per node, which is what the rebalancing switchovers on # 2026-09-12 produced. The rule must be SILENT here, and silent for the right reason: the # first assertion proves the join still resolves all three primaries (an alert that is quiet # because its query returns nothing is the failure mode this repo keeps hitting), the second # proves the >1 test then finds no over-loaded node. - interval: 1m name: one primary per node is silent input_series: - series: 'kube_endpointslice_endpoints{namespace="keycloak",endpointslice="keycloak-db-rw-k9phd",targetref_kind="Pod",targetref_name="keycloak-db-1",targetref_namespace="keycloak",endpoint_nodename="k3s-prod-2",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'kube_endpointslice_endpoints{namespace="nextcloud-db",endpointslice="nextcloud-db-rw-dxhx7",targetref_kind="Pod",targetref_name="nextcloud-db-1",targetref_namespace="nextcloud-db",endpoint_nodename="k3s-prod-3",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'kube_endpointslice_endpoints{namespace="semaphore-db",endpointslice="semaphore-db-rw-2xl8g",targetref_kind="Pod",targetref_name="semaphore-db-2",targetref_namespace="semaphore-db",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'cnpg_collector_up{namespace="keycloak",pod="keycloak-db-1",cluster="keycloak-db",container="postgres",job="keycloak/keycloak-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="nextcloud-db",pod="nextcloud-db-1",cluster="nextcloud-db",container="postgres",job="nextcloud-db/nextcloud-db"}' values: '1x60' - series: 'cnpg_collector_up{namespace="semaphore-db",pod="semaphore-db-2",cluster="semaphore-db",container="postgres",job="semaphore-db/semaphore-db"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-1",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-2",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-3",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-1"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-2"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-3"}' values: '1x60' promql_expr_test: # Quiet for the RIGHT reason: the join is alive and finds all three primaries, one per node. - expr: | count by (node) ( max by (namespace, pod, node) ( label_replace( label_replace( kube_endpointslice_endpoints{endpointslice=~".+-rw-.+", targetref_kind="Pod", ready="true"}, "pod", "$1", "targetref_name", "(.+)" ), "node", "$1", "endpoint_nodename", "(.+)" ) ) * on (namespace, pod) group_left () max by (namespace, pod) (cnpg_collector_up) ) eval_time: 10m exp_samples: - labels: '{node="k3s-prod-1"}' value: 1 - labels: '{node="k3s-prod-2"}' value: 1 - labels: '{node="k3s-prod-3"}' value: 1 # ... and no alert is ever raised, at any point in the hour. - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 31m exp_samples: [] - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 59m exp_samples: [] # ------------------------------------------------------------------------------------------- # 3. A NODE IS DOWN, so co-location is UNAVOIDABLE and correct — the rule must not nag during # an outage. k3s-prod-3 is NotReady for the first 10 minutes and nextcloud-db has failed over # onto k3s-prod-1, which now holds two primaries. Then the node comes back at t=11m and the # co-location is NOT cleaned up, which is precisely the drift worth reporting: the rule must # RE-ARM by itself and fire 30m later. This is the test that keeps the readiness gate from # turning into a permanent mute. - interval: 1m name: node down suppresses, then re-arms when the node returns input_series: - series: 'kube_endpointslice_endpoints{namespace="keycloak",endpointslice="keycloak-db-rw-k9phd",targetref_kind="Pod",targetref_name="keycloak-db-2",targetref_namespace="keycloak",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x90' - series: 'kube_endpointslice_endpoints{namespace="nextcloud-db",endpointslice="nextcloud-db-rw-dxhx7",targetref_kind="Pod",targetref_name="nextcloud-db-2",targetref_namespace="nextcloud-db",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x90' - series: 'cnpg_collector_up{namespace="keycloak",pod="keycloak-db-2",cluster="keycloak-db",container="postgres",job="keycloak/keycloak-db"}' values: '1x90' - series: 'cnpg_collector_up{namespace="nextcloud-db",pod="nextcloud-db-2",cluster="nextcloud-db",container="postgres",job="nextcloud-db/nextcloud-db"}' values: '1x90' - series: 'kube_node_status_condition{node="k3s-prod-1",condition="Ready",status="true"}' values: '1x90' - series: 'kube_node_status_condition{node="k3s-prod-2",condition="Ready",status="true"}' values: '1x90' # k3s-prod-3: NotReady (0) for t=0..10, Ready (1) from t=11 onwards. - series: 'kube_node_status_condition{node="k3s-prod-3",condition="Ready",status="true"}' values: '0x10 1x79' - series: 'kube_node_info{node="k3s-prod-1"}' values: '1x90' - series: 'kube_node_info{node="k3s-prod-2"}' values: '1x90' - series: 'kube_node_info{node="k3s-prod-3"}' values: '1x90' promql_expr_test: # Two primaries genuinely do share k3s-prod-1 at t=5m... - expr: | count by (node) ( max by (namespace, pod, node) ( label_replace( label_replace( kube_endpointslice_endpoints{endpointslice=~".+-rw-.+", targetref_kind="Pod", ready="true"}, "pod", "$1", "targetref_name", "(.+)" ), "node", "$1", "endpoint_nodename", "(.+)" ) ) * on (namespace, pod) group_left () max by (namespace, pod) (cnpg_collector_up) ) > 1 eval_time: 5m exp_samples: - labels: '{node="k3s-prod-1"}' value: 2 # ... but the readiness gate suppresses the alert entirely while a node is down: not even pending. - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 5m exp_samples: [] - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 10m exp_samples: [] # Node back at t=11m; the `for` timer starts from scratch, so still only pending at t=25m. - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 25m exp_samples: - labels: 'ALERTS{alertname="CNPGPrimariesCoLocated",alertstate="pending",node="k3s-prod-1",severity="warning"}' value: 1 # RE-ARMED: 30m of full-cluster health with the co-location still there -> firing. - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 45m exp_samples: - labels: 'ALERTS{alertname="CNPGPrimariesCoLocated",alertstate="firing",node="k3s-prod-1",severity="warning"}' value: 1 # ------------------------------------------------------------------------------------------- # 4. THE FALSE-POSITIVE GUARD. `endpointslice=~".+-rw-.+"` on its own is a loose match — any # Service in the cluster named `something-rw` produces an EndpointSlice that matches it. Two # such endpoints on one node must NOT be counted as CNPG primaries, which is what the # `* on (namespace, pod) group_left () cnpg_collector_up` factor is for. Without that factor # this test fires, so it is the thing that pins the join instead of a bare regex. - interval: 1m name: non-CNPG rw services are not counted input_series: - series: 'kube_endpointslice_endpoints{namespace="default",endpointslice="myapp-rw-aaaaa",targetref_kind="Pod",targetref_name="myapp-0",targetref_namespace="default",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'kube_endpointslice_endpoints{namespace="default",endpointslice="other-rw-bbbbb",targetref_kind="Pod",targetref_name="other-0",targetref_namespace="default",endpoint_nodename="k3s-prod-1",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' # One real CNPG primary, on a different node — present so the test cannot pass merely because # the whole query is empty. - series: 'kube_endpointslice_endpoints{namespace="keycloak",endpointslice="keycloak-db-rw-k9phd",targetref_kind="Pod",targetref_name="keycloak-db-2",targetref_namespace="keycloak",endpoint_nodename="k3s-prod-2",ready="true",serving="true",terminating="false",pod="kube-prometheus-stack-kube-state-metrics-0",job="kube-state-metrics"}' values: '1x60' - series: 'cnpg_collector_up{namespace="keycloak",pod="keycloak-db-2",cluster="keycloak-db",container="postgres",job="keycloak/keycloak-db"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-1",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-2",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_status_condition{node="k3s-prod-3",condition="Ready",status="true"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-1"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-2"}' values: '1x60' - series: 'kube_node_info{node="k3s-prod-3"}' values: '1x60' promql_expr_test: # Only the CNPG primary survives the join — the two decoy `*-rw` endpoints are dropped. - expr: | count by (node) ( max by (namespace, pod, node) ( label_replace( label_replace( kube_endpointslice_endpoints{endpointslice=~".+-rw-.+", targetref_kind="Pod", ready="true"}, "pod", "$1", "targetref_name", "(.+)" ), "node", "$1", "endpoint_nodename", "(.+)" ) ) * on (namespace, pod) group_left () max by (namespace, pod) (cnpg_collector_up) ) eval_time: 10m exp_samples: - labels: '{node="k3s-prod-2"}' value: 1 - expr: 'ALERTS{alertname="CNPGPrimariesCoLocated"}' eval_time: 45m exp_samples: []