diff --git a/platform/cnpg/alerts.yaml b/platform/cnpg/alerts.yaml index 9f2910c..3c0621d 100644 --- a/platform/cnpg/alerts.yaml +++ b/platform/cnpg/alerts.yaml @@ -48,6 +48,33 @@ # cnpg_collector_pg_wal{value="count"} vs {value="max"} prod: 17 vs 64 dev: 36 vs 64 (neither fires yet) # cnpg_last_error prod+dev: nextcloud-db = 1 # +# AGGREGATION SCOPE — WHICH RULES MAY KEY ON `pod`, AND WHY (incident, 2026-09-12). The first cut of +# this file grouped EVERY rule `by (namespace, pod)`. That is correct only for metrics every instance +# reports independently, and WRONG for the two backup-catalogue metrics, which ONLY THE PRIMARY +# populates. Verified live on prod 2026-09-12 (`cnpg_collector_first_recoverability_point`, raw): +# keycloak-db-2 = 1788764922 nextcloud-db-2 = 1788764955 semaphore-db-2 = 1788764954 <- PRIMARIES +# keycloak-db-1/-3, nextcloud-db-1/-3, semaphore-db-1/-3 = 0 <- REPLICAS +# Since the Clusters went to `instances: 3`, `CNPGNoRecoverabilityPoint` was therefore PERMANENTLY +# FIRING on all six replicas while all three Clusters reported firstRecoverabilityPoint=2026-09-07, +# lastSuccessfulBackup within hours, ContinuousArchiving=True and LastBackupSucceeded=True. Backups +# were fine; the alert was noise, and noise on a DR alarm is worse than no alarm at all — a REAL loss +# of recoverability would have arrived as a seventh identical email nobody reads. +# +# THE RULE OF THUMB, applied as an audit to every rule below: +# * `== 0` / staleness shapes ("nothing reports a value") MUST aggregate `pod` AWAY, or every pod +# that structurally never reports the value fires forever. +# * `> 0` shapes ("some pod reports a bad value") are SAFE per-pod: a pod sitting at 0 simply does +# not fire, and keeping `pod` names the instance to go look at. +# +# THE GROUPING LABEL IS `job`, NOT `namespace`. The metric carries NO `cnpg.io/cluster` label — its +# full label set on prod is {container, endpoint, instance, job, namespace, pod} — but the CNPG +# operator creates one PodMonitor per Cluster, named after the Cluster, in the Cluster's namespace, +# so prometheus-operator derives `job="/"`: `keycloak/keycloak-db`, +# `nextcloud-db/nextcloud-db`, `semaphore-db/semaphore-db`. That identifies the CNPG CLUSTER +# uniquely. Grouping by `namespace` alone happens to work today (one Cluster per namespace, verified) +# but would let two Clusters in one namespace MASK each other — the healthy one's non-zero value would +# satisfy `max` for both. `job` cannot do that, so `job` is what the DR rules group by. +# # SEVERITIES: every rule here is `warning` (-> the email-warning receiver). This is deliberate. # A database heading for disk-full deserves a warning with weeks of lead time, and this repo already # has alert fatigue from a permanently-firing `critical` (see the VPNTunnelDown post-mortem in @@ -82,6 +109,12 @@ spec: # 24h; prod has never been below 3 (min_over_time 24h: keycloak 9, nextcloud 5, semaphore 3). # A count threshold like `> 5` would have delayed semaphore-db's alert indefinitely — its # backlog sits at 3 because it writes almost nothing. + # + # SCOPE AUDIT (2026-09-12): keeps `pod`, deliberately. Every instance has its own pg_wal on its + # own PVC and every instance reports this metric (verified: all 9 prod pods publish + # `{value="ready"}` = 0 today), and this is a `> 0` shape — a replica sitting at 0 never fires, + # so there is no permanent-firing hazard and no masking (`> 0` fires if ANY pod is bad). `pod` + # earns its place here: it names the instance whose archiver is stuck. - alert: CNPGWALArchiveBacklog expr: | max by (namespace, pod) (cnpg_collector_pg_wal_archive_status{value="ready"}) > 0 @@ -102,15 +135,37 @@ spec: # `for: 6h` because a legitimately fresh Cluster reads 0 until its first base backup lands # (ScheduledBackup immediate:true, but MinIO/bucket setup can lag) — 6h keeps a normal cold # start quiet while still catching a bootstrap that never produced a recoverability point. + # + # SCOPE — `by (namespace, job)`, NOT `by (namespace, pod)`. THIS IS THE FIX FOR THE 2026-09-12 + # permanently-firing incident (see the AGGREGATION SCOPE block in the header). The recoverability + # point is a property of the CLUSTER's backup catalogue, which only the PRIMARY reads; a replica + # reports a hard 0 forever. Grouped by `pod`, each replica was evaluated on its own and this + # `== 0` fired permanently on all six of them. Aggregating `pod` away makes `max` take the + # PRIMARY's non-zero value as the cluster's answer, so the alert says what its own summary always + # claimed to say — "this Postgres CLUSTER has no recoverability point" — and fires only when NO + # instance of that Cluster reports one. + # + # PROVEN LIVE, read-only, prod, 2026-09-12 (promtool query instant): + # OLD max by (namespace, pod) (...) == 0 -> 6 series: keycloak-db-1/-3, nextcloud-db-1/-3, + # semaphore-db-1/-3 (and 6 ALERTS firing) + # NEW max by (namespace, job) (...) == 0 -> EMPTY + # and it is not merely vacuous — forcing the metric to 0 for one cluster's primary only + # (`* on(pod) group_left() ...{pod="keycloak-db-2"}`) yields exactly one series, + # {job="keycloak/keycloak-db"}, so a genuine single-cluster loss of DR still alerts, per cluster. + # + # A switchover cannot flap this: while the primary restarts, `max` briefly sees only replicas' 0, + # but `for: 6h` requires the condition at EVERY evaluation for six hours and a promotion takes + # seconds. Annotations key on `job` (= "/") because `pod` no longer exists on + # these series — a `{{ $labels.pod }}` left behind here would render as an empty string. - alert: CNPGNoRecoverabilityPoint expr: | - max by (namespace, pod) (cnpg_collector_first_recoverability_point) == 0 + max by (namespace, job) (cnpg_collector_first_recoverability_point) == 0 for: 6h labels: severity: warning annotations: - summary: "CNPG cluster has NO recoverability point: {{ $labels.namespace }}/{{ $labels.pod }}" - description: "PostgreSQL instance {{ $labels.namespace }}/{{ $labels.pod }} reports firstRecoverabilityPoint = null for 6h — there is NO point-in-time this database can be restored to. Off-cluster DR for this database does not exist (R-IDP-05 / R-STATE-03). Usually the barmanObjectStore credentials/endpoint/path are wrong; check `kubectl get cluster -n {{ $labels.namespace }} -o yaml` for the ContinuousArchiving condition." + summary: "CNPG cluster has NO recoverability point: {{ $labels.job }}" + description: "PostgreSQL cluster {{ $labels.job }} (namespace {{ $labels.namespace }}) reports firstRecoverabilityPoint = null on EVERY instance for 6h — there is NO point-in-time this database can be restored to. Off-cluster DR for this database does not exist (R-IDP-05 / R-STATE-03). This is a per-CLUSTER statement: the primary owns the backup catalogue, so a healthy cluster clears it even though its replicas report 0. Usually the barmanObjectStore credentials/endpoint/path are wrong; check `kubectl get cluster -n {{ $labels.namespace }} -o yaml` for the ContinuousArchiving condition." runbook_url: "https://${BASE_DOMAIN}/runbooks/cnpg-archiving" # --------------------------------------------------------------------------------------- # The daily ScheduledBackup silently stopped. Nothing else covers this: the Velero rules in @@ -119,16 +174,31 @@ spec: # backup). Guarded with `> 0` so a cluster that has NEVER had a backup is reported by # CNPGNoRecoverabilityPoint above and does not double-notify here (time()-0 would otherwise be # ~57 years and always true). 36h = one missed daily run plus slack. + # + # SCOPE — `by (namespace, job)`, fixed alongside CNPGNoRecoverabilityPoint (2026-09-12). + # `cnpg_collector_last_available_backup_timestamp` has the SAME primary-only defect, measured on + # prod the same day: keycloak-db-2 = 3.1h old, nextcloud-db-2 = 2.1h, semaphore-db-2 = 0.1h, and + # ALL SIX REPLICAS = 0, i.e. `time() - 0` = 497020h ≈ 56.7 YEARS. Unlike the rule above this one + # was not firing, because the `> 0` guard was load-bearing — it was the single thing standing + # between this rule and six more permanent false positives, which is far too thin a margin for a + # DR alarm. Aggregating `pod` away removes the dependence: `max` picks the PRIMARY's timestamp as + # the cluster's answer (a demoted primary drops to 0 and can never win the `max`, and a fresher + # backup always beats a staler one), the `> 0` guard then means exactly "this cluster has had at + # least one backup", and the result is a real per-cluster staleness statement. + # + # PROVEN LIVE, prod, 2026-09-12: OLD -> empty, NEW -> empty (so the fix changes no live verdict), + # while the per-CLUSTER ages the new aggregation actually evaluates are keycloak 3.09h / + # nextcloud 2.09h / semaphore 0.09h — the primaries' real values, all far inside 36h. - alert: CNPGNoRecentBaseBackup expr: | - (time() - max by (namespace, pod) (cnpg_collector_last_available_backup_timestamp)) > (36 * 3600) - and max by (namespace, pod) (cnpg_collector_last_available_backup_timestamp) > 0 + (time() - max by (namespace, job) (cnpg_collector_last_available_backup_timestamp)) > (36 * 3600) + and max by (namespace, job) (cnpg_collector_last_available_backup_timestamp) > 0 for: 2h labels: severity: warning annotations: - summary: "No recent CNPG base backup: {{ $labels.namespace }}/{{ $labels.pod }}" - description: "The most recent completed base backup for {{ $labels.namespace }}/{{ $labels.pod }} is over 36h old — the daily ScheduledBackup has stopped producing backups, so the PITR window is ageing out against the 30d retentionPolicy. Check `kubectl get backup -n {{ $labels.namespace }}`." + summary: "No recent CNPG base backup: {{ $labels.job }}" + description: "The most recent completed base backup for PostgreSQL cluster {{ $labels.job }} (namespace {{ $labels.namespace }}) is over 36h old — the daily ScheduledBackup has stopped producing backups, so the PITR window is ageing out against the 30d retentionPolicy. Per-CLUSTER: the newest timestamp across the cluster's instances is used, which is the primary's. Check `kubectl get backups.postgresql.cnpg.io -n {{ $labels.namespace }}` — the FULLY-QUALIFIED name is required: longhorn.io, postgresql.cnpg.io and velero.io all register kind Backup, and bare `kubectl get backup` resolves to Longhorn's (empty) set and reports 'No resources found' while CNPG backups exist." runbook_url: "https://${BASE_DOMAIN}/runbooks/cnpg-backup-missed" # --------------------------------------------------------------------------------------- # WAL-VOLUME-SPECIFIC ESCALATION. pg_wal holds more segments than max_wal_size allows for, @@ -146,6 +216,12 @@ spec: # any Cluster), which is why the PVC-fill rules cover it at all; the CNPG-native # `cnpg_collector_pg_wal{value="volume_size"}` is NaN on both clusters for that reason and is # therefore NOT usable as a signal. + # + # SCOPE AUDIT (2026-09-12): keeps `pod`, deliberately — and here it is REQUIRED. Every instance + # reports its own count/max (verified on all 9 prod pods: count 16-37 against max 64), pg_wal is + # per-instance on a per-instance PVC, and the comparison only means anything pod-against-itself: + # aggregating `pod` away would compare one instance's count with another's max. `>` is also a + # per-pod shape, so no instance can sit permanently true. - alert: CNPGWALVolumeNotRecycling expr: | max by (namespace, pod) (cnpg_collector_pg_wal{value="count"}) @@ -170,6 +246,15 @@ spec: # Remedy: grant the exporter role CONNECT on the application database. `for: 6h` keeps a pod # restart or a brief connection blip quiet; severity stays `warning` so it never becomes the # permanently-firing `critical` this repo has been burned by. + # + # SCOPE AUDIT (2026-09-12): keeps `pod`, UNCHANGED — this rule's meaning and firing set are + # deliberately left exactly as they were. `cnpg_last_error` is genuinely per-instance: each + # instance's own exporter reports its own SQL-collection health, and prod proves the metric + # discriminates rather than tracking role — all THREE nextcloud-db pods (primary -2 included) + # report 1 while all six keycloak/semaphore pods report 0. So there is no primary-only defect to + # fix, and per-pod is the right granularity: the remedy is a GRANT applied per database, and you + # want to see it clear on every instance. It is a TRUE POSITIVE right now (nextcloud-db, the + # 42501 CONNECT grant) and is owned by separate work; nothing here silences or rescopes it. - alert: CNPGMetricsCollectorErroring expr: | max by (namespace, pod) (cnpg_last_error) == 1