--- # Platform-health alert rules (docs/10, R-OBS-04). These complement the default node/kubernetes # rules bundled by kube-prometheus-stack and add FIRST-CLASS alerts for the platform's own moving # parts: nodes, Longhorn, Flux, cert-manager, networking/VPN, backups, identity, and the monitoring # stack's own capacity ("don't let the monitor blind itself"). # # Each rule maps to a docs/10 platform-health bullet and carries a severity (critical = page-worthy # vs warning = degraded/capacity) plus enough annotation context (summary + runbook) to act. # # The `release: kube-prometheus-stack` label makes the operator's Prometheus select these rules # (ruleSelector). Some expressions depend on exporters/ServiceMonitors owned by other workstreams # (Flux, cert-manager, Longhorn, Velero, Keycloak, Cilium/Hubble ship their own metrics); the rules # are authored to fire as soon as those series arrive. Metric names flagged TODO need confirming at # bring-up against the deployed exporter. # # runbook_url points at a docs base; swap for the real runbook location when it exists. # ===================================================================================== # Nodes # ===================================================================================== apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-nodes namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.nodes rules: - alert: NodeExporterDown expr: up{job="node-exporter"} == 0 for: 5m labels: severity: critical annotations: summary: "Node exporter down on {{ $labels.instance }}" description: "Prometheus cannot scrape node-exporter on {{ $labels.instance }} for 5m — the node may be down or unreachable." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-down" - alert: NodeMemoryPressureHigh expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90 for: 10m labels: severity: warning annotations: summary: "Memory pressure high on {{ $labels.instance }}" description: "Available memory below 10% for 10m on {{ $labels.instance }} — the ~22 GiB box is under pressure." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-memory" - alert: NodeDiskFillingUp expr: | (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"}) < 0.10 for: 15m labels: severity: warning annotations: summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})" description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-disk" - alert: NodeCPUPressureHigh # PSI "some" for CPU = fraction of wall-clock time at least one task was stalled waiting to # run. <5% is healthy; sustained >15% means bursty workloads are queueing against the vCPU # cap (measured 15-23% at 6 vCPU before the 8-vCPU resize). Averaged CPU% (kubectl top) hides # this — PSI is the metric that actually surfaces the "spikes cause blips of delay" symptom. expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.15 for: 15m labels: severity: warning annotations: summary: "CPU pressure high on {{ $labels.instance }}" description: "CPU PSI (some) above 15% for 15m on {{ $labels.instance }} — tasks are being delayed waiting for CPU (bursts exceeding the node's vCPU headroom). Add vCPUs or trim CPU-heavy workloads." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu" - alert: NodeCPULoadHigh # 5-min load average per core. Sustained >1 means the run queue exceeds the core count (more # runnable/blocked tasks than CPUs). Core count = number of idle-mode cpu series per node. # Pair with NodeCPUPressureHigh to confirm the contention is CPU (vs I/O, which inflates load # but not CPU-PSI). expr: | node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 1 for: 15m labels: severity: warning annotations: summary: "Load per core high on {{ $labels.instance }}" description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu" # --- I/O PRESSURE (added after the 2026-09-07 k3s-prod-1 NotReady incident) --------- # THE METRIC THAT WOULD HAVE CALLED THE 2026-09-07 OUTAGE AND DID NOT EXIST. # At 06:05:21Z k3s-prod-1 went NotReady ("Kubelet stopped posting node status") with the # host up, pingable and SSH-able. Root cause was disk I/O starvation of the embedded etcd: # slow fdatasync took=3.56s/6.71s/4.95s/3.40s (etcd expects 1s) # apply request took too long took=8.30s (expects 100ms) # etcdserver: too many requests # `/proc/pressure/io` FULL avg300 read 11.36 on k3s-prod-1 at the time, against 0.75 on # k3s-prod-2 (NVMe) and 1.29 on k3s-prod-3. The stack already alerted on CPU-PSI and # memory but had NO I/O alert at all, so the first signal the operator got was a node # already out of the cluster. # # WHY "full" AND NOT "some": node_pressure_io_stalled_seconds_total is PSI *full* — time # in which EVERY runnable task was blocked on I/O, i.e. the machine did no useful work. # `some` (node_pressure_io_waiting_seconds_total) is noisy on a storage node; `full` is # the one that tracked the outage. # # THRESHOLDS ARE MEASURED, NOT GUESSED. rate(...[5m]) is a 0-1 fraction, so the incident's # 11.36% is 0.1136 and the healthy steady state measured on the same node is ~0.01: # warning 0.05 for 10m — 5x steady state, well below the outage, leaves room to act # critical 0.10 for 5m — the level k3s-prod-1 actually sat at while etcd was stalling # Deliberately NOT scoped to prod: dev shares this file and its k3s VMs are HDD-backed, # where the same alert is if anything more likely to earn its keep. - alert: NodeIOPressureHigh expr: rate(node_pressure_io_stalled_seconds_total[5m]) > 0.05 for: 10m labels: severity: warning annotations: summary: "Disk I/O pressure high on {{ $labels.instance }}" description: "I/O PSI (full) above 5% for 10m on {{ $labels.instance }} — every task on the node is spending real time blocked on disk. On a node whose etcd WAL shares a partition with Longhorn replicas this is the leading indicator of etcd fsync stalls. Check `cat /proc/pressure/io`, `iostat -x 1`, and whether Longhorn is rebuilding replicas." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" - alert: NodeIOPressureCritical # 0.10 is the level k3s-prod-1 held while etcd logged multi-second fdatasync and then # dropped out of the cluster. At this point etcd is already at risk; do not wait 10m. expr: rate(node_pressure_io_stalled_seconds_total[5m]) > 0.10 for: 5m labels: severity: critical annotations: summary: "Disk I/O starvation on {{ $labels.instance }}" description: "I/O PSI (full) above 10% for 5m on {{ $labels.instance }} — the level at which k3s-prod-1 stalled etcd's WAL fsync and went NotReady on 2026-09-07 while the host stayed up. Expect etcd 'slow fdatasync' / 'apply request took too long' in `journalctl -u k3s`. Stop or throttle whatever is writing (Longhorn rebuild, backup, restore) before the node drops." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # ===================================================================================== # Storage — Longhorn # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-longhorn namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.longhorn rules: # ------------------------------------------------------------------------------------- # `longhorn_volume_robustness` IS A STATE-LABELLED GAUGE, NOT A 0-3 ENUM (fixed 2026-09-13) # # Both rules below used to read `longhorn_volume_robustness == 2` / `== 3`, with the comment # "0=unknown 1=healthy 2=degraded 3=faulted". That is the shape of an OLDER Longhorn. On # longhorn 1.12.1 (the version this cluster runs) the metric is ONE-HOT over a `state` # label and its value is only ever 0 or 1: # longhorn_volume_robustness{volume="…",state="healthy"} 1 # longhorn_volume_robustness{volume="…",state="degraded"} 0 # longhorn_volume_robustness{volume="…",state="faulted"} 0 # longhorn_volume_robustness{volume="…",state="unknown"} 0 # MEASURED on prod 2026-09-13, scrape working, 24 volumes attached: # count(longhorn_volume_robustness) => 96 (24 volumes x 4 states) # count by (state) (longhorn_volume_robustness == 1) => {state="healthy"} 24 # count(longhorn_volume_robustness == 2) => (empty) # count(longhorn_volume_robustness == 3) => (empty) # So `== 2` and `== 3` matched NO sample. Enabling the Longhorn ServiceMonitor earlier today # made the series exist; it did NOT make these two rules able to fire — a faulted volume # would still have alerted nobody. The "44 series / 24 healthy(1), 72 unknown(0, detached)" # reading recorded in platform/longhorn/helmrelease.yaml is the same misreading: those # zeros are the OTHER state labels of healthy volumes, not detached volumes. # # Pinned in both directions by platform/monitoring/tests/longhorn.test.yaml, which keeps a # regression case asserting the old `== 2` form matches nothing on this metric shape. # ------------------------------------------------------------------------------------- - alert: LonghornVolumeDegraded expr: longhorn_volume_robustness{state="degraded"} == 1 for: 10m labels: severity: warning annotations: summary: "Longhorn volume degraded: {{ $labels.volume }}" description: "Volume {{ $labels.volume }} has been degraded (replica rebuild / missing replica) for 10m." runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-degraded" - alert: LonghornVolumeFaulted expr: longhorn_volume_robustness{state="faulted"} == 1 for: 1m labels: severity: critical annotations: summary: "Longhorn volume FAULTED: {{ $labels.volume }}" description: "Volume {{ $labels.volume }} is faulted — data at risk / offline. Immediate attention required." runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-faulted" # ------------------------------------------------------------------------------------- # WHERE `LonghornVolumeCapacityHigh` WENT (deleted 2026-09-13) — IT MEASURED ALLOCATION AND # CALLED IT CAPACITY, AND ON ONE VOLUME IT COULD NEVER CLEAR. # # The deleted rule was # (longhorn_volume_actual_size_bytes / longhorn_volume_capacity_bytes) > 0.85 for 30m # summarised "Longhorn volume near capacity", described as "over 85% used". Within minutes # of the Longhorn scrape being enabled it went `pending` on the Prometheus TSDB volume. # MEASURED on prod 2026-09-13: # pvc-fa59563f-… (monitoring/prometheus-…-prometheus-db-…-0, 50Gi, class longhorn) # longhorn_volume_actual_size_bytes 52243681280 (48.66 GiB) # longhorn_volume_capacity_bytes 53687091200 (50 GiB) # ratio 0.9731 <- what the alert saw # next-highest volume in the cluster 0.2818 (keycloak-db-1) — a lone outlier # the SAME filesystem, from the kubelet: # kubelet_volume_stats_used_bytes / …_capacity_bytes 0.2080 (20.8% used) # # `actual_size` is not "used". It is the size of the replica's disk files: the live head PLUS # every snapshot still pinned. On this volume, # longhorn_snapshot_actual_size_bytes{user_created="false"} 40510484480 (37.7 GiB) # ONE Longhorn system snapshot — the kind taken for a full replica rebuild, which is exactly # what prod-1's drive replacement forced — holds 37.7 GiB of historical blocks; the live head # is the remaining ~10.9 GiB, matching the kubelet's 11.2 GiB. Longhorn also never returns # freed blocks to the replica file without discard/TRIM, so the number is monotonic: it was # climbing at ~152 MB/h (deriv over 2h) while the filesystem sat flat at 20%. With # `retention: 30d` and `retentionSize: 45GiB` deliberately targeting 90% of a 50Gi volume, # the ratio is by DESIGN pinned near 1.0 and cannot fall back under 0.85. A rule that can # only ever be true is the CNPGNoRecoverabilityPoint failure again — it teaches the operator # to ignore the storage row, which is the row that matters on this cluster. # # NOT REPLACED PER VOLUME, BECAUSE THE HONEST QUESTION IS ALREADY ASKED — measured, not # asserted: # * "is this PVC's filesystem filling?" -> kubelet_volume_stats_*. The monitoring # namespace is covered by MonitoringVolumeFillingUp (<15% free, 15m) later in THIS file; # every other namespace by PersistentVolumeFillingUp / PersistentVolumeAlmostFull in # platform/monitoring/pvc-capacity-alerts.yaml. COVERAGE MEASURED on prod 2026-09-13: # 24 Longhorn volumes exist; 23 of them appear in `kubelet_volume_stats_inodes > 0` and # so fall inside one of those two scopes. The 24th is unifi/unifi-rootdisk, volumeMode # Block — no filesystem exists for the kubelet OR for Longhorn to report usage of, so # the deleted rule was not covering it either (its 0.2097 was raw-device allocation). # Block volumes are covered at the pool level, below. # * "is replica allocation exhausting the pool?" -> that IS an actual_size question, and # it is answered per DISK below, where allocation is a budget, instead of per volume # against its own nominal size, where it is a tautology. # ------------------------------------------------------------------------------------- # THE ALLOCATION ALARM, AT THE LEVEL WHERE ALLOCATION IS A BUDGET. # # Renamed from LonghornNodeStorageHigh, re-keyed from longhorn_node_storage_* to # longhorn_disk_*, re-thresholded from 0.85 to 0.70. Each for a reason: # # * NAME AND WORDING. This is the number that legitimately counts blocks EVER allocated — # replica files plus their pinned snapshots — because that is what consumes the node's # data partition whether or not any guest filesystem thinks it is full. "Node storage # near capacity" invited the same misreading the deleted per-volume rule shipped; # "disk allocation high" says what it measures. # * PER DISK, not per node. longhorn_node_storage_usage_bytes SUMS a node's disks, so one # full disk could hide behind a half-empty second one. longhorn_disk_usage_bytes carries # a `disk` label. Identical today (one disk per node) and correct if a disk is added. # * 0.70, not 0.85, because 0.85 is PAST THE CLIFF. Longhorn's own # storage-minimal-available-percentage is 25 on this cluster (read live from # settings.longhorn.io), so a disk stops accepting new replicas at 75% used. The old # threshold could therefore only fire AFTER scheduling had already been refused — and # with hard node anti-affinity (replicaSoftAntiAffinity false, numberOfReplicas 3) a disk # that will not take a replica pins a degraded volume at 2 replicas indefinitely. 0.70 # gives lead time on the way to Longhorn's own limit. # # MEASURED on prod 2026-09-13 — quiet, with room, on all three nodes: # node capacity usage ratio scheduled (nominal) # k3s-prod-1 445098295296 100326506496 0.2254 288836550656 # k3s-prod-2 456002658304 105149128704 0.2306 288836550656 # k3s-prod-3 445098295296 105464528896 0.2369 288836550656 # ------------------------------------------------------------------------------------- - alert: LonghornDiskAllocationHigh expr: (longhorn_disk_usage_bytes / longhorn_disk_capacity_bytes) > 0.70 for: 30m labels: severity: warning annotations: summary: "Longhorn disk allocation high: {{ $labels.node }} disk {{ $labels.disk }}" description: "Longhorn disk {{ $labels.disk }} on node {{ $labels.node }} has over 70% of its bytes ALLOCATED to replica files and their pinned snapshots for 30m. This is allocation, not filesystem usage inside any one PVC — freed blocks are not returned without discard/TRIM, and a system snapshot from a replica rebuild keeps its historical blocks. At 75% Longhorn stops scheduling new replicas here (storage-minimal-available-percentage is 25), which with hard node anti-affinity pins any degraded volume at 2 replicas. Look for snapshots to purge (`kubectl -n longhorn-system get snapshots.longhorn.io`) before adding capacity." runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity" # The terminal form of the same condition, straight from Longhorn rather than inferred from a # threshold: the disk itself reports whether it will accept another replica. More honest than # re-deriving the scheduling formula in PromQL, which folds in storage-reserved-percentage # 20, storage-over-provisioning-percentage 200 and storage-minimal-available-percentage 25 — # three settings NO metric exports, so the rule would silently mis-measure if any changed. # MEASURED on prod 2026-09-13: longhorn_disk_status{condition="schedulable"} == 1 on all # three disks, so this is not a permanently-firing rule. `for: 15m` rides out the blip while # a longhorn-manager restarts. - alert: LonghornDiskUnschedulable expr: longhorn_disk_status{condition="schedulable"} == 0 for: 15m labels: severity: warning annotations: summary: "Longhorn will not place replicas on {{ $labels.node }} disk {{ $labels.disk }}" description: "Longhorn marked disk {{ $labels.disk }} on node {{ $labels.node }} UNSCHEDULABLE for 15m — it will accept no new or rebuilt replica. On a 3-node cluster with hard node anti-affinity that means any degraded volume stays at 2 replicas until this clears, so R-STATE-02's survive-a-node-loss promise is suspended meanwhile. Usually the disk is out of space (cross-read LonghornDiskAllocationHigh) or scheduling was disabled by hand on the Longhorn node." runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity" # ===================================================================================== # GitOps — Flux # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-flux namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.flux rules: - alert: FluxReconciliationFailure expr: | max by (namespace, name, kind) ( gotk_reconcile_condition{type="Ready", status="False"} ) == 1 for: 15m labels: severity: warning annotations: summary: "Flux resource not ready: {{ $labels.kind }}/{{ $labels.name }}" description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has failed to reconcile (Ready=False) for 15m — GitOps drift/failure (R-GITOPS-03)." runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-reconcile" - alert: FluxResourceSuspended expr: | max by (namespace, name, kind) (gotk_suspend_status) == 1 for: 1h labels: severity: warning annotations: summary: "Flux resource suspended: {{ $labels.kind }}/{{ $labels.name }}" description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has been suspended for 1h — reconciliation/drift-correction is off." runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-suspended" # ===================================================================================== # Certificates — cert-manager # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-certificates namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.certificates rules: - alert: CertificateExpiringSoon expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (14 * 24 * 3600) for: 1h labels: severity: warning annotations: summary: "Certificate expiring soon: {{ $labels.name }}" description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 14 days." runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry" - alert: CertificateExpiryImminent expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (3 * 24 * 3600) for: 1h labels: severity: critical annotations: summary: "Certificate expiry imminent: {{ $labels.name }}" description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 3 days and has not renewed." runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry" - alert: CertificateNotReady expr: certmanager_certificate_ready_status{condition="False"} == 1 for: 1h labels: severity: warning annotations: summary: "Certificate not ready / issuance failing: {{ $labels.name }}" description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} has been in a not-ready state for 1h (ACME issuance failure?)." runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-issuance" # ===================================================================================== # Networking — VPN / egress-gateway / denied flows (Hubble) # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-networking namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.networking rules: # VPN tunnel health (ADR-0015: node-level host OpenVPN tunnel on the egress-gateway node, steered # by Cilium Egress Gateway — there is NO in-cluster gateway pod). OpenVPN creates/destroys the tun0 # device with the connection, so the tun0 netdev series exists iff the tunnel is up. node_network_up # is unreliable here (tun0 operstate=unknown => 0), so key on the byte-counter's PRESENCE. absent() # also fires if the gateway node / its node-exporter is down (the tun0 series goes stale), covering # gateway-node-down as well. Metric confirmed live on the dev cluster 2026-08-24. # # FIXED 2026-09-06 — the bare `absent(...)` was evaluated CLUSTER-WIDE, so on a cluster with NO # egress-gateway node it could never clear. Prod has no gateway node, and this alert fired # continuously from 2026-09-05T21:23Z into `email-critical` — unbounded noise on the one channel # that must stay trustworthy. The expression is now GATED on the cluster actually being supposed # to have a tunnel, via two independent guards OR'd together: # # 1. A node carries `homelab.keithjackson.dev/egress-gateway=true` — the same label the # CiliumEgressGatewayPolicy nodeSelector uses, so the alert is armed exactly when the # steering datapath has a target. Requires the kube-state-metrics node-label allowlist # enabled in kube-prometheus-stack.yaml (KSM exposes no custom labels by default). This is # what makes a gateway-less cluster SELF-ARM the moment a gateway node is labelled — no # manual re-enable to forget. The Node object survives a NotReady node, so this stays true # while the gateway node is DOWN: gateway-node-down is still alerted. # 2. `tun0` was observed at all in the last hour — holds the alert armed through a # kube-state-metrics outage, and keeps dev's behaviour unchanged even before/independently # of the KSM rollout. # # `and on()` joins on the empty label set deliberately: absent() carries `device="tun0"` while the # guards carry no labels, so an unqualified `and` would never match. Both clauses verified live # against dev and prod Prometheus (read-only) on 2026-09-06: empty on prod (cannot fire), empty on # dev (tunnel up), and clause 2 already true on dev. - alert: VPNTunnelDown expr: | absent(node_network_receive_bytes_total{device="tun0"}) and on() ( (count(kube_node_labels{label_homelab_keithjackson_dev_egress_gateway="true"}) > 0) or (count(count_over_time(node_network_receive_bytes_total{device="tun0"}[1h])) > 0) ) for: 5m labels: severity: critical annotations: summary: "Egress VPN tunnel down" description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)." runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-down" # (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun # retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd # service, steering is Cilium eBPF). Tunnel + gateway-node health is covered by `VPNTunnelDown` # above (tun0 presence, which also goes absent if the gateway node is down); Cilium datapath # health is covered by the Cilium platform alerts. # *** KNOWN INERT — DELIBERATELY NOT FIXED IN THIS CHANGE. DO NOT "fix" it by adding a # *** ServiceMonitor: a monitor alone cannot make this rule fire. # Registered as such in conformance/tests/l0-static/alert-metrics-have-scrape-targets # (KNOWN_UNWIRED_METRICS), which is what keeps this comment from rotting into a lie: a # stale entry there FAILS the gate, so this cannot quietly become fiction. # # VERIFIED live on prod 2026-09-13: # count(hubble_drop_total) => (empty) # count({__name__=~"hubble.*"}) => (empty) # count({__name__=~"cilium.*"}) => (empty) # count by (job) (up) => 20 jobs, none Cilium or Hubble # (re-verified unchanged on prod 2026-09-13 at 21:19Z, ~15h after the first reading) # So the rule evaluates `rate()` and returns no series: it cannot fire, and the # platform has NO policy-drop visibility at all today. # # WHY A MONITOR IS NOT THE FIX. Unlike Longhorn/cert-manager (whose charts already serve # /metrics and only lacked a ServiceMonitor) Hubble publishes NOTHING to scrape until it is # switched on in Cilium's own config. platform/cilium/helmrelease.yaml sets # `hubble.{enabled,relay.enabled,ui.enabled}: true` but NOT `hubble.metrics.enabled`, and not # `prometheus.enabled` for the agent — which is why the cilium_* family is missing too. Turning # them on means: # * `hubble.metrics.enabled: [drop, ...]` + `hubble.metrics.serviceMonitor.enabled: true`, # which OPENS PORT 9965 ON EVERY cilium-agent AND ROLLS THE CNI DaemonSet across all # three nodes. A cilium-agent roll is the single most dangerous rollout on this cluster: # the 2026-09-01 k8sServiceHost outage is in DECISIONS-LOG, and cilium was moved into its # OWN Flux Kustomization precisely because a CNI rollout needs an isolated blast radius. # * a matching change to the ANSIBLE bootstrap values. platform/cilium/helmrelease.yaml # carries `# [parity] MUST equal Ansible: hubble.{enabled,relay.enabled,ui.enabled}: true` # against roles/cilium/templates/cilium-values.yaml.j2. Changing only the HelmRelease makes # a rebuilt node's bootstrap CNI config disagree with the reconciled one — a drift that is # invisible until the next node rebuild. # * per-flow label cardinality: `drop` metrics carry source/destination context, and the # default contexts can multiply series across every pod pair that gets denied — on a # default-deny cluster, denials are routine, not exceptional. # RECOMMENDATION (for the PM, not to be done silently here): schedule this as its own change # on the cilium Kustomization, with the Ansible parity edit in the same commit, the # labelsContext pinned narrow (e.g. `hubble.metrics.enabled: ["drop:sourceContext=pod;destinationContext=pod"]`), # and a one-node canary before the full DaemonSet roll. Until then this alert is honestly # inert and the gap is: NO policy-drop / NetworkPolicy-regression visibility (docs/07 security # audit), which also means a NetworkPolicy change cannot be observed to have started dropping # legitimate traffic. - alert: HubbleDeniedFlowSpike # Cilium/Hubble denied (policy-drop) flows. Requires hubble metrics enabled (docs/03). expr: sum(rate(hubble_drop_total{reason="POLICY_DENIED"}[5m])) > 5 for: 10m labels: severity: warning annotations: summary: "Spike in policy-denied network flows" description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)." runbook_url: "https://${BASE_DOMAIN}/runbooks/denied-flows" # ===================================================================================== # Backups — Velero # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-backups namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.backups rules: - alert: VeleroBackupFailure expr: increase(velero_backup_failure_total[1h]) > 0 for: 5m labels: severity: critical annotations: summary: "Velero backup failed: schedule {{ $labels.schedule }}" description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-failure" - alert: VeleroBackupPartialFailure expr: increase(velero_backup_partial_failure_total[1h]) > 0 for: 5m labels: severity: warning annotations: summary: "Velero backup partially failed" description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial" # Covers "backed up before, then STOPPED". It is deliberately per-schedule and # un-aggregated, but it is blind to a schedule that has never succeeded ONCE — see the # block below, which closes that hole rather than complicating this expression. - alert: VeleroNoRecentBackup expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600) for: 1h labels: severity: critical annotations: summary: "No successful Velero backup in 24h: {{ $labels.schedule }}" description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # --- THE "NEVER BACKED UP" BLIND SPOT (closed 2026-09-13) ------------------------------ # VeleroNoRecentBackup cannot fire for a schedule that has NEVER succeeded. Velero creates # velero_backup_last_successful_timestamp{schedule="X"} only when a backup for X actually # succeeds, so for a never-successful schedule the series is ABSENT and # `time() - ` evaluates to no series at all. The rule detects "stopped # backing up"; it is structurally inert on "never backed up". # # That is not hypothetical. Measured on prod 2026-09-12/13: `velero-daily` — the schedule # that carries VOLUME DATA — sat `paused: true` with zero successful backups, so the cluster # had NO volume backups and nothing alerted. It was doubly masked, because # `velero-daily-resources-only` ran cleanly every 03:00: every other backup signal # (failure_total, partial_failure_total, "last backup" in the UI) looked green while only # Kubernetes OBJECTS were being protected. # # WHY NOT A PROMQL-ONLY FIX (verified, prod 2026-09-13, velero v1.18.1): the Velero metrics # endpoint exports 35 velero_* metric names, and EVERY one carrying a `schedule` label is # created by a backup ATTEMPT, not by the Schedule object. A throwaway Schedule created with # `paused: true` produced ZERO velero_* series (`curl velero:8085/metrics | grep -c ` # -> 0). There is no "schedule exists" metric and no `spec.paused` metric, so no expression # over velero_* can see either failure mode. The Schedule CRs are therefore exported to # Prometheus by kube-state-metrics' custom-resource-state collector as # kube_velero_schedule_{created,paused,protects_volume_data,info} — configured in # kube-prometheus-stack.yaml, which is the same repo-owned exporter already used for # kube_node_labels, not a new sidecar. # # MASKING, and why the rules key on a PROPERTY rather than a schedule NAME: # `kube_velero_schedule_protects_volume_data` is read straight from # `spec.template.defaultVolumesToFsBackup`, which IS the definition of "this schedule backs # up volume data". So VeleroNoVolumeDataBackup asks the operator's real question — "is # anything at all protecting volume data right now?" — and a resources-only schedule can # never answer it, whatever it is called. Same stance as platform/velero/volume-policies.yaml # (test the property, not a name list a future object would slip past): renaming, adding or # removing schedules cannot re-open the hole. - alert: VeleroScheduleNeverBackedUp expr: | (time() - kube_velero_schedule_created) > (36 * 3600) unless on (schedule) velero_backup_last_successful_timestamp{schedule!=""} for: 1h labels: severity: critical annotations: summary: "Velero schedule has NEVER produced a successful backup: {{ $labels.schedule }}" description: "Schedule {{ $labels.schedule }} has existed over 36h and Velero has never recorded a successful backup for it — the case VeleroNoRecentBackup is structurally blind to (prod ran with a paused velero-daily and NO volume backups this way). Check `kubectl -n velero get schedules.velero.io` for paused, and `backups.velero.io` (never the bare `backup` short name)." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # The direct root-cause signal. A paused backup schedule is a legitimate but strictly # TEMPORARY state; it must never be silently indefinite. Warning, not critical, because the # outcome alerts above escalate to critical on their own if the pause outlasts a day — # this one exists to name the cause while it is still cheap to fix. - alert: VeleroSchedulePaused expr: kube_velero_schedule_paused == 1 for: 30m labels: severity: warning annotations: summary: "Velero schedule is PAUSED: {{ $labels.schedule }}" description: "Schedule {{ $labels.schedule }} has spec.paused=true for over 30m and is creating no backups. Unpause it, or delete the Schedule if it is genuinely retired — an indefinitely paused schedule is a silent gap in R-STATE-03 off-cluster DR." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # THE ANTI-MASKING RULE. Fires unless at least one schedule that actually captures volume # data has succeeded in the last 36h, so a healthy resources-only schedule can never make # the platform look protected. `or vector(0)` is load-bearing: `count() == 0` returns # NO series and would itself be inert — exactly the bug being fixed — so the empty case is # given an explicit 0. It also means this rule fires if kube-state-metrics stops exporting # the Schedule CRs at all: ABSENCE OF THE SIGNAL IS THE ALARM, by design. # Corollary, measured: evaluated against prod BEFORE the exporter exists this expression # already returns `{} = 0`, i.e. it would fire. That is correct — there is genuinely no # proof of volume-data protection without it — and the `for: 1h` is what keeps it from # paging during the reconcile: the PrometheusRule and the kube-state-metrics values ship in # the SAME Flux Kustomization (platform/monitoring), and the KSM rollout completes in # minutes. If this ever pages an hour after a monitoring reconcile, the exporter did not # come up — check `kubectl -n monitoring logs deploy/...-kube-state-metrics` for a # `customresourcedefinitions ... is forbidden` RBAC error before suspecting Velero. - alert: VeleroNoVolumeDataBackup expr: | ( count( (kube_velero_schedule_protects_volume_data == 1) and on (schedule) ((time() - velero_backup_last_successful_timestamp{schedule!=""}) < (36 * 3600)) ) or vector(0) ) == 0 for: 1h labels: severity: critical annotations: summary: "NO Velero schedule has backed up volume DATA in 36h" description: "No Schedule with spec.template.defaultVolumesToFsBackup=true has succeeded in 36h, so PVC CONTENTS are unprotected off-cluster however healthy the resources-only schedule looks. A metadata-only backup restores objects, not data — R-STATE-03 is not satisfied." runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # ===================================================================================== # Identity — Keycloak (SSO is a critical dependency — docs/15) # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-identity namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.identity rules: - alert: KeycloakDown # Scope to the Keycloak SERVER StatefulSet pods (keycloak-0, keycloak-1). The keycloak # namespace also holds the operator (keycloak-operator-*) and DB (keycloak-db-*) pods, so an # un-scoped max() would be masked by those being ready — this must reflect the IdP itself. expr: | max(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-[0-9]+", condition="true"}) == 0 or absent(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-[0-9]+"}) for: 5m labels: severity: critical annotations: summary: "Keycloak IdP down" description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)." runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-down" - alert: KeycloakDatabaseDown # Keycloak's PostgreSQL runs as a CloudNativePG Cluster `keycloak-db` (pods keycloak-db-N). # DR-critical, Longhorn-backed. (CNPG also exports cnpg_* metrics via its PodMonitor.) expr: | max(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-db-.*", condition="true"}) == 0 or absent(kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-db-.*"}) for: 5m labels: severity: critical annotations: summary: "Keycloak database unavailable" description: "Keycloak's PostgreSQL is not ready for 5m — the IdP will fail (docs/15)." runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-db" # --- The client reconciler (platform/keycloak/client-secret-reconciler.yaml) --- # WHY THESE EXIST: on 2026-09-06 every OIDC login on prod failed because the realm import # had stored the LITERAL `(env:...)` placeholder as all 12 client secrets — and every # signal the cluster offered was GREEN (the import Job succeeded, the CR read Done=True, # KeycloakDown/KeycloakDatabaseDown were quiet, the conformance test that asserts # Done=True passed). The only observable was a human failing to log in. The CronJob is # the healer; these two rules make its SILENCE observable, so the class cannot sit # undetected again. They are the runtime counterpart of the L2 audit test. # # THE CronJob NOW RECONCILES TWO THINGS — client secrets, and the environment-derived # client URLs (redirectUris, webOrigins, rootUrl, baseUrl, adminUrl) that a BASE_DOMAIN # migration re-renders in git and the one-shot import never carries to the realm. The # pair below is EXTENDED to say so rather than DUPLICATED: one CronJob gets one # Failing/Stale pair, because a second pair keyed to the same job_name regex would fire # twice for a single cause and train the operator to ignore both. The Job log names which # pass failed, which client, and for a URL the exact before/after. # # UNVERIFIED-ON-CLUSTER at authoring time: no CronJob existed on either cluster yet, so # these series had no samples to check against. `kube_job_status_failed` / # `kube_job_status_succeeded` and the `job_name` label WERE confirmed present (the realm # import Job exports them); the reconciler's own series appear with its first run. # Confirm with: kube_job_status_succeeded{namespace="keycloak", # job_name=~"keycloak-client-secret-reconciler-.*"} - alert: KeycloakClientSecretReconcilerFailing # A reconciler run failed. Either it found drift it could not repair, or it could not # reach the admin API. Fires alongside KeycloakDown during an IdP outage — that is # deliberate (the reconciler genuinely is not running), so treat KeycloakDown as the # cause when both are lit. Self-clears when the failed Jobs age out (TTL 1h). expr: | max(kube_job_status_failed{namespace="keycloak", job_name=~"keycloak-client-secret-reconciler-.*"}) > 0 for: 15m labels: severity: warning annotations: summary: "Keycloak client reconciler failing" description: >- A keycloak-client-secret-reconciler Job has been failing for 15m — an OIDC client's stored SECRET or its REDIRECT URIs may differ from git (an unresolved import placeholder, a pre-migration hostname the one-shot realm import never updated, or an unrepairable drift), and SSO logins for that client will fail — a bad redirectUri fails with a redirect_uri mismatch at the IdP. Read the Job log: it names the pass, the client and the exact URL before/after, and never prints a secret. runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-client-secrets" - alert: KeycloakClientSecretReconcilerStale # No SUCCESSFUL run in scope. Catches the failure modes the rule above cannot see: the # CronJob suspended, never applied by Flux, or unable to schedule a pod at all — i.e. # the healer being quietly absent, which is exactly the "looks applied and isn't" # failure this whole change exists to end. expr: | absent(kube_job_status_succeeded{namespace="keycloak", job_name=~"keycloak-client-secret-reconciler-.*"} == 1) for: 2h labels: severity: warning annotations: summary: "Keycloak client reconciler has not succeeded" description: >- No successful keycloak-client-secret-reconciler run in 2h (schedule is every 15m). The CronJob may be suspended, unapplied, or unable to schedule — client-secret AND redirect-URI drift would go uncorrected and unnoticed (docs/15). This is the rule that would catch the healer being quietly absent during a domain migration. runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-client-secrets" # ===================================================================================== # The monitor's own capacity — don't let the monitor blind itself (docs/10) # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-monitoring-self namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.monitoring-self rules: - alert: MonitoringVolumeFillingUp expr: | (kubelet_volume_stats_available_bytes{namespace="monitoring"} / kubelet_volume_stats_capacity_bytes{namespace="monitoring"}) < 0.15 for: 15m labels: severity: warning annotations: summary: "Monitoring PVC filling up: {{ $labels.persistentvolumeclaim }}" description: "PVC {{ $labels.persistentvolumeclaim }} (Prometheus/Loki) is under 15% free — retention/PVC size may need raising before the monitor goes blind." runbook_url: "https://${BASE_DOMAIN}/runbooks/monitoring-capacity" - alert: PrometheusTargetsMissing expr: | (count(up == 0) / count(up)) > 0.10 for: 15m labels: severity: warning annotations: summary: "More than 10% of Prometheus targets are down" description: "Over 10% of scrape targets have been down for 15m — a broad monitoring blind spot." runbook_url: "https://${BASE_DOMAIN}/runbooks/targets-down" # ===================================================================================== # Automated updates — Flux image-automation + hands-off update health (docs/11, ADR-0009) # R-UPD-02 (a failed/rolled-back update alerts) + R-OBS. All Flux controllers (incl. the image # reflector/automation + helm controllers) export the shared `gotk_reconcile_condition` / # `gotk_suspend_status` series; these rules add UPDATE-SPECIFIC visibility on top of the generic # platform-health.flux group so a silently-stalled or a rolled-back update is not missed. # ===================================================================================== --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: platform-health-automated-updates namespace: monitoring labels: release: kube-prometheus-stack platform-health: "true" spec: groups: - name: platform-health.automated-updates rules: # The write-back engine (ADR-0009) is broken → hands-off image updates SILENTLY stop advancing. - alert: ImageUpdateAutomationFailing expr: | max by (namespace, name) ( gotk_reconcile_condition{kind="ImageUpdateAutomation", type="Ready", status="False"} ) == 1 for: 30m labels: severity: warning annotations: summary: "Flux image-update-automation failing: {{ $labels.name }}" description: "ImageUpdateAutomation {{ $labels.namespace }}/{{ $labels.name }} cannot reconcile (git write-back) for 30m — stable image advances have stopped (R-UPD-02). Check the git write credential." runbook_url: "https://${BASE_DOMAIN}/runbooks/image-automation" # An ImagePolicy/ImageRepository can't resolve the latest stable tag (registry rate-limit / # auth / bad range) → that service stops receiving updates even though automation is healthy. - alert: ImagePolicyStalled expr: | max by (namespace, name, kind) ( gotk_reconcile_condition{kind=~"ImagePolicy|ImageRepository", type="Ready", status="False"} ) == 1 for: 1h labels: severity: warning annotations: summary: "Image scan/policy stalled: {{ $labels.kind }}/{{ $labels.name }}" description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} cannot resolve its stable tag for 1h (registry rate-limit / creds / range) — no longer receiving updates (R-UPD-01)." runbook_url: "https://${BASE_DOMAIN}/runbooks/image-policy-stalled" # A bad update was health-gated and AUTO-ROLLED-BACK (or is stuck mid-remediation). helm-controller # holds Ready=False while a failed upgrade remediates/rolls back to the last-good revision # (HelmRelease.spec.upgrade.remediation + .spec.rollback). This is the R-UPD-03 safety rail firing. # TODO(bring-up): tighten with the `reason` label (e.g. UpgradeFailed / RolledBack) once confirmed # against the deployed helm-controller metrics. - alert: UpdateHealthGateFailed expr: | max by (namespace, name) ( gotk_reconcile_condition{kind="HelmRelease", type="Ready", status="False"} ) == 1 for: 10m labels: severity: critical annotations: summary: "Update health-gate failed / rolled back: {{ $labels.name }}" description: "HelmRelease {{ $labels.namespace }}/{{ $labels.name }} failed its health-gated upgrade and is rolling back to the last-good revision (bad-update auto-rollback — R-UPD-03). Consider an `update.pin` escape hatch (docs/11)." runbook_url: "https://${BASE_DOMAIN}/runbooks/update-rollback"