diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index 24624b3..5260b8e 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -87,6 +87,49 @@ spec: summary: "Load per core high on {{ $labels.instance }}" description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu" + # --- I/O PRESSURE (added after the 2026-09-07 k3s-prod-1 NotReady incident) --------- + # THE METRIC THAT WOULD HAVE CALLED THE 2026-09-07 OUTAGE AND DID NOT EXIST. + # At 06:05:21Z k3s-prod-1 went NotReady ("Kubelet stopped posting node status") with the + # host up, pingable and SSH-able. Root cause was disk I/O starvation of the embedded etcd: + # slow fdatasync took=3.56s/6.71s/4.95s/3.40s (etcd expects 1s) + # apply request took too long took=8.30s (expects 100ms) + # etcdserver: too many requests + # `/proc/pressure/io` FULL avg300 read 11.36 on k3s-prod-1 at the time, against 0.75 on + # k3s-prod-2 (NVMe) and 1.29 on k3s-prod-3. The stack already alerted on CPU-PSI and + # memory but had NO I/O alert at all, so the first signal the operator got was a node + # already out of the cluster. + # + # WHY "full" AND NOT "some": node_pressure_io_stalled_seconds_total is PSI *full* — time + # in which EVERY runnable task was blocked on I/O, i.e. the machine did no useful work. + # `some` (node_pressure_io_waiting_seconds_total) is noisy on a storage node; `full` is + # the one that tracked the outage. + # + # THRESHOLDS ARE MEASURED, NOT GUESSED. rate(...[5m]) is a 0-1 fraction, so the incident's + # 11.36% is 0.1136 and the healthy steady state measured on the same node is ~0.01: + # warning 0.05 for 10m — 5x steady state, well below the outage, leaves room to act + # critical 0.10 for 5m — the level k3s-prod-1 actually sat at while etcd was stalling + # Deliberately NOT scoped to prod: dev shares this file and its k3s VMs are HDD-backed, + # where the same alert is if anything more likely to earn its keep. + - alert: NodeIOPressureHigh + expr: rate(node_pressure_io_stalled_seconds_total[5m]) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "Disk I/O pressure high on {{ $labels.instance }}" + description: "I/O PSI (full) above 5% for 10m on {{ $labels.instance }} — every task on the node is spending real time blocked on disk. On a node whose etcd WAL shares a partition with Longhorn replicas this is the leading indicator of etcd fsync stalls. Check `cat /proc/pressure/io`, `iostat -x 1`, and whether Longhorn is rebuilding replicas." + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" + - alert: NodeIOPressureCritical + # 0.10 is the level k3s-prod-1 held while etcd logged multi-second fdatasync and then + # dropped out of the cluster. At this point etcd is already at risk; do not wait 10m. + expr: rate(node_pressure_io_stalled_seconds_total[5m]) > 0.10 + for: 5m + labels: + severity: critical + annotations: + summary: "Disk I/O starvation on {{ $labels.instance }}" + description: "I/O PSI (full) above 10% for 5m on {{ $labels.instance }} — the level at which k3s-prod-1 stalled etcd's WAL fsync and went NotReady on 2026-09-07 while the host stayed up. Expect etcd 'slow fdatasync' / 'apply request took too long' in `journalctl -u k3s`. Stop or throttle whatever is writing (Longhorn rebuild, backup, restore) before the node drops." + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # ===================================================================================== # Storage — Longhorn diff --git a/platform/monitoring/node-pressure-dashboard.yaml b/platform/monitoring/node-pressure-dashboard.yaml index ed86626..5e44690 100644 --- a/platform/monitoring/node-pressure-dashboard.yaml +++ b/platform/monitoring/node-pressure-dashboard.yaml @@ -1,9 +1,17 @@ --- -# Grafana dashboard for node CPU/memory PRESSURE (docs/10 — platform-health capacity view). A plain +# Grafana dashboard for node CPU/memory/IO PRESSURE (docs/10 — platform-health capacity view). A plain # ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads # (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node # dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay". -# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh alerts in alerts.yaml. +# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh / NodeIOPressureHigh alerts in alerts.yaml. +# +# THE I/O ROW WAS ADDED AFTER 2026-09-07. k3s-prod-1 went NotReady that morning with the host +# still up: its embedded etcd could not complete WAL fdatasync (3.5-6.7 s against a 1 s +# expectation) because Longhorn replicas and etcd share one partition on a 2011 consumer SATA +# SSD. `/proc/pressure/io` FULL avg300 was 11.36 on that node vs 0.75 (NVMe node 2) and 1.29 +# (node 3) — and this dashboard, like the alerts, plotted only CPU and memory. I/O PSI *full* +# (every task blocked on disk) is the series that tracked the outage; `some` is too noisy on a +# storage node to read. Both are plotted, `full` first. apiVersion: v1 kind: ConfigMap metadata: @@ -14,7 +22,7 @@ metadata: data: node-pressure.json: | { - "title": "Nodes — CPU / Memory Pressure", + "title": "Nodes — CPU / Memory / I-O Pressure", "uid": "node-pressure", "schemaVersion": 39, "timezone": "browser", @@ -102,6 +110,43 @@ data: "targets": [ {"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"} ] + }, + { + "id": 8, + "type": "stat", + "title": "Max I/O pressure (PSI full, now)", + "description": "Fraction of time EVERY task on the node was blocked on disk. Steady state on these nodes is ~1%. k3s-prod-1 sat at 11.4% while its etcd stalled and the node went NotReady on 2026-09-07.", + "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, + "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [ + {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10} + ]}}}, + "targets": [ + {"expr": "max(rate(node_pressure_io_stalled_seconds_total[5m])) * 100", "legendFormat": "max IO PSI full"} + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "I/O pressure (PSI full) per node — etcd stall risk", + "description": "Thresholds match the NodeIOPressureHigh (5%) and NodeIOPressureCritical (10%) alerts. A node held above 10% is the shape that starved etcd's WAL fsync on 2026-09-07.", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 22}, + "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}, "thresholds": {"mode": "absolute", "steps": [ + {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10} + ]}}}, + "targets": [ + {"expr": "rate(node_pressure_io_stalled_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"} + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "I/O pressure (PSI some) per node", + "description": "Time at least one task was blocked on disk. Always higher than `full`; useful for spotting which node carries the write load, not for alerting.", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 22}, + "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}}, + "targets": [ + {"expr": "rate(node_pressure_io_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"} + ] } ] }