monitoring: node CPU-pressure alerts + PSI/load dashboard (#4)

Adds NodeCPUPressureHigh (CPU PSI 'some' > 15% for 15m) and NodeCPULoadHigh (load5/core > 1 for 15m) to platform-health.nodes, plus a 'Nodes — CPU / Memory Pressure' Grafana dashboard (PSI + load-per-core + utilization). These surface the bursty CPU contention that averaged CPU%/kubectl top hides — the signal behind 'spikes cause blips of delay'. Makes the 6->8 vCPU resize measurable. Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-08-30 21:51:15 -05:00
1 parent 3e61f817b6
commit a720c73fcd
@@ -0,0 +1,107 @@
+---
+# Grafana dashboard for node CPU/memory PRESSURE (docs/10 — platform-health capacity view). A plain
+# ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads
+# (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node
+# dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay".
+# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh alerts in alerts.yaml.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-pressure-dashboard
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  node-pressure.json: |
+    {
+      "title": "Nodes — CPU / Memory Pressure",
+      "uid": "node-pressure",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "tags": ["nodes", "platform", "capacity"],
+      "time": {"from": "now-6h", "to": "now"},
+      "templating": {"list": []},
+      "panels": [
+        {
+          "id": 1,
+          "type": "stat",
+          "title": "Max CPU pressure (PSI some, now)",
+          "description": "Fraction of time some task was stalled waiting for CPU. <5% healthy, >15% contended.",
+          "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0},
+          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
+            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 15}
+          ]}}},
+          "targets": [
+            {"expr": "max(rate(node_pressure_cpu_waiting_seconds_total[5m])) * 100", "legendFormat": "max CPU PSI"}
+          ]
+        },
+        {
+          "id": 2,
+          "type": "stat",
+          "title": "Max load per core (5m, now)",
+          "description": "node_load5 / cores. >1 means the run queue exceeds the core count.",
+          "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0},
+          "fieldConfig": {"defaults": {"unit": "short", "decimals": 2, "thresholds": {"steps": [
+            {"color": "green", "value": null}, {"color": "yellow", "value": 0.8}, {"color": "red", "value": 1}
+          ]}}},
+          "targets": [
+            {"expr": "max(node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"}))", "legendFormat": "max load/core"}
+          ]
+        },
+        {
+          "id": 3,
+          "type": "stat",
+          "title": "Max CPU utilization (now)",
+          "description": "Averaged busy%. Note: this looks calm even while PSI is high — that is the point.",
+          "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0},
+          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
+            {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90}
+          ]}}},
+          "targets": [
+            {"expr": "max(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "max CPU util"}
+          ]
+        },
+        {
+          "id": 4,
+          "type": "timeseries",
+          "title": "CPU pressure (PSI some) per node",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
+          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
+          "targets": [
+            {"expr": "rate(node_pressure_cpu_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
+          ]
+        },
+        {
+          "id": 5,
+          "type": "timeseries",
+          "title": "Load per core per node",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
+          "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 10, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"steps": [
+            {"color": "green", "value": null}, {"color": "red", "value": 1}
+          ]}}},
+          "targets": [
+            {"expr": "node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"})", "legendFormat": "{{instance}}"}
+          ]
+        },
+        {
+          "id": 6,
+          "type": "timeseries",
+          "title": "CPU utilization per node",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14},
+          "fieldConfig": {"defaults": {"unit": "percent", "max": 100, "custom": {"drawStyle": "line", "fillOpacity": 10}}},
+          "targets": [
+            {"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "{{instance}}"}
+          ]
+        },
+        {
+          "id": 7,
+          "type": "timeseries",
+          "title": "Memory pressure (PSI some) per node",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14},
+          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
+          "targets": [
+            {"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
+          ]
+        }
+      ]
+    }