diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index 4323c66..f6d7ed9 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -59,6 +59,34 @@ spec: summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})" description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m." runbook_url: "https://home.keithjackson.dev/runbooks/node-disk" + - alert: NodeCPUPressureHigh + # PSI "some" for CPU = fraction of wall-clock time at least one task was stalled waiting to + # run. <5% is healthy; sustained >15% means bursty workloads are queueing against the vCPU + # cap (measured 15-23% at 6 vCPU before the 8-vCPU resize). Averaged CPU% (kubectl top) hides + # this — PSI is the metric that actually surfaces the "spikes cause blips of delay" symptom. + expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.15 + for: 15m + labels: + severity: warning + annotations: + summary: "CPU pressure high on {{ $labels.instance }}" + description: "CPU PSI (some) above 15% for 15m on {{ $labels.instance }} — tasks are being delayed waiting for CPU (bursts exceeding the node's vCPU headroom). Add vCPUs or trim CPU-heavy workloads." + runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu" + - alert: NodeCPULoadHigh + # 5-min load average per core. Sustained >1 means the run queue exceeds the core count (more + # runnable/blocked tasks than CPUs). Core count = number of idle-mode cpu series per node. + # Pair with NodeCPUPressureHigh to confirm the contention is CPU (vs I/O, which inflates load + # but not CPU-PSI). + expr: | + node_load5 + / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 1 + for: 15m + labels: + severity: warning + annotations: + summary: "Load per core high on {{ $labels.instance }}" + description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count." + runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu" # ===================================================================================== # Storage — Longhorn diff --git a/platform/monitoring/kustomization.yaml b/platform/monitoring/kustomization.yaml index 4755db3..e1ef68a 100644 --- a/platform/monitoring/kustomization.yaml +++ b/platform/monitoring/kustomization.yaml @@ -7,6 +7,9 @@ resources: - kube-prometheus-stack.yaml - loki.yaml - alloy.yaml + # Node CPU/memory PRESSURE dashboard — plain ConfigMap (grafana_dashboard label), no CRD dependency, + # so it applies in this pass. Pairs with the NodeCPUPressureHigh/NodeCPULoadHigh rules in alerts.yaml. + - node-pressure-dashboard.yaml # NOTE: alerts.yaml (PrometheusRule CRs) needs the monitoring.coreos.com CRDs that the # kube-prometheus-stack HelmRelease above installs → applied by the platform-config Kustomization # (dependsOn platform) so the CRDs exist first. File stays here for locality. diff --git a/platform/monitoring/node-pressure-dashboard.yaml b/platform/monitoring/node-pressure-dashboard.yaml new file mode 100644 index 0000000..ed86626 --- /dev/null +++ b/platform/monitoring/node-pressure-dashboard.yaml @@ -0,0 +1,107 @@ +--- +# Grafana dashboard for node CPU/memory PRESSURE (docs/10 — platform-health capacity view). A plain +# ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads +# (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node +# dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay". +# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh alerts in alerts.yaml. +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-pressure-dashboard + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + node-pressure.json: | + { + "title": "Nodes — CPU / Memory Pressure", + "uid": "node-pressure", + "schemaVersion": 39, + "timezone": "browser", + "tags": ["nodes", "platform", "capacity"], + "time": {"from": "now-6h", "to": "now"}, + "templating": {"list": []}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Max CPU pressure (PSI some, now)", + "description": "Fraction of time some task was stalled waiting for CPU. <5% healthy, >15% contended.", + "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, + "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [ + {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 15} + ]}}}, + "targets": [ + {"expr": "max(rate(node_pressure_cpu_waiting_seconds_total[5m])) * 100", "legendFormat": "max CPU PSI"} + ] + }, + { + "id": 2, + "type": "stat", + "title": "Max load per core (5m, now)", + "description": "node_load5 / cores. >1 means the run queue exceeds the core count.", + "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, + "fieldConfig": {"defaults": {"unit": "short", "decimals": 2, "thresholds": {"steps": [ + {"color": "green", "value": null}, {"color": "yellow", "value": 0.8}, {"color": "red", "value": 1} + ]}}}, + "targets": [ + {"expr": "max(node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"}))", "legendFormat": "max load/core"} + ] + }, + { + "id": 3, + "type": "stat", + "title": "Max CPU utilization (now)", + "description": "Averaged busy%. Note: this looks calm even while PSI is high — that is the point.", + "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, + "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [ + {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} + ]}}}, + "targets": [ + {"expr": "max(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "max CPU util"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "CPU pressure (PSI some) per node", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}}, + "targets": [ + {"expr": "rate(node_pressure_cpu_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"} + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Load per core per node", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 10, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"steps": [ + {"color": "green", "value": null}, {"color": "red", "value": 1} + ]}}}, + "targets": [ + {"expr": "node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"})", "legendFormat": "{{instance}}"} + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "CPU utilization per node", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, + "fieldConfig": {"defaults": {"unit": "percent", "max": 100, "custom": {"drawStyle": "line", "fillOpacity": 10}}}, + "targets": [ + {"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "{{instance}}"} + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Memory pressure (PSI some) per node", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, + "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}}, + "targets": [ + {"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"} + ] + } + ] + }