monitoring: node CPU-pressure alerts + PSI/load dashboard (#4)
Adds NodeCPUPressureHigh (CPU PSI 'some' > 15% for 15m) and NodeCPULoadHigh (load5/core > 1 for 15m) to platform-health.nodes, plus a 'Nodes — CPU / Memory Pressure' Grafana dashboard (PSI + load-per-core + utilization). These surface the bursty CPU contention that averaged CPU%/kubectl top hides — the signal behind 'spikes cause blips of delay'. Makes the 6->8 vCPU resize measurable. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed
1 parent
3e61f817b6
commit
a720c73fcd
3 files changed
+138
No files matched your search
@@ -59,6 +59,34 @@ spec:
|
||||
summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
||||
description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-disk"
|
||||
- alert: NodeCPUPressureHigh
|
||||
# PSI "some" for CPU = fraction of wall-clock time at least one task was stalled waiting to
|
||||
# run. <5% is healthy; sustained >15% means bursty workloads are queueing against the vCPU
|
||||
# cap (measured 15-23% at 6 vCPU before the 8-vCPU resize). Averaged CPU% (kubectl top) hides
|
||||
# this — PSI is the metric that actually surfaces the "spikes cause blips of delay" symptom.
|
||||
expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.15
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CPU pressure high on {{ $labels.instance }}"
|
||||
description: "CPU PSI (some) above 15% for 15m on {{ $labels.instance }} — tasks are being delayed waiting for CPU (bursts exceeding the node's vCPU headroom). Add vCPUs or trim CPU-heavy workloads."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu"
|
||||
- alert: NodeCPULoadHigh
|
||||
# 5-min load average per core. Sustained >1 means the run queue exceeds the core count (more
|
||||
# runnable/blocked tasks than CPUs). Core count = number of idle-mode cpu series per node.
|
||||
# Pair with NodeCPUPressureHigh to confirm the contention is CPU (vs I/O, which inflates load
|
||||
# but not CPU-PSI).
|
||||
expr: |
|
||||
node_load5
|
||||
/ count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Load per core high on {{ $labels.instance }}"
|
||||
description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu"
|
||||
|
||||
# =====================================================================================
|
||||
# Storage — Longhorn
|
||||
|
||||
@@ -7,6 +7,9 @@ resources:
|
||||
- kube-prometheus-stack.yaml
|
||||
- loki.yaml
|
||||
- alloy.yaml
|
||||
# Node CPU/memory PRESSURE dashboard — plain ConfigMap (grafana_dashboard label), no CRD dependency,
|
||||
# so it applies in this pass. Pairs with the NodeCPUPressureHigh/NodeCPULoadHigh rules in alerts.yaml.
|
||||
- node-pressure-dashboard.yaml
|
||||
# NOTE: alerts.yaml (PrometheusRule CRs) needs the monitoring.coreos.com CRDs that the
|
||||
# kube-prometheus-stack HelmRelease above installs → applied by the platform-config Kustomization
|
||||
# (dependsOn platform) so the CRDs exist first. File stays here for locality.
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
---
|
||||
# Grafana dashboard for node CPU/memory PRESSURE (docs/10 — platform-health capacity view). A plain
|
||||
# ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads
|
||||
# (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node
|
||||
# dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay".
|
||||
# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh alerts in alerts.yaml.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: node-pressure-dashboard
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
node-pressure.json: |
|
||||
{
|
||||
"title": "Nodes — CPU / Memory Pressure",
|
||||
"uid": "node-pressure",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"tags": ["nodes", "platform", "capacity"],
|
||||
"time": {"from": "now-6h", "to": "now"},
|
||||
"templating": {"list": []},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Max CPU pressure (PSI some, now)",
|
||||
"description": "Fraction of time some task was stalled waiting for CPU. <5% healthy, >15% contended.",
|
||||
"gridPos": {"h": 6, "w": 6, "x": 0, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
|
||||
{"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 15}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "max(rate(node_pressure_cpu_waiting_seconds_total[5m])) * 100", "legendFormat": "max CPU PSI"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Max load per core (5m, now)",
|
||||
"description": "node_load5 / cores. >1 means the run queue exceeds the core count.",
|
||||
"gridPos": {"h": 6, "w": 6, "x": 6, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "short", "decimals": 2, "thresholds": {"steps": [
|
||||
{"color": "green", "value": null}, {"color": "yellow", "value": 0.8}, {"color": "red", "value": 1}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "max(node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"}))", "legendFormat": "max load/core"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Max CPU utilization (now)",
|
||||
"description": "Averaged busy%. Note: this looks calm even while PSI is high — that is the point.",
|
||||
"gridPos": {"h": 6, "w": 6, "x": 12, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
|
||||
{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "max(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "max CPU util"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "CPU pressure (PSI some) per node",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
|
||||
"targets": [
|
||||
{"expr": "rate(node_pressure_cpu_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Load per core per node",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
|
||||
"fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 10, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"steps": [
|
||||
{"color": "green", "value": null}, {"color": "red", "value": 1}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"})", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "CPU utilization per node",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 14},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "max": 100, "custom": {"drawStyle": "line", "fillOpacity": 10}}},
|
||||
"targets": [
|
||||
{"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Memory pressure (PSI some) per node",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 14},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
|
||||
"targets": [
|
||||
{"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user