---
# Grafana dashboard for node CPU/memory/IO PRESSURE (docs/10 — platform-health capacity view). A plain
# ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads
# (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node
# dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay".
# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh / NodeIOPressureHigh alerts in alerts.yaml.
#
# THE I/O ROW WAS ADDED AFTER 2026-09-07. k3s-prod-1 went NotReady that morning with the host
# still up: its embedded etcd could not complete WAL fdatasync (3.5-6.7 s against a 1 s
# expectation) because Longhorn replicas and etcd share one partition on a 2011 consumer SATA
# SSD. `/proc/pressure/io` FULL avg300 was 11.36 on that node vs 0.75 (NVMe node 2) and 1.29
# (node 3) — and this dashboard, like the alerts, plotted only CPU and memory. I/O PSI *full*
# (every task blocked on disk) is the series that tracked the outage; `some` is too noisy on a
# storage node to read. Both are plotted, `full` first.
apiVersion: v1
kind: ConfigMap
metadata:
  name: node-pressure-dashboard
  namespace: monitoring
  labels:
    grafana_dashboard: "1"
data:
  node-pressure.json: |
    {
      "title": "Nodes — CPU / Memory / I-O Pressure",
      "uid": "node-pressure",
      "schemaVersion": 39,
      "timezone": "browser",
      "tags": ["nodes", "platform", "capacity"],
      "time": {"from": "now-6h", "to": "now"},
      "templating": {"list": []},
      "panels": [
        {
          "id": 1,
          "type": "stat",
          "title": "Max CPU pressure (PSI some, now)",
          "description": "Fraction of time some task was stalled waiting for CPU. <5% healthy, >15% contended.",
          "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0},
          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 15}
          ]}}},
          "targets": [
            {"expr": "max(rate(node_pressure_cpu_waiting_seconds_total[5m])) * 100", "legendFormat": "max CPU PSI"}
          ]
        },
        {
          "id": 2,
          "type": "stat",
          "title": "Max load per core (5m, now)",
          "description": "node_load5 / cores. >1 means the run queue exceeds the core count.",
          "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0},
          "fieldConfig": {"defaults": {"unit": "short", "decimals": 2, "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 0.8}, {"color": "red", "value": 1}
          ]}}},
          "targets": [
            {"expr": "max(node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"}))", "legendFormat": "max load/core"}
          ]
        },
        {
          "id": 3,
          "type": "stat",
          "title": "Max CPU utilization (now)",
          "description": "Averaged busy%. Note: this looks calm even while PSI is high — that is the point.",
          "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0},
          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90}
          ]}}},
          "targets": [
            {"expr": "max(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "max CPU util"}
          ]
        },
        {
          "id": 4,
          "type": "timeseries",
          "title": "CPU pressure (PSI some) per node",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "rate(node_pressure_cpu_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 5,
          "type": "timeseries",
          "title": "Load per core per node",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
          "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 10, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 1}
          ]}}},
          "targets": [
            {"expr": "node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"})", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 6,
          "type": "timeseries",
          "title": "CPU utilization per node",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14},
          "fieldConfig": {"defaults": {"unit": "percent", "max": 100, "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 7,
          "type": "timeseries",
          "title": "Memory pressure (PSI some) per node",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 8,
          "type": "stat",
          "title": "Max I/O pressure (PSI full, now)",
          "description": "Fraction of time EVERY task on the node was blocked on disk. Steady state on these nodes is ~1%. k3s-prod-1 sat at 11.4% while its etcd stalled and the node went NotReady on 2026-09-07.",
          "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0},
          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10}
          ]}}},
          "targets": [
            {"expr": "max(rate(node_pressure_io_stalled_seconds_total[5m])) * 100", "legendFormat": "max IO PSI full"}
          ]
        },
        {
          "id": 9,
          "type": "timeseries",
          "title": "I/O pressure (PSI full) per node — etcd stall risk",
          "description": "Thresholds match the NodeIOPressureHigh (5%) and NodeIOPressureCritical (10%) alerts. A node held above 10% is the shape that starved etcd's WAL fsync on 2026-09-07.",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 22},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10}
          ]}}},
          "targets": [
            {"expr": "rate(node_pressure_io_stalled_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 10,
          "type": "timeseries",
          "title": "I/O pressure (PSI some) per node",
          "description": "Time at least one task was blocked on disk. Always higher than `full`; useful for spotting which node carries the write load, not for alerting.",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 22},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "rate(node_pressure_io_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        }
      ]
    }