---
# Grafana dashboard for node CPU/memory/IO PRESSURE (docs/10 — platform-health capacity view). A plain
# ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads
# (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node
# dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay".
# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh / NodeIOPressureHigh alerts in alerts.yaml.
#
# THE I/O ROW WAS ADDED AFTER 2026-09-07. k3s-prod-1 went NotReady that morning with the host
# still up: its embedded etcd could not complete WAL fdatasync (3.5-6.7 s against a 1 s
# expectation) because Longhorn replicas and etcd share one partition on a 2011 consumer SATA
# SSD. `/proc/pressure/io` FULL avg300 was 11.36 on that node vs 0.75 (NVMe node 2) and 1.29
# (node 3) — and this dashboard, like the alerts, plotted only CPU and memory. I/O PSI *full*
# (every task blocked on disk) is the series that tracked the outage; `some` is too noisy on a
# storage node to read. Both are plotted, `full` first.
apiVersion: v1
kind: ConfigMap
metadata:
  name: node-pressure-dashboard
  namespace: monitoring
  labels:
    grafana_dashboard: "1"
data:
  node-pressure.json: |
    {
      "title": "Nodes — CPU / Memory / I-O Pressure",
      "uid": "node-pressure",
      "schemaVersion": 39,
      "timezone": "browser",
      "tags": ["nodes", "platform", "capacity"],
      "time": {"from": "now-6h", "to": "now"},
      "templating": {"list": []},
      "panels": [
        {
          "id": 1,
          "type": "stat",
          "title": "Max CPU pressure (PSI some, now)",
          "description": "Fraction of time some task was stalled waiting for CPU. <5% healthy, >15% contended.",
          "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0},
          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 15}
          ]}}},
          "targets": [
            {"expr": "max(rate(node_pressure_cpu_waiting_seconds_total[5m])) * 100", "legendFormat": "max CPU PSI"}
          ]
        },
        {
          "id": 2,
          "type": "stat",
          "title": "Max load per core (5m, now)",
          "description": "node_load5 / cores. >1 means the run queue exceeds the core count.",
          "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0},
          "fieldConfig": {"defaults": {"unit": "short", "decimals": 2, "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 0.8}, {"color": "red", "value": 1}
          ]}}},
          "targets": [
            {"expr": "max(node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"}))", "legendFormat": "max load/core"}
          ]
        },
        {
          "id": 3,
          "type": "stat",
          "title": "Max CPU utilization (now)",
          "description": "Averaged busy%. Note: this looks calm even while PSI is high — that is the point.",
          "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0},
          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90}
          ]}}},
          "targets": [
            {"expr": "max(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "max CPU util"}
          ]
        },
        {
          "id": 4,
          "type": "timeseries",
          "title": "CPU pressure (PSI some) per node",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "rate(node_pressure_cpu_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 5,
          "type": "timeseries",
          "title": "Load per core per node",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
          "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "fillOpacity": 10, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 1}
          ]}}},
          "targets": [
            {"expr": "node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\"})", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 6,
          "type": "timeseries",
          "title": "CPU utilization per node",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14},
          "fieldConfig": {"defaults": {"unit": "percent", "max": 100, "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 7,
          "type": "timeseries",
          "title": "Memory pressure (PSI some) per node",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 8,
          "type": "stat",
          "title": "Max I/O pressure (PSI full, now)",
          "description": "Fraction of time EVERY task on the node was blocked on disk. Steady state on these nodes is ~1%. k3s-prod-1 sat at 11.4% while its etcd stalled and the node went NotReady on 2026-09-07.",
          "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0},
          "fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10}
          ]}}},
          "targets": [
            {"expr": "max(rate(node_pressure_io_stalled_seconds_total[5m])) * 100", "legendFormat": "max IO PSI full"}
          ]
        },
        {
          "id": 9,
          "type": "timeseries",
          "title": "I/O pressure (PSI full) per node — etcd stall risk",
          "description": "Thresholds match the NodeIOPressureHigh (5%) and NodeIOPressureCritical (10%) alerts. A node held above 10% is the shape that starved etcd's WAL fsync on 2026-09-07.",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 22},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10}
          ]}}},
          "targets": [
            {"expr": "rate(node_pressure_io_stalled_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 10,
          "type": "timeseries",
          "title": "I/O pressure (PSI some) per node",
          "description": "Time at least one task was blocked on disk. Always higher than `full`; useful for spotting which node carries the write load, not for alerting.",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 22},
          "fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "rate(node_pressure_io_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 11,
          "type": "row",
          "title": "I/O pressure — WHY: saturation / device latency / NFS / etcd / workload",
          "collapsed": false,
          "gridPos": {"h": 1, "w": 24, "x": 0, "y": 30},
          "panels": []
        },
        {
          "id": 12,
          "type": "timeseries",
          "title": "CAUSE 1 — device busy fraction, per device",
          "description": "rate(node_disk_io_time_seconds_total[5m]). SATURATION: the device is queue-bound. Alert line at 0.85 (NodeDiskDeviceSaturated). Normal cluster maximum over 48h is 0.61. Tonight's 2026-09-14 spike read 0.27 and 2026-09-07 read 0.79 — so a flat line down here means saturation is NOT the cause, which is the single most useful thing this row tells you.",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 31},
          "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 0.85}
          ]}}},
          "targets": [
            {"expr": "topk(8, rate(node_disk_io_time_seconds_total[5m]))", "legendFormat": "{{instance}} {{device}}"}
          ]
        },
        {
          "id": 13,
          "type": "timeseries",
          "title": "CAUSE 2 — write service time per op (devices above 10 writes/s)",
          "description": "THE DRIVE-IS-DYING SIGNAL, and the one the stack lacked on 2026-09-07 when prod-1's 2011 SSD measured 1,039 ms per write. Alert lines at 20 ms (warning) and 50 ms (critical). Healthy on this cluster is 1-3 ms; the 48h maximum is 3.1 ms. Guarded at >10 writes/s, which also restricts the panel to the etcd-WAL/Longhorn partitions rather than idle media disks.",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 31},
          "fieldConfig": {"defaults": {"unit": "s", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "yellow", "value": 0.020}, {"color": "red", "value": 0.050}
          ]}}},
          "targets": [
            {"expr": "rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m]) and rate(node_disk_writes_completed_total[5m]) > 10", "legendFormat": "{{instance}} {{device}}"}
          ]
        },
        {
          "id": 14,
          "type": "timeseries",
          "title": "CAUSE 3 — read service time per op (devices above 10 reads/s)",
          "description": "Alert line at 250 ms (NodeDiskReadLatencyHigh). Deliberately far above the write thresholds: prod-1's sdg is a spinning media disk and legitimately sustains 156 ms while serving media, so anything under the line here is normal and not a fault. The SSD/NVMe devices sit at 1.6-6.3 ms.",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 39},
          "fieldConfig": {"defaults": {"unit": "s", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 0.250}
          ]}}},
          "targets": [
            {"expr": "rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m]) and rate(node_disk_reads_completed_total[5m]) > 10", "legendFormat": "{{instance}} {{device}}"}
          ]
        },
        {
          "id": 15,
          "type": "timeseries",
          "title": "CAUSE 4 — NFS RPC retransmission ratio, per node",
          "description": "node_nfs_rpc_retransmissions_total / node_nfs_rpcs_total, from /proc/net/rpc/nfs. Alert line at 1% (NodeNFSRetransmissionRateHigh). MEASURED BASELINE IS EXACTLY ZERO on all three nodes over 48h, including throughout the 2026-09-14 spike — so any visible line here means the export at 192.168.2.102, or the path to it, is dropping requests. A flat zero does NOT clear NFS: it only clears the TRANSPORT. A merely SLOW export needs per-mount RPC round-trip time, which arrives with the mountstats collector (see kube-prometheus-stack.yaml) and is the known gap recorded in runbooks/node-io.md.",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 39},
          "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 0.01}
          ]}}},
          "targets": [
            {"expr": "sum by (instance) (rate(node_nfs_rpc_retransmissions_total[5m])) / sum by (instance) (rate(node_nfs_rpcs_total[5m]))", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 16,
          "type": "timeseries",
          "title": "Is it OUR NFS traffic? — client op rate per node and method",
          "description": "THE PANEL THAT ANSWERS THE QUESTION NodeIOPressureNotLocalDisk ASKS. If I/O PSI is high, the local devices are idle and THIS is high, our own workload is hammering the export (prod-1 peaked 114 reads/s during the 2026-09-14 spike, against 0 on the other two nodes). If PSI is high and this is near zero, the load is coming from OUTSIDE this cluster — the Proxmox media stack still has operational control of the same files — and nothing in-cluster will explain it.",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 47},
          "fieldConfig": {"defaults": {"unit": "ops", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 10}}},
          "targets": [
            {"expr": "sum by (instance, method) (rate(node_nfs_requests_total{method=~\"Read|Write|Commit|Getattr|Lookup\"}[5m]))", "legendFormat": "{{instance}} {{method}}"}
          ]
        },
        {
          "id": 17,
          "type": "timeseries",
          "title": "CAUSE 5 — etcd requests slower than 1s, as a fraction, per node",
          "description": "The apiserver's CLIENT-SIDE view of its own local embedded etcd member, read off the le=\"1.0\" bucket. Alert line at 0.5% (NodeEtcdRequestsSlow). etcd's own disk metrics do NOT exist on this cluster — etcd_disk_wal_fsync_duration_seconds and etcd_disk_backend_commit_duration_seconds are absent because kubeEtcd is disabled for k3s' embedded etcd — so this is the proxy, and it is the closest thing to the NotReady precursor. Measured 0 across 48h; at the 2026-09-07 outage window 0.5-2% on all three members. Rises for any etcd slowness, not only disk.",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 47},
          "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 0.005}
          ]}}},
          "targets": [
            {"expr": "1 - (sum by (instance) (rate(etcd_request_duration_seconds_bucket{job=\"apiserver\",le=\"1.0\"}[5m])) / sum by (instance) (rate(etcd_request_duration_seconds_bucket{job=\"apiserver\",le=\"+Inf\"}[5m])))", "legendFormat": "{{instance}}"}
          ]
        },
        {
          "id": 18,
          "type": "timeseries",
          "title": "WHO is stalled — per-container I/O PSI (full), top 10",
          "description": "cAdvisor's per-cgroup PSI. Node PSI says the machine stalled; this says which workload. Alert line at 0.15 (ContainerIOStallHigh). On 2026-09-14 at 02:40Z this read sonarr 0.268 and qbittorrent 0.070 on k3s-prod-1 against under 0.013 everywhere else — one panel, one answer, and sonarr is an NFS media consumer. `container!=\"\"` drops cAdvisor's pod-level duplicate of the same cgroup.",
          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 55},
          "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "custom": {"drawStyle": "line", "fillOpacity": 5, "thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [
            {"color": "green", "value": null}, {"color": "red", "value": 0.15}
          ]}}},
          "targets": [
            {"expr": "topk(10, rate(container_pressure_io_stalled_seconds_total{container!=\"\",namespace!=\"\"}[5m]))", "legendFormat": "{{node}} {{namespace}}/{{pod}} {{container}}"}
          ]
        }
      ]
    }