# promtool rule-unit tests for the platform-health.node-io-cause group (platform/monitoring/alerts.yaml) # # Run with: platform/monitoring/tests/run.sh # (the merge gate runs the same harness via # conformance/tests/l0-static/alert-rules-unit-tested) # # WHAT THIS FILE IS FOR # `NodeIOPressureCritical` has now fired twice on k3s-prod-1 for two OPPOSITE causes and said the # same thing both times. This group adds one rule per cause. Every rule in it is therefore pinned # in both directions against REAL MEASURED PROFILES, not invented ones: # # fixture source of the numbers # -------------------------- ---------------------------------------------------------------- # healthy baseline prod, 2026-09-14 ~03:05Z, and 24-48h maxima from the same TSDB # tonight prod, 2026-09-14 02:40-02:45Z (PSI 20.98%, devices idle) # 2026-09-07 prod, re-queried from the same TSDB at offset 164h # (PSI 73.97%, 1,039 ms per write on prod-1's dying 2011 SSD) # rebuild storm a device pinned at 0.97 with HEALTHY per-op latency # nfs transport a retransmitting export, plus a low-traffic node # read latency a failing device, plus a device below the op-rate guard # # THE ACCEPTANCE TEST the operator asked for is the three middle fixtures read together: # * tonight -> NodeIOPressureNotLocalDisk FIRES, every device rule stays quiet, # ContainerIOStallHigh names sonarr # * 2026-09-07 -> NodeDiskWriteLatencyHigh/Critical FIRE and name the device, # NodeEtcdRequestsSlow FIRES, and NodeIOPressureNotLocalDisk STAYS QUIET # * healthy -> all eight rules silent # One alert set, three verdicts. That is the gap this change closes. # # NON-VACUITY. Three hazards are asserted directly with promql_expr_test rather than trusted: # 1. PROMQL PRECEDENCE. `(ratio) and rate(ops) > 10 > T` does NOT mean what it reads like: # `>` binds tighter than `and`, so it parses as `(ratio) and ((rate(ops) > 10) > T)` and the # threshold T never touches the ratio at all — the rule then fires on ANY busy device. That # was the first draft of three of these rules. The healthy fixture asserts the broken form # returning series and the shipped form returning none, so the mistake cannot come back. # 2. THE OP-RATE GUARDS ARE LOAD-BEARING, NOT DECORATION. Each latency fixture carries a device # whose per-op latency is a full SECOND at a trickle of operations — the arithmetic artefact # the guard exists to suppress — and asserts it does not fire. # 3. `container!=""` IS LOAD-BEARING. cAdvisor emits a pod-level cgroup series alongside the # container one. The healthy fixture gives the pod-level series a value that WOULD fire and # asserts the shipped selector excludes it (double counting a stall is how one event becomes # two pages). # # CLOCK CONVENTION (as in longhorn.test.yaml): promtool's test clock starts at the unix epoch with # a 1m sample interval, so nothing is ever stale inside Prometheus' 5m lookback. Counter fixtures # are written as per-MINUTE increments, so a `0+12.588x120` series is a rate of 12.588/60 = 0.2098 # per second. Every increment below is annotated with the per-second figure it encodes. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # ============================================================================================= # 1. HEALTHY BASELINE — every rule in the group is SILENT. # This is the "not permanently firing" half of the proof for all eight rules, and it also # carries the three non-vacuity assertions described in the header. # ============================================================================================= - interval: 1m name: measured healthy prod state 2026-09-14 - all eight node-io-cause rules quiet input_series: # --- node I/O PSI (full). Measured 0.0120 / 0.0136 / 0.0061 per second. ----------------- - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+0.72x120' # 0.0120/s - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.11:9100"}' values: '0+0.816x120' # 0.0136/s - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.12:9100"}' values: '0+0.366x120' # 0.0061/s # --- device busy fraction. 48h maxima were 0.610 / 0.217 / 0.295; tonight's spike 0.271. -- - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+5.4x120' # 0.09 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdi"}' values: '0+12.6x120' # 0.21 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+3x120' # 0.05 (a spinning media disk) - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.11:9100",device="nvme0n1"}' values: '0+1.2x120' # 0.02 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.12:9100",device="sda"}' values: '0+1.8x120' # 0.03 # --- per-op WRITE service time. Guarded maxima across 48h were 1-3 ms. ------------------ - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+3000x120' # 50 writes/s -> above the >10/s guard - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+4.8x120' # 0.08 s/s / 50 ops/s = 1.6 ms per write - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdi"}' values: '0+1200x120' # 20 writes/s -> above the guard - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdi"}' values: '0+1.92x120' # 1.6 ms per write # THE GUARD'S REASON FOR EXISTING: a media disk doing 3 writes a MINUTE, each taking a full # second. 1,000 ms per write is 20x NodeDiskWriteLatencyCritical's threshold and must not # fire, because three samples are not evidence of anything. - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+3x120' # 0.05 writes/s -> BELOW the >10/s guard - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+3x120' # 1.0 SECOND per write, and deliberately ignored # --- per-op READ service time. The media HDD legitimately measures 156 ms. -------------- - series: 'node_disk_reads_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+1200x120' # 20 reads/s -> above the guard - series: 'node_disk_read_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+187.2x120' # 3.12 s/s / 20 ops/s = 156 ms per read, MEASURED on prod - series: 'node_disk_reads_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+900x120' # 15 reads/s - series: 'node_disk_read_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+1.962x120' # 2.18 ms per read # --- NFS client. Retransmissions measured EXACTLY ZERO across 48h on all three nodes. ---- - series: 'node_nfs_rpcs_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+7200x120' # 120 RPC/s - series: 'node_nfs_rpc_retransmissions_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+0x120' # zero # --- etcd request duration, apiserver client side. Healthy = no request over 1s. --------- - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.10:6443",operation="get",le="1.0"}' values: '0+600x120' # 10/s - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.10:6443",operation="get",le="+Inf"}' values: '0+600x120' # 10/s -> fraction over 1s = 0 - series: 'etcd_request_duration_seconds_count{job="apiserver",instance="192.168.2.10:6443",operation="get"}' values: '0+600x120' # 10/s -> above the >0.1/s traffic guard # --- per-container I/O PSI from cAdvisor. -------------------------------------------------- - series: 'container_pressure_io_stalled_seconds_total{job="kubelet",node="k3s-prod-1",namespace="sonarr",pod="sonarr-6999bc759d-qlzqx",container="sonarr"}' values: '0+0.3x120' # 0.005 - series: 'container_pressure_io_stalled_seconds_total{job="kubelet",node="k3s-prod-1",namespace="longhorn-system",pod="instance-manager-bb75a71171b0251af787ab065f38964e",container="instance-manager"}' values: '0+0.24x120' # 0.004 # cAdvisor's POD-level cgroup series has no `container` label. Given a value that WOULD fire, # so the `container!=""` selector is proven to be what stops the same stall paging twice. - series: 'container_pressure_io_stalled_seconds_total{job="kubelet",node="k3s-prod-1",namespace="sonarr",pod="sonarr-6999bc759d-qlzqx"}' values: '0+18x120' # 0.30 — five times over the 0.15 threshold alert_rule_test: - eval_time: 60m alertname: NodeDiskDeviceSaturated exp_alerts: [] - eval_time: 60m alertname: NodeDiskWriteLatencyHigh exp_alerts: [] - eval_time: 60m alertname: NodeDiskWriteLatencyCritical exp_alerts: [] - eval_time: 60m alertname: NodeDiskReadLatencyHigh exp_alerts: [] - eval_time: 60m alertname: NodeNFSRetransmissionRateHigh exp_alerts: [] - eval_time: 60m alertname: NodeIOPressureNotLocalDisk exp_alerts: [] - eval_time: 60m alertname: NodeEtcdRequestsSlow exp_alerts: [] - eval_time: 60m alertname: ContainerIOStallHigh exp_alerts: [] # The two PSI rules this group exists to explain are also quiet on this fixture, which is # what makes it the healthy baseline and not just a quiet one. - eval_time: 60m alertname: NodeIOPressureHigh exp_alerts: [] - eval_time: 60m alertname: NodeIOPressureCritical exp_alerts: [] promql_expr_test: # --- NON-VACUITY 1: the guard admits exactly the two devices it is meant to. ------------- - expr: count(rate(node_disk_writes_completed_total[5m]) > 10) eval_time: 60m exp_samples: - labels: '{}' value: 2 # --- NON-VACUITY 2: PROMQL PRECEDENCE. The broken form returns those same two devices # (i.e. it fires on "busy", ignoring the 20 ms threshold entirely); the SHIPPED form # returns nothing. If a future edit reintroduces the trailing-comparison shape, the # second assertion below starts failing. - expr: | count( (rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m])) and rate(node_disk_writes_completed_total[5m]) > 10 > 0.020 ) eval_time: 60m exp_samples: - labels: '{}' value: 2 - expr: | count( (rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m])) > 0.020 and rate(node_disk_writes_completed_total[5m]) > 10 ) eval_time: 60m exp_samples: [] # --- NON-VACUITY 3: `container!=""` is what stops the pod-level duplicate paging. -------- - expr: count(rate(container_pressure_io_stalled_seconds_total[5m]) > 0.15) eval_time: 60m exp_samples: - labels: '{}' value: 1 - expr: count(rate(container_pressure_io_stalled_seconds_total{container!="",namespace!=""}[5m]) > 0.15) eval_time: 60m exp_samples: [] # ============================================================================================= # 2. TONIGHT — 2026-09-14 02:40-02:45Z on k3s-prod-1. # PSI(full) 20.98% with every local device idle and fast. The page must say "not local disk" # and name the stalled workload. This is the fixture the whole change exists for. # ============================================================================================= - interval: 1m name: 2026-09-14 02-40Z prod-1 - NotLocalDisk fires and names NFS, device rules stay quiet input_series: - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+12.588x120' # 0.2098 <- MEASURED peak - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.11:9100"}' values: '0+0.852x120' # 0.0142 <- prod-2 peaked under 1.5% - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.12:9100"}' values: '0+0.396x120' # 0.0066 # Local devices: MEASURED at 0.21 (sdi) and 0.09 (sda). Nowhere near saturated. - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdi"}' values: '0+12.6x120' # 0.21 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+5.4x120' # 0.09 # Writes: MEASURED 1.6 ms per op on sdi. 3.5 MB/s and 2.9 MB/s of traffic — trivial. - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdi"}' values: '0+1200x120' # 20 writes/s - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdi"}' values: '0+1.92x120' # 1.6 ms per write - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+3000x120' # 50 writes/s - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+4.8x120' # 1.6 ms per write # Reads from local devices were ~0; the media disk keeps its normal 156 ms. - series: 'node_disk_reads_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+1200x120' # 20 reads/s - series: 'node_disk_read_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+187.2x120' # 156 ms per read -> under the 250 ms threshold # NFS: retransmissions MEASURED at zero throughout, while the op rate was 114 reads/s. This # pair is the whole diagnosis — the transport is fine, we are simply waiting on the export. - series: 'node_nfs_rpcs_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+6840x120' # 114 RPC/s - series: 'node_nfs_rpc_retransmissions_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+0x120' # etcd was healthy throughout; the node never went NotReady (kube_node_status_condition # Ready held at 1 over the whole 2h window). - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.10:6443",operation="get",le="1.0"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.10:6443",operation="get",le="+Inf"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_count{job="apiserver",instance="192.168.2.10:6443",operation="get"}' values: '0+600x120' # The workload attribution, MEASURED: sonarr 0.268, qbittorrent 0.070, everything else under # 0.013. sonarr is an NFS media consumer, which is the answer. - series: 'container_pressure_io_stalled_seconds_total{job="kubelet",node="k3s-prod-1",namespace="sonarr",pod="sonarr-6999bc759d-qlzqx",container="sonarr"}' values: '0+16.08x120' # 0.268 - series: 'container_pressure_io_stalled_seconds_total{job="kubelet",node="k3s-prod-1",namespace="qbittorrent",pod="qbittorrent-56d6cfcf9c-k6rd8",container="qbittorrent"}' values: '0+4.2x120' # 0.070 -> below 0.15, must NOT page - series: 'container_pressure_io_stalled_seconds_total{job="kubelet",node="k3s-prod-1",namespace="monitoring",pod="prometheus-kube-prometheus-stack-prometheus-0",container="prometheus"}' values: '0+0.72x120' # 0.012 alert_rule_test: # THE DISCRIMINATOR FIRES, and only for prod-1. - eval_time: 60m alertname: NodeIOPressureNotLocalDisk exp_alerts: - exp_labels: severity: critical io_cause: not-local-disk instance: 192.168.2.10:9100 exp_annotations: summary: "CAUSE not local disk: 192.168.2.10:9100 is I/O-stalled while every local device is idle and healthy" description: "I/O PSI (full) above 10% for 5m on 192.168.2.10:9100 (.10 is k3s-prod-1, .11 is prod-2, .12 is prod-3) while NO local block device is above 50% busy, NO device exceeds 20ms per write and NO device exceeds 250ms per read. The stall is therefore NOT this node's disks. On this cluster that leaves the NFS export at 192.168.2.102: I/O PSI counts NFS stalls, which is why the block-layer metrics look innocent. This is exactly the 2026-09-14 02:40Z profile — PSI 20.98%, busiest device 0.27, 1.6ms writes, 114 NFS reads/s. Do NOT replace a disk and do NOT throttle Longhorn on this alert. The next question is whether OUR OWN NFS op rate is high (our workload is hammering the export) or near zero (something outside this cluster is — the Proxmox media stack still has operational control of the same files)." diagnostic_query: "sum by (instance, method) (rate(node_nfs_requests_total{method=~'Read|Write'}[5m])) — high on this node means our workload, near zero with PSI still high means an external consumer of 192.168.2.102. Then name the pod: topk(5, rate(container_pressure_io_stalled_seconds_total{container!=''}[5m]))" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # AND THE PAGE NAMES THE WORKLOAD — sonarr, not qbittorrent, not prometheus. - eval_time: 60m alertname: ContainerIOStallHigh exp_alerts: - exp_labels: severity: warning io_cause: workload job: kubelet node: k3s-prod-1 namespace: sonarr pod: sonarr-6999bc759d-qlzqx container: sonarr exp_annotations: summary: "WHO is stalled: sonarr/sonarr-6999bc759d-qlzqx (sonarr) on k3s-prod-1 is blocked on I/O" description: "Container sonarr in sonarr/sonarr-6999bc759d-qlzqx on k3s-prod-1 spent more than 15% of wall clock with every task in its cgroup blocked on I/O, for 5m. This is the WHO, not the why — pair it with the cause rules in this group. If the pod mounts the NFS media export (sonarr, radarr, plex, qbittorrent, nextcloud) and no local-device alert is firing, the stall is on 192.168.2.102. If it is a Longhorn instance-manager, look for a replica rebuild. Measured 2026-09-14 02:40Z: sonarr 0.268 on k3s-prod-1 while the node itself read 0.210." diagnostic_query: "topk(10, rate(container_pressure_io_stalled_seconds_total{container!=''}[5m])) — and the write side: topk(10, sum by (namespace, pod) (rate(container_fs_writes_bytes_total[5m])))" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # Every local-hardware and NFS-transport and etcd rule stays SILENT. This is the half the # operator had to establish with eight ad-hoc queries. - eval_time: 60m alertname: NodeDiskDeviceSaturated exp_alerts: [] - eval_time: 60m alertname: NodeDiskWriteLatencyHigh exp_alerts: [] - eval_time: 60m alertname: NodeDiskWriteLatencyCritical exp_alerts: [] - eval_time: 60m alertname: NodeDiskReadLatencyHigh exp_alerts: [] - eval_time: 60m alertname: NodeNFSRetransmissionRateHigh exp_alerts: [] - eval_time: 60m alertname: NodeEtcdRequestsSlow exp_alerts: [] promql_expr_test: # The detector that already existed does fire — the problem was never detection. - expr: count(rate(node_pressure_io_stalled_seconds_total[5m]) > 0.10) eval_time: 60m exp_samples: - labels: '{}' value: 1 # The measured peak, pinned as a band rather than an exact float (promtool compares sample # values bit-for-bit and a counter-increment fixture lands at 2.0979999999999943E-01), so # this fixture cannot quietly drift away from the incident it encodes. - expr: | count( rate(node_pressure_io_stalled_seconds_total[5m]) > 0.2097 and rate(node_pressure_io_stalled_seconds_total[5m]) < 0.2099 ) eval_time: 60m exp_samples: - labels: '{}' value: 1 # ============================================================================================= # 3. 2026-09-07 — THE FAILING DRIVE, re-queried from prod's own TSDB at offset 164h. # The opposite verdict from the same alert set: the device rules fire and name the disk, the # etcd rule fires, and NodeIOPressureNotLocalDisk correctly REFUSES to fire. # ============================================================================================= - interval: 1m name: 2026-09-07 failing drive - write-latency and etcd fire, NotLocalDisk correctly silent input_series: - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+44.382x120' # 0.7397 <- MEASURED at offset 164h - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.11:9100"}' values: '0+6.2x120' # 0.1033 - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.12:9100"}' values: '0+11.322x120' # 0.1887 # Device busy, every value MEASURED at the same window. prod-1's own devices peaked 0.745 # (sda) / 0.788 (sdb) — high, but NOT pinned, which is why NodeDiskDeviceSaturated stays # quiet for the dying drive: a failing device services FEW SLOW ops. Nodes 2 and 3 were # rebuilding Longhorn replicas at 0.858 and 0.768, and prod-2's sdb DOES cross 0.85 — so on # 2026-09-07 the correct page set is "saturation on prod-2, dying device on prod-1", which is # exactly the picture the operator lacked. Including nodes 2 and 3 here is also what makes # the NodeIOPressureNotLocalDisk assertion below honest: both of them were stalled above 10% # PSI, and it is their busy devices that rule them out, not the absence of data. - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+44.7x120' # 0.745 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdb"}' values: '0+47.28x120' # 0.788 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.11:9100",device="sdb"}' values: '0+51.48x120' # 0.858 -> the only device over 0.85 that morning - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.11:9100",device="nvme0n1"}' values: '0+33.96x120' # 0.566 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.12:9100",device="sdd"}' values: '0+46.08x120' # 0.768 # THE SIGNAL THAT WAS MISSING: 1,039 ms per write on prod-1's 2011 C400. - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+900x120' # 15 writes/s -> above the guard - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+935.01x120' # 15.5835 s/s / 15 ops/s = 1.0389 s per write # The healthy members, measured the same morning: 4.3 ms and 1.1 ms. - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.11:9100",device="nvme0n1"}' values: '0+1800x120' # 30 writes/s - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.11:9100",device="nvme0n1"}' values: '0+7.74x120' # 4.3 ms per write - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.12:9100",device="sda"}' values: '0+1800x120' # 30 writes/s - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.12:9100",device="sda"}' values: '0+1.98x120' # 1.1 ms per write # The etcd storage path, as the apiserver saw it: 0.54% / 2.05% / 1.65% of requests over 1s. - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.10:6443",operation="get",le="1.0"}' values: '0+596.76x120' # 9.946/s of 10/s -> fraction 0.0054 - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.10:6443",operation="get",le="+Inf"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_count{job="apiserver",instance="192.168.2.10:6443",operation="get"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.11:6443",operation="get",le="1.0"}' values: '0+587.7x120' # fraction 0.0205 - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.11:6443",operation="get",le="+Inf"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_count{job="apiserver",instance="192.168.2.11:6443",operation="get"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.12:6443",operation="get",le="1.0"}' values: '0+590.1x120' # fraction 0.0165 - series: 'etcd_request_duration_seconds_bucket{job="apiserver",instance="192.168.2.12:6443",operation="get",le="+Inf"}' values: '0+600x120' - series: 'etcd_request_duration_seconds_count{job="apiserver",instance="192.168.2.12:6443",operation="get"}' values: '0+600x120' # NFS was not the story that morning. - series: 'node_nfs_rpcs_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+600x120' # 10 RPC/s - series: 'node_nfs_rpc_retransmissions_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+0x120' alert_rule_test: # The device is named, twice, at two urgencies. - eval_time: 60m alertname: NodeDiskWriteLatencyCritical exp_alerts: - exp_labels: severity: critical io_cause: device-latency job: node-exporter instance: 192.168.2.10:9100 device: sda exp_annotations: summary: "CAUSE device latency CRITICAL: sda on 192.168.2.10:9100 is taking over 50ms per write — etcd is at risk" description: "Average write service time on sda of 192.168.2.10:9100 has exceeded 50ms for 5m under real write load. On these nodes the etcd WAL and every Longhorn replica share one partition, so a device this slow starves etcd's fdatasync and the node leaves the cluster while the host stays up, pingable and SSH-able — exactly the 2026-09-07 failure, where this same figure measured 1,039ms. Treat as HARDWARE until SMART says otherwise. With one etcd member already impaired, losing a second ends quorum: do not reboot or drain another node." diagnostic_query: "rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m]) and rate(node_disk_writes_completed_total[5m]) > 10 — then on the node: journalctl -u k3s --since '30 min ago' | grep -Ei 'fdatasync|apply request took'" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" - eval_time: 60m alertname: NodeDiskWriteLatencyHigh exp_alerts: - exp_labels: severity: warning io_cause: device-latency job: node-exporter instance: 192.168.2.10:9100 device: sda exp_annotations: summary: "CAUSE device latency: sda on 192.168.2.10:9100 is taking over 20ms per write" description: "Average write service time on sda of 192.168.2.10:9100 has exceeded 20ms for 10m while the device was doing more than 10 writes/s. Healthy on this cluster is 1-3ms (48h maximum 3.1ms). This is the LATENCY class: the device itself is slow, not merely busy. Check SMART for reallocated and pending sectors — prod-1's original 2011 C400 had 61,440 reallocated and measured 1,039ms per write on 2026-09-07 before the node went NotReady. If SMART is clean, suspect a write storm queueing on the shared partition ahead of etcd's fsync." diagnostic_query: "rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m]) and rate(node_disk_writes_completed_total[5m]) > 10 — then on the node: sudo smartctl -a /dev/sda | grep -Ei 'Reallocated|Pending|Uncorrectable|Wear'" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # THE ACCEPTANCE CHECK THAT MATTERS MOST: the "not local disk" page does NOT go out for a # failing disk. On prod-1 the write-latency clause of the `unless` chain suppresses it; on # prod-2 and prod-3, which were BOTH stalled above 10% PSI that morning, the busy-device # clause does. Not one of the three raises a misdirecting page. - eval_time: 60m alertname: NodeIOPressureNotLocalDisk exp_alerts: [] # Saturation is attributed where it belongs: prod-2's rebuilding sdb at 0.858, and NOT to # prod-1's dying sda at 0.745 — busy is not the same fault as slow. - eval_time: 60m alertname: NodeDiskDeviceSaturated exp_alerts: - exp_labels: severity: warning io_cause: device-saturation job: node-exporter instance: 192.168.2.11:9100 device: sdb exp_annotations: summary: "CAUSE device saturation: sdb on 192.168.2.11:9100 is busy >85% of the time" description: "Block device sdb on 192.168.2.11:9100 has been servicing I/O for more than 85% of wall clock for 10m — it is queue-bound, not slow. This is the SATURATION class of I/O pressure: too much work for the device, while latency per operation may be perfectly normal. Most likely a Longhorn replica rebuild, a Velero backup or a CDI import. The normal cluster maximum is 0.61 (prod-1, measured over 48h). Throttle the writer; do NOT replace a disk on this signal alone — see the per-op latency rules for that." diagnostic_query: "topk(5, rate(node_disk_io_time_seconds_total[5m])) — then compare with the latency rules: saturated-but-fast is a workload problem, slow-but-not-saturated is a hardware problem." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # THE URGENCY SIGNAL. All three members crossed 0.5% of requests over one second that # morning, so all three page — which is the correct reading: etcd quorum was at risk, not # just one node's disk. This block is pinned as an alert_rule_test and not only as a # promql_expr_test because a promql_expr_test restates the threshold in the TEST file, so it # cannot notice the SHIPPED threshold being changed; mutation M7 (0.005 -> 0.5) survived # until this block existed. - eval_time: 60m alertname: NodeEtcdRequestsSlow exp_alerts: - exp_labels: severity: critical io_cause: etcd instance: 192.168.2.10:6443 exp_annotations: summary: "CAUSE etcd: over 0.5% of etcd requests from 192.168.2.10:6443 are taking more than a second" description: "The apiserver on 192.168.2.10:6443 is seeing more than 0.5% of its requests to its LOCAL embedded etcd member take longer than one second, sustained 5m. The healthy measured value on all three members is exactly 0 over 48h; at the 2026-09-07 outage window it was 0.5-2%. This is the precursor to 'Kubelet stopped posting node status' with the host still up. Expect 'slow fdatasync' and 'apply request took too long' in journalctl -u k3s. If a device-latency alert is firing on the same node the cause is the disk; if not, look at etcd leader changes and CPU. Do not reboot or drain a second node — with one member impaired, losing another ends quorum." diagnostic_query: "1 - (sum by (instance) (rate(etcd_request_duration_seconds_bucket{job='apiserver',le='1.0'}[5m])) / sum by (instance) (rate(etcd_request_duration_seconds_bucket{job='apiserver',le='+Inf'}[5m]))) — then on the node: journalctl -u k3s --since '30 min ago' | grep -Ei 'fdatasync|apply request took|too many requests|leader'" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" - exp_labels: severity: critical io_cause: etcd instance: 192.168.2.11:6443 exp_annotations: summary: "CAUSE etcd: over 0.5% of etcd requests from 192.168.2.11:6443 are taking more than a second" description: "The apiserver on 192.168.2.11:6443 is seeing more than 0.5% of its requests to its LOCAL embedded etcd member take longer than one second, sustained 5m. The healthy measured value on all three members is exactly 0 over 48h; at the 2026-09-07 outage window it was 0.5-2%. This is the precursor to 'Kubelet stopped posting node status' with the host still up. Expect 'slow fdatasync' and 'apply request took too long' in journalctl -u k3s. If a device-latency alert is firing on the same node the cause is the disk; if not, look at etcd leader changes and CPU. Do not reboot or drain a second node — with one member impaired, losing another ends quorum." diagnostic_query: "1 - (sum by (instance) (rate(etcd_request_duration_seconds_bucket{job='apiserver',le='1.0'}[5m])) / sum by (instance) (rate(etcd_request_duration_seconds_bucket{job='apiserver',le='+Inf'}[5m]))) — then on the node: journalctl -u k3s --since '30 min ago' | grep -Ei 'fdatasync|apply request took|too many requests|leader'" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" - exp_labels: severity: critical io_cause: etcd instance: 192.168.2.12:6443 exp_annotations: summary: "CAUSE etcd: over 0.5% of etcd requests from 192.168.2.12:6443 are taking more than a second" description: "The apiserver on 192.168.2.12:6443 is seeing more than 0.5% of its requests to its LOCAL embedded etcd member take longer than one second, sustained 5m. The healthy measured value on all three members is exactly 0 over 48h; at the 2026-09-07 outage window it was 0.5-2%. This is the precursor to 'Kubelet stopped posting node status' with the host still up. Expect 'slow fdatasync' and 'apply request took too long' in journalctl -u k3s. If a device-latency alert is firing on the same node the cause is the disk; if not, look at etcd leader changes and CPU. Do not reboot or drain a second node — with one member impaired, losing another ends quorum." diagnostic_query: "1 - (sum by (instance) (rate(etcd_request_duration_seconds_bucket{job='apiserver',le='1.0'}[5m])) / sum by (instance) (rate(etcd_request_duration_seconds_bucket{job='apiserver',le='+Inf'}[5m]))) — then on the node: journalctl -u k3s --since '30 min ago' | grep -Ei 'fdatasync|apply request took|too many requests|leader'" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" - eval_time: 60m alertname: NodeNFSRetransmissionRateHigh exp_alerts: [] - eval_time: 60m alertname: NodeDiskReadLatencyHigh exp_alerts: [] promql_expr_test: # All three etcd members raise the urgency signal. Asserted as a count so the arithmetic of # the bucket ratio is pinned without depending on float-exact division. - expr: | count( ( 1 - ( sum by (instance) (rate(etcd_request_duration_seconds_bucket{job="apiserver",le="1.0"}[5m])) / sum by (instance) (rate(etcd_request_duration_seconds_bucket{job="apiserver",le="+Inf"}[5m])) ) ) > 0.005 and sum by (instance) (rate(etcd_request_duration_seconds_count{job="apiserver"}[5m])) > 0.1 ) eval_time: 60m exp_samples: - labels: '{}' value: 3 # And the measured per-write service time: 1.0389 s, pinned as a band (see the note in the # tonight fixture on promtool's bit-exact sample comparison). - expr: | count( ( rate(node_disk_write_time_seconds_total{device="sda",instance="192.168.2.10:9100"}[5m]) / rate(node_disk_writes_completed_total{device="sda",instance="192.168.2.10:9100"}[5m]) ) > 1.038 and ( rate(node_disk_write_time_seconds_total{device="sda",instance="192.168.2.10:9100"}[5m]) / rate(node_disk_writes_completed_total{device="sda",instance="192.168.2.10:9100"}[5m]) ) < 1.040 ) eval_time: 60m exp_samples: - labels: '{}' value: 1 # ============================================================================================= # 4. LONGHORN REBUILD STORM — a device genuinely pinned, with HEALTHY per-op latency. # Proves the saturation rule fires on its own cause, that the latency rules do not steal it, # and that NodeIOPressureNotLocalDisk stays quiet when the local disks ARE the explanation. # ============================================================================================= - interval: 1m name: device saturation with healthy latency - saturation fires, NotLocalDisk suppressed input_series: - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+15x120' # 0.25 — well over the critical PSI threshold - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+58.2x120' # 0.97 — pinned - series: 'node_disk_writes_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+18000x120' # 300 writes/s - series: 'node_disk_write_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+90x120' # 1.5 s/s / 300 ops/s = 5 ms per write — HEALTHY - series: 'node_nfs_rpcs_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+600x120' - series: 'node_nfs_rpc_retransmissions_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+0x120' alert_rule_test: - eval_time: 60m alertname: NodeDiskDeviceSaturated exp_alerts: - exp_labels: severity: warning io_cause: device-saturation job: node-exporter instance: 192.168.2.10:9100 device: sda exp_annotations: summary: "CAUSE device saturation: sda on 192.168.2.10:9100 is busy >85% of the time" description: "Block device sda on 192.168.2.10:9100 has been servicing I/O for more than 85% of wall clock for 10m — it is queue-bound, not slow. This is the SATURATION class of I/O pressure: too much work for the device, while latency per operation may be perfectly normal. Most likely a Longhorn replica rebuild, a Velero backup or a CDI import. The normal cluster maximum is 0.61 (prod-1, measured over 48h). Throttle the writer; do NOT replace a disk on this signal alone — see the per-op latency rules for that." diagnostic_query: "topk(5, rate(node_disk_io_time_seconds_total[5m])) — then compare with the latency rules: saturated-but-fast is a workload problem, slow-but-not-saturated is a hardware problem." runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" # Saturated but fast: the hardware rules must not claim this event. - eval_time: 60m alertname: NodeDiskWriteLatencyHigh exp_alerts: [] - eval_time: 60m alertname: NodeDiskWriteLatencyCritical exp_alerts: [] # And the discriminator stands down, because the local disk IS the explanation. - eval_time: 60m alertname: NodeIOPressureNotLocalDisk exp_alerts: [] # ============================================================================================= # 5. NFS TRANSPORT — a retransmitting export, plus a node whose op rate is below the guard. # ============================================================================================= - interval: 1m name: nfs retransmissions - transport rule fires on prod-1, traffic guard suppresses prod-2 input_series: - series: 'node_nfs_rpcs_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+7200x120' # 120 RPC/s - series: 'node_nfs_rpc_retransmissions_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+300x120' # 5/s -> 4.17% retransmitted # THE TRAFFIC GUARD'S REASON FOR EXISTING: 10% of six RPCs a minute is not a signal. - series: 'node_nfs_rpcs_total{job="node-exporter",instance="192.168.2.11:9100"}' values: '0+6x120' # 0.1 RPC/s -> BELOW the >1/s guard - series: 'node_nfs_rpc_retransmissions_total{job="node-exporter",instance="192.168.2.11:9100"}' values: '0+0.6x120' # 0.01/s -> ratio 0.10, and deliberately ignored - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+0.72x120' # 0.012 — the node is not even stalled yet alert_rule_test: - eval_time: 60m alertname: NodeNFSRetransmissionRateHigh exp_alerts: - exp_labels: severity: warning io_cause: nfs-transport instance: 192.168.2.10:9100 exp_annotations: summary: "CAUSE NFS transport: 192.168.2.10:9100 is retransmitting over 1% of its NFS RPCs" description: "More than 1% of this node's NFS client RPCs are being retransmitted, sustained for 10m at more than 1 RPC/s. The measured baseline on all three nodes is exactly ZERO over 48h, so any value here is real: the NFS server 192.168.2.102, or the path to it, is dropping or timing out requests. Because I/O PSI counts NFS stalls as I/O stalls, this is the cause class that makes a node look disk-starved while every local device is idle. Investigate the server and the link — not this node's disks." diagnostic_query: "sum by (instance) (rate(node_nfs_rpc_retransmissions_total[5m])) / sum by (instance) (rate(node_nfs_rpcs_total[5m])) — and the op rate driving it: sum by (instance, method) (rate(node_nfs_requests_total{method=~'Read|Write'}[5m]))" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" - eval_time: 60m alertname: NodeIOPressureNotLocalDisk exp_alerts: [] promql_expr_test: # The ratio is above threshold on BOTH nodes; only the traffic guard separates them. - expr: | count( sum by (instance) (rate(node_nfs_rpc_retransmissions_total[5m])) / sum by (instance) (rate(node_nfs_rpcs_total[5m])) > 0.01 ) eval_time: 60m exp_samples: - labels: '{}' value: 2 # ============================================================================================= # 6. READ LATENCY — a failing device, plus a device below the op-rate guard. # ============================================================================================= - interval: 1m name: read latency - fires at 400ms on a busy device, guard suppresses a 1s trickle input_series: - series: 'node_disk_reads_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+1200x120' # 20 reads/s -> above the guard - series: 'node_disk_read_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdg"}' values: '0+480x120' # 8 s/s / 20 ops/s = 400 ms per read # 1 SECOND per read at half a read per second: the artefact the guard exists to drop. - series: 'node_disk_reads_completed_total{job="node-exporter",instance="192.168.2.10:9100",device="sdh"}' values: '0+30x120' # 0.5 reads/s -> BELOW the guard - series: 'node_disk_read_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sdh"}' values: '0+30x120' # 1.0 s per read, and deliberately ignored alert_rule_test: - eval_time: 60m alertname: NodeDiskReadLatencyHigh exp_alerts: - exp_labels: severity: warning io_cause: device-latency job: node-exporter instance: 192.168.2.10:9100 device: sdg exp_annotations: summary: "CAUSE device latency: sdg on 192.168.2.10:9100 is taking over 250ms per read" description: "Average read service time on sdg of 192.168.2.10:9100 has exceeded 250ms for 15m at more than 10 reads/s. The spinning media disks on these nodes peak at 156ms and the SSDs at 6ms in normal operation, so this threshold means a failing or resetting device, not a slow one. Check SMART and dmesg for ATA resets and link errors." diagnostic_query: "rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m]) and rate(node_disk_reads_completed_total[5m]) > 10" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" promql_expr_test: # Both devices exceed 250 ms per read; only the guard tells them apart. - expr: | count( rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m]) > 0.250 ) eval_time: 60m exp_samples: - labels: '{}' value: 2 # ============================================================================================= # 7. THE `unless` SEMANTICS, ASSERTED DIRECTLY. # NodeIOPressureNotLocalDisk must fire for a stalled node whose devices are SO quiet that the # op-rate-guarded latency expressions produce NO SERIES AT ALL. An `and < threshold` # formulation returns nothing here and the alert vanishes exactly when it is most needed — # the structurally-inert shape this repo shipped in VeleroNoRecentBackup. `unless` with an # empty right-hand side removes nothing, so the alert survives. Only PSI and one idle device # are supplied: no write or read counters exist in this fixture. # ============================================================================================= - interval: 1m name: unless semantics - NotLocalDisk still fires when no device is busy enough to measure input_series: - series: 'node_pressure_io_stalled_seconds_total{job="node-exporter",instance="192.168.2.10:9100"}' values: '0+12.588x120' # 0.2098 - series: 'node_disk_io_time_seconds_total{job="node-exporter",instance="192.168.2.10:9100",device="sda"}' values: '0+1.2x120' # 0.02 — practically idle alert_rule_test: - eval_time: 60m alertname: NodeIOPressureNotLocalDisk exp_alerts: - exp_labels: severity: critical io_cause: not-local-disk instance: 192.168.2.10:9100 exp_annotations: summary: "CAUSE not local disk: 192.168.2.10:9100 is I/O-stalled while every local device is idle and healthy" description: "I/O PSI (full) above 10% for 5m on 192.168.2.10:9100 (.10 is k3s-prod-1, .11 is prod-2, .12 is prod-3) while NO local block device is above 50% busy, NO device exceeds 20ms per write and NO device exceeds 250ms per read. The stall is therefore NOT this node's disks. On this cluster that leaves the NFS export at 192.168.2.102: I/O PSI counts NFS stalls, which is why the block-layer metrics look innocent. This is exactly the 2026-09-14 02:40Z profile — PSI 20.98%, busiest device 0.27, 1.6ms writes, 114 NFS reads/s. Do NOT replace a disk and do NOT throttle Longhorn on this alert. The next question is whether OUR OWN NFS op rate is high (our workload is hammering the export) or near zero (something outside this cluster is — the Proxmox media stack still has operational control of the same files)." diagnostic_query: "sum by (instance, method) (rate(node_nfs_requests_total{method=~'Read|Write'}[5m])) — high on this node means our workload, near zero with PSI still high means an external consumer of 192.168.2.102. Then name the pod: topk(5, rate(container_pressure_io_stalled_seconds_total{container!=''}[5m]))" runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io" promql_expr_test: # The guarded latency expression really does produce nothing on this fixture — which is what # makes the assertion above a test of `unless` and not an accident. - expr: | count( (rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m])) and rate(node_disk_writes_completed_total[5m]) > 10 ) eval_time: 60m exp_samples: []