diff --git a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml index bca8337..bb4bbba 100644 --- a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml +++ b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml @@ -357,3 +357,29 @@ k3s_base_domain: "prod.home.keithjackson.dev" # from the prod nodes — it is outside every cluster and can simply be powered off; # - the `sops-age` Secret is installed out of band on prod (bootstrap/sops). flux_bootstrap_env_path: "clusters/prod" + +# --- shared etcd/Longhorn device I/O arbitration (2026-09-07 incident) -------------- +# TURNS ON roles/etcd_io_priority FOR PROD ONLY. Role default is false, and dev must stay +# that way: dev has a DEDICATED etcd device (/dev/vdc mounted at server/db, +# docs/plans/ssd-etcd-isolation.md) so it has nothing to arbitrate, and pulling a LIVE +# cluster onto a different I/O scheduler as a side effect of sharing a role would be a real +# change for no benefit. Same seam as node_baseline_backports_kernel above. +# +# WHY PROD NEEDS IT. The §2 ruling put /data/k3s (etcd WAL) and /data/longhorn (every +# replica) on ONE partition per node and recorded the contention as "accepted to be +# MEASURED, not assumed". It has now been measured, the hard way: 2026-09-07 06:05:21Z +# k3s-prod-1 went NotReady with the host still up, because etcd could not finish a WAL +# fdatasync (3.4-6.7 s against a 1 s expectation) while Longhorn wrote to the same disk. +# /proc/pressure/io FULL avg300 was 11.36 there against 0.75 (node 2, NVMe) and 1.29 +# (node 3). See PROD-PROVISIONING.md §2 "THE MEASUREMENT CAME BACK". +# +# WHAT ENABLING THIS DOES *NOT* DO: nothing, until the operator runs +# playbooks/etcd-io-priority.yml. It is not applied by any other play except through +# site.yml, and site.yml is not run against prod casually. Apply to k3s-prod-3 (the WD Blue, +# IO PSI 1.29) FIRST and watch NodeIOPressureHigh for a day — the role's own README explains +# why, what it cannot fix, and how to roll it back in three commands. +# +# k3s-prod-2 is NVMe-backed; the role detects that and deliberately installs nothing there +# rather than leaving an inert drop-in behind, so this group-wide `true` is still correct +# for it. +etcd_io_priority_enabled: true diff --git a/infra/ansible/playbooks/etcd-io-priority.yml b/infra/ansible/playbooks/etcd-io-priority.yml new file mode 100644 index 0000000..6180ced --- /dev/null +++ b/infra/ansible/playbooks/etcd-io-priority.yml @@ -0,0 +1,38 @@ +--- +# Give the k3s server process — and therefore the EMBEDDED ETCD inside it — a larger share of +# the block device it shares with Longhorn. Mitigates the 2026-09-07 k3s-prod-1 outage, where +# etcd's WAL fdatasync took 3.4-6.7 s (expects 1 s) and the node went NotReady while the host +# stayed up. See roles/etcd_io_priority/README.md for the mechanism, the measurements, and the +# list of things in this space that do NOT work on these nodes. +# +# NO-OP BY DEFAULT. The role's master switch (etcd_io_priority_enabled) is false in +# defaults/main.yml and true only in inventory/group_vars/k3s_prod/vars.yml, so running this +# with no extra vars against dev changes nothing on any node — dev already has a dedicated +# etcd device and has nothing to arbitrate. +# +# TARGET (R-ENV-02): same `k3s_hosts` seam every other whole-cluster play uses. +# ansible-playbook playbooks/etcd-io-priority.yml # dev, a no-op +# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod # prod +# +# RUN IT ON ONE NODE FIRST, AND CONFIRM THE TARGET BEFORE YOU RUN IT AT ALL. A prior mistake in +# this repo silently targeted dev's three nodes; --list-hosts is how that is caught: +# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \ +# --limit k3s-prod-3 --list-hosts +# k3s-prod-3 first, deliberately: it is the WD Blue SATA node (IO PSI 1.29 during the incident), +# so it exercises the identical code path as the failing node 1 (11.36) at a fraction of the +# stakes. Watch NodeIOPressureHigh / the "Nodes — CPU / Memory / I-O Pressure" dashboard for a +# day before touching node 1. +# +# ROLLBACK is immediate and needs no playbook — see the header of +# /usr/local/sbin/k3s-etcd-io-priority on the node. + +- name: I/O priority for the shared etcd + Longhorn device + hosts: "{{ k3s_hosts | default('k3s_servers') }}" + become: true + # serial: 1 — one node at a time. Nothing here restarts k3s, but changing a live node's I/O + # scheduler is a real change to a cluster member, and the whole point is that a bad outcome + # on one node must never be simultaneous across all three. + serial: 1 + gather_facts: true + roles: + - role: etcd_io_priority diff --git a/infra/ansible/playbooks/site.yml b/infra/ansible/playbooks/site.yml index e9f8247..a968a68 100644 --- a/infra/ansible/playbooks/site.yml +++ b/infra/ansible/playbooks/site.yml @@ -23,6 +23,17 @@ - name: Import - install k3s HA control plane ansible.builtin.import_playbook: k3s.yml +# ADDED 2026-09-07, after the k3s-prod-1 etcd I/O-starvation outage. It must come AFTER +# k3s.yml because it writes an IOWeight drop-in for k3s.service and verifies the value in +# that unit's LIVE cgroup — on a cold build the cgroup does not exist until k3s runs, and +# the role reports that rather than asserting on it. It restarts nothing. +# +# A NO-OP FOR DEV, by design: roles/etcd_io_priority defaults its master switch to false and +# only inventory/group_vars/k3s_prod/vars.yml sets it true. Dev has a dedicated etcd device +# and must not be dragged onto a different I/O scheduler by a shared role. +- name: Import - I/O arbitration for the shared etcd + Longhorn device (prod only) + ansible.builtin.import_playbook: etcd-io-priority.yml + # ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and # Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint = # kube_vip_address) rather than the founder's own node IP, so installing Cilium before the diff --git a/infra/ansible/roles/etcd_io_priority/README.md b/infra/ansible/roles/etcd_io_priority/README.md new file mode 100644 index 0000000..3ab1a16 --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/README.md @@ -0,0 +1,142 @@ +# roles/etcd_io_priority + +Give the k3s server process — and therefore the **embedded etcd** it hosts — a larger share of +the block device it shares with Longhorn. + +## The incident + +2026-09-07 06:05:21Z, `k3s-prod-1` went NotReady (`Kubelet stopped posting node status`) while +the host stayed up, pingable and SSH-able. From the k3s journal: + +``` +slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s) +apply request took too long took=8.30s (expects 100ms) +etcdserver: too many requests +Failed to check local etcd status for learner management: context deadline exceeded +``` + +No OOM, no MCE, no hung task. All three prod nodes put `/data/k3s` (etcd's WAL **and** the +containerd image store) and `/data/longhorn` (every replica) in two directories on **one +partition**: + +| node | device | drive | `/proc/pressure/io` full avg300 | +|---|---|---|---| +| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM — 2011 consumer SATA SSD, no PLP, 61,440 reallocated sectors | **11.36** | +| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 | +| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 | + +Recovery took a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt onto the same +disk etcd was trying to fsync on. The failure is self-reinforcing. + +## What this role does + +Two things, both instantly reversible, neither requiring a k3s restart: + +1. **`k3s-etcd-io-priority.service`** (oneshot, ordered `Before=k3s.service`) switches the + shared device to the **BFQ** I/O scheduler and lowers `nr_requests` from 64 to 32. +2. **`IOWeight=10000` drop-ins** on `system.slice` and `k3s.service`, which BFQ turns into + `io.bfq.weight=1000` — a 10:1 share against the default-weighted `kubepods.slice` where + Longhorn's instance-managers live. + +Then it **reads every one of those values back out of the kernel and asserts on them**. If the +scheduler did not switch, or the weight did not land, the play fails. A mitigation that looks +applied and is not is worse than none. + +## Why not the obvious things — all checked on the live nodes, all inert + +**`IOSchedulingClass=realtime` on `k3s.service`.** The scheduler here (`mq-deadline`, and BFQ +too) *does* honour the I/O priority class, so this looks like the answer. It is not: I/O +priority is inherited across `fork` and preserved across `exec` — `block/blk-ioc.c`'s +`__copy_io()` copies `ioc->ioprio` into the child — and **containerd is a child of the k3s +server process**. Verified on k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the +`k3s-server` PID (84553 → 84333), and + +``` +$ ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"' +best-effort: prio 0 +best-effort: prio 0 +``` + +confirms inheritance through two levels of fork+exec on that node. Putting k3s in the realtime +class therefore puts **every container on the node** in the realtime class, Longhorn included — +no relative gain, and it demotes the rest of `system.slice` (sshd, journald) for nothing. +Getting this to work would need containerd out of k3s's process tree, i.e. an external +containerd via `--container-runtime-endpoint`. That is an architecture change, not a tweak. + +**`IOSchedulingPriority` (the level within a class).** `mq-deadline` buckets requests by +priority **class** only — `IOPRIO_CLASS_RT/BE/IDLE` map to three `dd_per_prio` queues and the +level inside a class is not read. So best-effort 0 versus best-effort 4 is a no-op here. + +**`io.latency` (blk-iolatency).** The ideal mechanism for this problem — scheduler-independent, +protects one cgroup's latency target by throttling its siblings. **Not available:** Debian +trixie's kernel does not build it. `grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64` +returns `CONFIG_BLK_CGROUP_IOCOST=y` and no `IOLATENCY`, and there is no `io.latency` file +anywhere under `/sys/fs/cgroup`. + +**`IOWeight=` on the stock configuration.** systemd writes `IOWeight=` to both `io.weight` +(blk-iocost) and `io.bfq.weight` (BFQ). On these nodes as shipped, **neither reader exists**: +the scheduler is `mq-deadline`, which has no cgroup awareness, and blk-iocost is compiled in but +disabled (`/sys/fs/cgroup/io.cost.qos` is empty). Adding a drop-in without switching the +scheduler would be pure decoration. That is why the scheduler change and the weights are one +role and one verification, not two independent knobs. + +**`IOWeight=1000` instead of `10000`.** systemd rescales `IOWeight`'s 1..10000 range onto +`io.bfq.weight`'s 1..1000, so 1000 maps to roughly 101 — BFQ's default for an unweighted cgroup +is 100. Only the top of the range separates anything. The role asserts the resulting +`io.bfq.weight` is ≥ `etcd_io_priority_min_bfq_weight` (500) rather than trusting that mapping. + +**A udev rule instead of a unit.** `ATTR{queue/scheduler}="bfq"` on the *boot* disk fires its +`add` event in the initramfs, where `/etc/udev/rules.d` is absent and `bfq.ko` is not loaded — +it silently does nothing and the disk comes up on `mq-deadline` with no error anywhere. + +**A drop-in on `kubepods.slice`** (to lower Longhorn instead of raising etcd). `kubepods.slice` +is a **transient** unit the kubelet creates through the systemd API (`systemctl cat +kubepods.slice` shows `/run/systemd/transient/kubepods.slice`), and whether systemd merges an +`/etc` drop-in into a transient unit was not verified on these nodes. Raising `system.slice` +gets the same ratio through a unit systemd definitely owns. + +## What this role does NOT fix + +- **It does not separate etcd from Longhorn.** Both still write to one ext4 filesystem on one + partition. Weights arbitrate the *device queue*; they cannot arbitrate the **shared ext4 + journal**. `fdatasync` on the WAL waits on a jbd2 commit, jbd2 is a kernel thread in the root + cgroup, and Longhorn's replica writes dirty the same journal. That serialization point is + invisible to every cgroup control in this file and is a hard limit of co-tenancy on one + filesystem. +- **It is unproven on this hardware.** BFQ is the right mechanism on paper and it is what + systemd's `IOWeight` was built for, but nothing here has been measured on a C400 under a real + Longhorn rebuild. The role fails loudly if the settings do not *land*; it cannot tell you they + *helped*. Apply to `k3s-prod-3` first and watch the `NodeIOPressureHigh` alert and the + "Nodes — CPU / Memory / I-O Pressure" dashboard before touching node 1. +- **BFQ has costs.** It disables writeback throttling (`wbt_lat_usec` is 2000 on sda today), + spends more CPU per request, and gives up some peak throughput. On a device that is already + the bottleneck this is the right trade; it is still a trade. +- **k3s-prod-2 gets nothing.** Its data dir is on NVMe, BFQ is a poor fit for NVMe, and its IO + PSI was 0.75 during the incident. The role detects this and installs nothing rather than + leaving inert files behind. +- **The real fix is hardware and is the operator's**: replace the C400, or give etcd its own + device and flip `node_baseline_etcd_source` back to `device` — the path + `roles/node_baseline` still supports unchanged, and the exit condition already written into + `infra/PROD-PROVISIONING.md` §2. + +## Running it + +```bash +cd infra/ansible + +# ALWAYS confirm the target first — a prior mistake in this repo silently hit dev. +ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \ + --limit k3s-prod-3 --list-hosts + +ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 +``` + +Rollback, per node, immediate: + +```bash +systemctl disable --now k3s-etcd-io-priority.service +printf 'mq-deadline\n' > /sys/block/sda/queue/scheduler +printf '64\n' > /sys/block/sda/queue/nr_requests +rm -f /etc/systemd/system/{system.slice,k3s.service}.d/10-etcd-io-weight.conf +systemctl daemon-reload +``` diff --git a/infra/ansible/roles/etcd_io_priority/defaults/main.yml b/infra/ansible/roles/etcd_io_priority/defaults/main.yml new file mode 100644 index 0000000..0bf5bf7 --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/defaults/main.yml @@ -0,0 +1,84 @@ +--- +# etcd_io_priority — give the k3s process (and therefore the embedded etcd it hosts) a +# larger share of the block device it SHARES with Longhorn, on nodes where those two +# tenants sit on one partition. +# +# READ roles/etcd_io_priority/README.md BEFORE CHANGING ANY OF THESE. Two of the obvious +# knobs in this space are INERT on these nodes and are deliberately absent: +# * IOSchedulingClass= / IOSchedulingPriority= on k3s.service. I/O priority is inherited +# across fork and preserved across exec (block/blk-ioc.c __copy_io copies +# ioc->ioprio), and containerd is a CHILD of the k3s server process — verified live on +# k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the k3s-server PID, and +# `ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort +# prio 0 at BOTH levels. So raising k3s to the realtime I/O class raises every container +# on the node with it, Longhorn's instance-managers included, and buys nothing against +# the thing we are trying to beat. +# * io.latency (blk-iolatency), which WOULD be the ideal scheduler-independent protection. +# Not available: CONFIG_BLK_CGROUP_IOLATENCY is absent from Debian trixie's kernel +# (`grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64` returns IOCOST but not +# IOLATENCY), and there is no /sys/fs/cgroup/**/io.latency on any prod node. +# What is left, and what this role does, is cgroup-proportional weighting — which requires +# a scheduler that implements it. + +# MASTER SWITCH. Default FALSE so the LIVE dev cluster is untouched by a shared role: +# dev already has a DEDICATED etcd device (/dev/vdc mounted at server/db, +# docs/plans/ssd-etcd-isolation.md), so it has nothing to arbitrate and no reason to take a +# scheduler change. Set true for the [k3s_prod] group in +# inventory/group_vars/k3s_prod/vars.yml — the same seam node_baseline_backports_kernel, +# kube_vip_address and k3s_data_dir use. +etcd_io_priority_enabled: false + +# THE I/O SCHEDULER, AND WHY IT HAS TO CHANGE FOR ANY OF THIS TO WORK. +# Measured on k3s-prod-1: `cat /sys/block/sda/queue/scheduler` -> `none [mq-deadline]`. +# mq-deadline honours the I/O priority CLASS (RT/BE/IDLE) but knows nothing about cgroups, +# and cgroup v2's io.weight is implemented by blk-iocost — which is compiled in +# (CONFIG_BLK_CGROUP_IOCOST=y) but DISABLED: /sys/fs/cgroup/io.cost.qos is empty. So on the +# nodes as they stand today, systemd's IOWeight= writes a number nothing reads. +# BFQ is the one scheduler here that implements cgroup weights (CONFIG_IOSCHED_BFQ=m, +# CONFIG_BFQ_GROUP_IOSCHED=y — module present at +# /lib/modules/7.1.8+deb13-amd64/kernel/block/bfq.ko.xz). Under BFQ, systemd's IOWeight= +# lands in io.bfq.weight and becomes real. +# +# THE COSTS, STATED: BFQ turns OFF writeback throttling (wbt_lat_usec currently 2000 on +# sda), costs some peak throughput and some CPU, and is a poor fit for NVMe — which is why +# the role SKIPS the whole scheduler+weight path on an NVMe-backed node (k3s-prod-2), rather +# than pretending to help there. Set to "" to install nothing but still run the report. +etcd_io_priority_scheduler: bfq + +# Block-layer queue depth for the shared device. sda's current value is 64 (blk-mq derives +# it from the SATA NCQ depth). A shallower queue bounds how many Longhorn writes can already +# be sitting in front of an etcd WAL flush at the DEVICE, below any scheduler's reach — the +# one lever in this file that does not depend on cgroups or on the scheduler honouring +# anything. Costs sequential throughput. Set to "" to leave nr_requests alone. +etcd_io_priority_nr_requests: 32 + +# systemd IOWeight= for system.slice (which contains k3s.service, hence etcd) and for +# k3s.service itself. THE VALUE IS 10000 — systemd's maximum — AND THAT IS NOT COSMETIC. +# systemd rescales IOWeight (1..10000) onto io.bfq.weight (1..1000) linearly, so IOWeight=1000 +# maps to roughly 101 — indistinguishable from BFQ's own default of 100 for an unweighted +# cgroup. Only near the top of the range does the weight actually separate: 10000 -> 1000, +# a 10:1 share against the default-weighted kubepods.slice. A "reasonable-looking" 500 or +# 1000 here would be a change that reads as applied and does nothing. +etcd_io_priority_weight: 10000 + +# FAIL-LOUD FLOOR for the verification step. The role reads io.bfq.weight back out of the +# live cgroups and asserts it is at least this. It exists because the rescaling above is +# systemd's business, not ours: if a systemd version maps IOWeight differently, this role +# must FAIL rather than leave a drop-in that looks applied. 500 = 5x an unweighted sibling. +etcd_io_priority_min_bfq_weight: 500 + +# The k3s data dir whose backing device is the one to protect. Defers to the CLUSTER-WIDE +# k3s_data_dir (group_vars/k3s_prod sets /data/k3s; the roles/k3s_server default restates +# k3s's own /var/lib/rancher/k3s for dev) — NOT a second independent variable, for the same +# reason node_baseline_k3s_data_dir defers to it. +etcd_io_priority_k3s_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}" + +# The systemd units that get an IOWeight drop-in, and the section each one takes. +# BOTH ENTRIES ARE LOAD-BEARING — see tasks/main.yml "THE CGROUP WEIGHTS" for why weighting +# k3s.service alone would arbitrate k3s against sshd and not against Longhorn at all. +# system.slice must come first so the parent is weighted before its child. +etcd_io_priority_weighted_units: + - unit: system.slice + section: Slice + - unit: k3s.service + section: Service diff --git a/infra/ansible/roles/etcd_io_priority/handlers/main.yml b/infra/ansible/roles/etcd_io_priority/handlers/main.yml new file mode 100644 index 0000000..b68ce64 --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/handlers/main.yml @@ -0,0 +1,18 @@ +--- +# Re-run the oneshot so a changed script/unit takes effect NOW rather than at the next boot. +# It only writes two sysfs files; there is nothing to drain and nothing to restart. +- name: Apply k3s etcd I/O settings + ansible.builtin.systemd_service: + name: k3s-etcd-io-priority.service + state: restarted + daemon_reload: true + +# daemon-reload re-realizes cgroup attributes for units that are already running, which is how +# a changed IOWeight= reaches a LIVE k3s.service without restarting it. It is deliberately NOT +# a `systemctl restart k3s`: restarting the k3s server to apply a performance tweak is exactly +# the disruption this role exists to prevent, and on the primary it bounces etcd. If the reload +# turns out not to be enough, the role's verification step FAILS and tells the operator to do a +# rolling restart deliberately — it never quietly leaves the weight unapplied. +- name: Reload systemd for I/O weights + ansible.builtin.systemd_service: + daemon_reload: true diff --git a/infra/ansible/roles/etcd_io_priority/tasks/main.yml b/infra/ansible/roles/etcd_io_priority/tasks/main.yml new file mode 100644 index 0000000..c684a6f --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/tasks/main.yml @@ -0,0 +1,234 @@ +--- +# etcd_io_priority — see README.md in this directory for the incident, the mechanism, and the +# honest list of what this does NOT achieve. Everything runs under one `when` on the master +# switch, so a run against dev (default false) touches nothing at all. +# +# TAG: `io-priority`. Every task carries it, so this role can be run alone against a live +# cluster without dragging in any other role's work: +# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 +# +# ALWAYS --limit AND ALWAYS CHECK --list-hosts FIRST. This role changes the I/O scheduler of a +# live node's boot disk; it is safe and instantly reversible, but it is a real change and the +# blast radius must be one node at a time (playbooks/etcd-io-priority.yml sets serial: 1 too). + +- name: I/O priority for the shared etcd device + when: etcd_io_priority_enabled | bool + tags: [io-priority] + block: + + # ------------------------------------------------------------------------------------- + # DISCOVER THE DEVICE. Never hardcoded, and never taken from inventory: k3s-prod-1 and -3 + # boot from SATA (/dev/sda5) while k3s-prod-2 boots from NVMe (/dev/nvme0n1p5), and a + # per-host var freezes today's hardware into inventory where it goes stale silently the + # first time a disk is swapped. Inventory declares the INTENT ("the disk under k3s's data + # dir"); the node answers what that is. Same reasoning as roles/node_baseline's disk + # discovery. + # ------------------------------------------------------------------------------------- + - name: Find the filesystem source backing the k3s data dir + ansible.builtin.command: + cmd: "findmnt --noheadings --output SOURCE --target {{ etcd_io_priority_k3s_data_dir }}" + register: _eio_findmnt + changed_when: false + check_mode: false + + - name: Find the whole-disk parent of that filesystem source + ansible.builtin.command: + cmd: "lsblk --noheadings --output PKNAME {{ _eio_findmnt.stdout | trim }}" + register: _eio_pkname + changed_when: false + check_mode: false + + # PKNAME is empty when the source IS already a whole disk (no partition), so fall back to + # its basename rather than producing an empty device name. + - name: Resolve the block device to tune + ansible.builtin.set_fact: + _eio_device: >- + {{ (_eio_pkname.stdout_lines | map('trim') | reject('equalto', '') | list + + [_eio_findmnt.stdout | trim | basename]) | first }} + + # ------------------------------------------------------------------------------------- + # NVMe IS DELIBERATELY LEFT ALONE, and this is the whole reason the role discovers the + # device instead of assuming it. BFQ is a bad fit for NVMe (single software queue, real + # throughput cost) and k3s-prod-2 — the NVMe node — measured /proc/pressure/io full avg300 + # of 0.75 during the 2026-09-07 incident against k3s-prod-1's 11.36. It does not have this + # problem. Installing an inert weight drop-in there would be exactly the false confidence + # this role exists to avoid, so nothing is written on such a node. + # ------------------------------------------------------------------------------------- + - name: Decide whether a scheduler change applies to this device + ansible.builtin.set_fact: + _eio_scheduler: "{{ '' if _eio_device is match('nvme') else etcd_io_priority_scheduler }}" + + - name: Report that this node is intentionally skipped (NVMe-backed) + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: k3s data dir {{ etcd_io_priority_k3s_data_dir }} is on + /dev/{{ _eio_device }} (NVMe). BFQ and its cgroup weights are not applied to NVMe — + nothing installed here. This is intentional, not a failure. + when: not _eio_scheduler + + # ------------------------------------------------------------------------------------- + # THE BLOCK-LAYER SETTINGS (scheduler + queue depth). + # ------------------------------------------------------------------------------------- + - name: Install the block-layer settings script + ansible.builtin.template: + src: k3s-etcd-io-priority.sh.j2 + dest: /usr/local/sbin/k3s-etcd-io-priority + owner: root + group: root + mode: "0755" + when: _eio_scheduler | length > 0 + notify: Apply k3s etcd I/O settings + + - name: Install the boot-time unit that applies them + ansible.builtin.template: + src: k3s-etcd-io-priority.service.j2 + dest: /etc/systemd/system/k3s-etcd-io-priority.service + owner: root + group: root + mode: "0644" + when: _eio_scheduler | length > 0 + notify: Apply k3s etcd I/O settings + + - name: Enable and start the block-layer settings unit + ansible.builtin.systemd_service: + name: k3s-etcd-io-priority.service + enabled: true + state: started + daemon_reload: true + when: _eio_scheduler | length > 0 + + # ------------------------------------------------------------------------------------- + # THE CGROUP WEIGHTS. + # + # BOTH LEVELS ARE REQUIRED AND NEITHER IS SUFFICIENT. cgroup v2 weights are hierarchical: + # k3s.service's weight only decides its share WITHIN system.slice, and system.slice's + # weight is what competes with kubepods.slice (where Longhorn's instance-managers live). + # Weighting only k3s.service — the intuitive thing to do — would arbitrate k3s against + # sshd and journald and would not touch Longhorn at all. + # + # kubepods.slice is deliberately NOT given a drop-in of its own. It is a TRANSIENT unit + # created by the kubelet through the systemd API (`systemctl cat kubepods.slice` on + # k3s-prod-1 shows /run/systemd/transient/kubepods.slice), and whether systemd merges an + # /etc drop-in into a transient unit is not something this change verified on these nodes. + # Raising system.slice achieves the same ratio through a unit systemd definitely owns. + # ------------------------------------------------------------------------------------- + - name: Ensure the systemd drop-in directories exist + ansible.builtin.file: + path: "/etc/systemd/system/{{ item.unit }}.d" + state: directory + owner: root + group: root + mode: "0755" + loop: "{{ etcd_io_priority_weighted_units }}" + loop_control: + label: "{{ item.unit }}" + when: _eio_scheduler | length > 0 + + - name: Write the IOWeight drop-ins + ansible.builtin.template: + src: io-weight.conf.j2 + dest: "/etc/systemd/system/{{ item.unit }}.d/10-etcd-io-weight.conf" + owner: root + group: root + mode: "0644" + vars: + _eio_dropin_unit: "{{ item.unit }}" + _eio_dropin_section: "{{ item.section }}" + loop: "{{ etcd_io_priority_weighted_units }}" + loop_control: + label: "{{ item.unit }}" + when: _eio_scheduler | length > 0 + notify: Reload systemd for I/O weights + + # Handlers must run BEFORE the verification below, or the verification reads the state from + # before this run's changes and reports a stale pass. + - name: Apply pending I/O changes before verifying them + ansible.builtin.meta: flush_handlers + + # ------------------------------------------------------------------------------------- + # VERIFY — the point of the role, not a formality. + # + # This repo has repeatedly shipped configuration that looked applied and was not (the + # Keycloak reconciler that could not read its own Secret; Longhorn's orphanAutoDeletion, + # silently dropped by the chart for two minor versions). Every mechanism here has a + # plausible silent-failure mode: the scheduler module may not load; a systemd version may + # rescale IOWeight differently; daemon-reload may not re-realize a running unit's cgroup. + # So each is READ BACK OUT OF THE KERNEL and asserted. A failure here means the mitigation + # is not in place — which is strictly better than believing that it is. + # ------------------------------------------------------------------------------------- + - name: Read back the live block-layer state + ansible.builtin.command: + cmd: "cat /sys/block/{{ _eio_device }}/queue/scheduler /sys/block/{{ _eio_device }}/queue/nr_requests" + register: _eio_queue_state + changed_when: false + check_mode: false + when: _eio_scheduler | length > 0 + + - name: VERIFY the scheduler and queue depth actually took effect + ansible.builtin.assert: + that: + - "'[' ~ _eio_scheduler ~ ']' in _eio_queue_state.stdout_lines[0]" + - "etcd_io_priority_nr_requests | string | length == 0 + or _eio_queue_state.stdout_lines[1] | trim | int == etcd_io_priority_nr_requests | int" + fail_msg: >- + /dev/{{ _eio_device }} did not take the requested settings. Wanted scheduler + '{{ _eio_scheduler }}' and nr_requests '{{ etcd_io_priority_nr_requests }}'; the kernel + reports {{ _eio_queue_state.stdout_lines | join(' / ') }}. The cgroup weights below are + MEANINGLESS without the scheduler, so this is a hard failure, not a warning. Check + `journalctl -u k3s-etcd-io-priority` and that bfq.ko exists for this kernel. + success_msg: >- + /dev/{{ _eio_device }}: scheduler {{ _eio_scheduler }}, nr_requests + {{ _eio_queue_state.stdout_lines[1] | trim }}. + when: _eio_scheduler | length > 0 + + # k3s.service's cgroup does not exist until k3s is running, so a cold-build run legitimately + # has nothing to read there. Reported, not asserted away. + - name: Check whether k3s.service has a live cgroup to verify + ansible.builtin.stat: + path: /sys/fs/cgroup/system.slice/k3s.service + register: _eio_k3s_cgroup + when: _eio_scheduler | length > 0 + + - name: Report that k3s is not running, so its own cgroup weight cannot be verified yet + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: /sys/fs/cgroup/system.slice/k3s.service does not exist — k3s + is not running on this node, so its drop-in is written but UNVERIFIED. Re-run + playbooks/etcd-io-priority.yml once k3s is up. + when: + - _eio_scheduler | length > 0 + - not _eio_k3s_cgroup.stat.exists + + - name: Read back the live cgroup I/O weights + ansible.builtin.command: + cmd: "cat /sys/fs/cgroup/{{ item }}/io.bfq.weight" + register: _eio_bfq_weights + changed_when: false + check_mode: false + failed_when: false + loop: >- + {{ ['system.slice'] + + (['system.slice/k3s.service'] if _eio_k3s_cgroup.stat.exists else []) }} + when: _eio_scheduler | length > 0 + + - name: VERIFY every live cgroup weight actually carries the boost + ansible.builtin.assert: + that: + - item.rc == 0 + - item.stdout | trim | int >= etcd_io_priority_min_bfq_weight | int + fail_msg: >- + /sys/fs/cgroup/{{ item.item }}/io.bfq.weight reads + '{{ item.stdout | default('') | trim }}' (rc={{ item.rc }}), below the + required floor of {{ etcd_io_priority_min_bfq_weight }}. IOWeight={{ etcd_io_priority_weight }} + did NOT reach the kernel. Most likely causes, in order: (1) systemd rescaled the value + differently from what defaults/main.yml assumes — compare + `systemctl show {{ item.item | basename }} -p IOWeight` against the file; (2) the cgroup + was not re-realized by daemon-reload, in which case a ROLLING `systemctl restart k3s` + (one node at a time — etcd quorum is 2/3) applies it; (3) BFQ is not active on the + device, which the previous assert should already have caught. The drop-in on disk is + NOT the mitigation — this file is. + success_msg: "{{ item.item }} io.bfq.weight = {{ item.stdout | default('') | trim }}" + loop: "{{ _eio_bfq_weights.results | default([]) }}" + loop_control: + label: "{{ item.item | default('') }}" + when: _eio_scheduler | length > 0 diff --git a/infra/ansible/roles/etcd_io_priority/templates/io-weight.conf.j2 b/infra/ansible/roles/etcd_io_priority/templates/io-weight.conf.j2 new file mode 100644 index 0000000..1f14f3e --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/templates/io-weight.conf.j2 @@ -0,0 +1,21 @@ +# {{ ansible_managed }} +# +# Proportional share of {{ _eio_device }} for {{ _eio_dropin_unit }}, which is where the k3s server +# process — and therefore the EMBEDDED ETCD it hosts — does its WAL writes. Its competitor is +# kubepods.slice (Longhorn's instance-managers), which stays at the default weight. +# +# THIS ONLY WORKS BECAUSE THE SCHEDULER IS {{ _eio_scheduler | upper }}. systemd writes IOWeight= to both +# io.weight (blk-iocost, which is compiled in but DISABLED here — /sys/fs/cgroup/io.cost.qos is +# empty) and io.bfq.weight (BFQ). Under the stock mq-deadline this file is a no-op; the sibling +# k3s-etcd-io-priority.service is what makes it mean something. roles/etcd_io_priority verifies +# io.bfq.weight after writing this and FAILS the play if the value did not land. +# +# NOT IOSchedulingClass=. I/O priority is inherited across fork and kept across exec, and +# containerd is a child of the k3s server process — so realtime here would be realtime for every +# container on the node, Longhorn included. See roles/etcd_io_priority/README.md. +# +# {{ etcd_io_priority_weight }} is systemd's MAXIMUM and that is deliberate: systemd rescales +# 1..10000 onto io.bfq.weight's 1..1000, so IOWeight=1000 would map to ~101 — BFQ's unweighted +# default is 100. Only the top of the range actually separates this cgroup from its siblings. +[{{ _eio_dropin_section }}] +IOWeight={{ etcd_io_priority_weight }} diff --git a/infra/ansible/roles/etcd_io_priority/templates/k3s-etcd-io-priority.service.j2 b/infra/ansible/roles/etcd_io_priority/templates/k3s-etcd-io-priority.service.j2 new file mode 100644 index 0000000..50075f5 --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/templates/k3s-etcd-io-priority.service.j2 @@ -0,0 +1,24 @@ +# {{ ansible_managed }} +# +# WHY A ONESHOT UNIT AND NOT A UDEV RULE. The obvious form of this is +# `ACTION=="add|change", KERNEL=="{{ _eio_device }}", ATTR{queue/scheduler}="{{ _eio_scheduler }}"`, +# and it is unreliable for the ONE device that matters: {{ _eio_device }} is the boot disk, so its +# `add` event fires in the initramfs, where /etc/udev/rules.d is not present and bfq.ko is not +# loaded — the ATTR assignment silently does nothing and the disk comes up on mq-deadline with +# no error anywhere. A unit ordered Before=k3s.service runs after modules are loadable, fails +# LOUDLY if the scheduler is unavailable, and is visible in `systemctl status`. +[Unit] +Description=Block-layer I/O settings for the shared etcd + Longhorn device ({{ _eio_device }}) +Documentation=file:///etc/systemd/system/k3s-etcd-io-priority.service +DefaultDependencies=no +After=systemd-modules-load.service +Before=k3s.service shutdown.target +Conflicts=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/k3s-etcd-io-priority + +[Install] +WantedBy=multi-user.target diff --git a/infra/ansible/roles/etcd_io_priority/templates/k3s-etcd-io-priority.sh.j2 b/infra/ansible/roles/etcd_io_priority/templates/k3s-etcd-io-priority.sh.j2 new file mode 100644 index 0000000..38cc942 --- /dev/null +++ b/infra/ansible/roles/etcd_io_priority/templates/k3s-etcd-io-priority.sh.j2 @@ -0,0 +1,38 @@ +#!/bin/sh +# {{ ansible_managed }} +# +# Block-layer settings for the device that carries BOTH etcd's WAL ({{ etcd_io_priority_k3s_data_dir }}) +# and every Longhorn replica on this node. Run at boot by k3s-etcd-io-priority.service, and +# safe to run by hand at any time — it is idempotent and takes effect immediately. +# +# Rollback (immediate, no reboot): +# printf 'mq-deadline\n' > /sys/block/{{ _eio_device }}/queue/scheduler +# printf '64\n' > /sys/block/{{ _eio_device }}/queue/nr_requests +# and `systemctl disable --now k3s-etcd-io-priority.service` so it does not come back. +set -eu + +DEV="{{ _eio_device }}" +QUEUE="/sys/block/${DEV}/queue" + +if [ ! -d "${QUEUE}" ]; then + echo "k3s-etcd-io-priority: /sys/block/${DEV} does not exist — device renamed?" >&2 + exit 1 +fi + +{% if _eio_scheduler %} +# BFQ is a module here (CONFIG_IOSCHED_BFQ=m); loading it is what makes it selectable at all. +modprobe -q {{ _eio_scheduler }} 2>/dev/null || true + +if ! grep -qw '{{ _eio_scheduler }}' "${QUEUE}/scheduler"; then + echo "k3s-etcd-io-priority: scheduler '{{ _eio_scheduler }}' not available for ${DEV}: $(cat "${QUEUE}/scheduler")" >&2 + exit 1 +fi +printf '%s\n' '{{ _eio_scheduler }}' > "${QUEUE}/scheduler" +{% endif %} + +{% if etcd_io_priority_nr_requests %} +# ORDER MATTERS: switching the elevator resets nr_requests, so this must come second. +printf '%s\n' '{{ etcd_io_priority_nr_requests }}' > "${QUEUE}/nr_requests" +{% endif %} + +exit 0