From 9f59c36000719901733e819667b855994b75c278 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 31 Aug 2026 23:20:25 -0500 Subject: [PATCH] =?UTF-8?q?feat(ansible):=20node=5Fhealth=20role=20?= =?UTF-8?q?=E2=80=94=20read-only=20hardware=20health=20for=20prod=20+=20de?= =?UTF-8?q?v=20nodes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator ask: implement the roles needed to check prod node hardware health, and run them to find out our state. This is that role; the run's findings are below. roles/node_health is a read-only sibling to cluster_verify (cluster health vs hardware health), styled after node_baseline. One unprivileged collector script does the work in a single SSH round trip — reading /sys/class/dmi, thermal + hwmon, EDAC, cpufreq + thermal_throttle, /proc/pressure, /sys/block incl. SCSI ioerr/iotmo counters, lsblk, lscpu, mounts + statvfs, per-NIC stats, resolv.conf. THE DESIGN POINT: an unreadable field is emitted as null WITH A RECORDED REASON, never as zero. The summary ends with a "NOT READ (and why) — these are UNKNOWNS, not passes" section. A fake zero on a reallocated-sector count is exactly the kind of false green this project refuses. 22 checks report PASS/WARN/FAIL/SKIP/INFO; every SKIP says why. Report-only by default; thresholds live in defaults/main.yml. Portable across bare metal AND VMs — validated against k3s-prod-1 (bare metal, no root) and k3s-dev-1 (VM, root). VM-absent things (SMART on virtio, coretemp, EDAC) skip cleanly, and iSCSI/Longhorn volumes are split out of the physical-disk set so a live replica is never mistaken for a spare device. playbooks/node-health.yml takes its target group as a variable (-e node_health_hosts= k3s_prod), deliberately unlike the other playbooks that hardcode k3s_servers — see the group-name gap already flagged in hosts.ini. Two become facts found the hard way and documented in the role: a group_vars `ansible_become: true` OUTRANKS `become: false` as a play/task keyword, so unprivileged tasks must override the variable; and `failed_when: false` does NOT catch "Missing sudo password" — only `ignore_errors` does. FIRST RUN, k3s-prod-1, 2026-08-31: 0 FAIL · 3 WARN · 12 PASS · 3 SKIP · 4 INFO. Box is healthy, cool and unstressed: i7-9700T 8C/8T, VT-x + /dev/kvm (KubeVirt OK), zero throttle events, hottest sensor 33C, PSI 0.00 across the board, 31.1 GiB RAM, eno1 up at 1000 Mb/s, BIOS R21 02.26.00, no reboot pending. DRIVE HEALTH IS STILL UNKNOWN — the headline. SMART did not run, blocked TWICE over: no vaulted ansible_become_password for prod, AND smartmontools is not on the node at all (node_baseline has never run there). Power-on hours, wear, reallocated/pending sectors and drive temperature all remain unmeasured on a ~2011 consumer SSD. WARNs: kernel ioerr_cnt=6 on sda (read cautiously — four freshly attached iSCSI disks on dev each showed a flat 7, suggesting a fixed per-device discovery cost, not media trouble; attribution needs dmesg/root); 0.95 GiB swap still active (node_baseline does the swapoff); one IPv4 resolver only (the other two upstreams are IPv6 Google), which refines rather than resolves the DNS-SPOF finding. ESCALATION — ECC status is UNKNOWABLE on this hardware: /sys/devices/system/edac/mc exists but no controller is bound. Non-ECC consumer RAM. Silent memory corruption on a prod node would be undetectable. Recorded as a SKIP with reason, not a pass. Serves: R-ENV-01, R-ENV-03, R-OBS-04. Task: T-7.1. Co-Authored-By: Claude Opus 5 (1M context) --- infra/ansible/.gitignore | 5 + infra/ansible/playbooks/node-health.yml | 30 + infra/ansible/roles/node_health/README.md | 91 +++ .../roles/node_health/defaults/main.yml | 78 +++ .../node_health/files/node_health_collect.py | 630 ++++++++++++++++++ .../roles/node_health/tasks/assess.yml | 509 ++++++++++++++ .../roles/node_health/tasks/collect_smart.yml | 83 +++ .../tasks/collect_unprivileged.yml | 30 + .../ansible/roles/node_health/tasks/main.yml | 60 ++ .../node_health/tasks/probe_privilege.yml | 35 + .../roles/node_health/tasks/report.yml | 84 +++ .../node_health/templates/summary.txt.j2 | 147 ++++ 12 files changed, 1782 insertions(+) create mode 100644 infra/ansible/playbooks/node-health.yml create mode 100644 infra/ansible/roles/node_health/README.md create mode 100644 infra/ansible/roles/node_health/defaults/main.yml create mode 100644 infra/ansible/roles/node_health/files/node_health_collect.py create mode 100644 infra/ansible/roles/node_health/tasks/assess.yml create mode 100644 infra/ansible/roles/node_health/tasks/collect_smart.yml create mode 100644 infra/ansible/roles/node_health/tasks/collect_unprivileged.yml create mode 100644 infra/ansible/roles/node_health/tasks/main.yml create mode 100644 infra/ansible/roles/node_health/tasks/probe_privilege.yml create mode 100644 infra/ansible/roles/node_health/tasks/report.yml create mode 100644 infra/ansible/roles/node_health/templates/summary.txt.j2 diff --git a/infra/ansible/.gitignore b/infra/ansible/.gitignore index 8d7676f..249fd8f 100644 --- a/infra/ansible/.gitignore +++ b/infra/ansible/.gitignore @@ -1,3 +1,8 @@ # Runtime cache written by the cilium/verify/flux plays (fetched kubeconfig, # rendered Cilium values). Never commit — the kubeconfig is a credential. .cache/ + +# Generated node-health reports (roles/node_health, playbooks/node-health.yml). +# Regenerated on every run and they carry drive serial numbers — never commit. Same +# convention as conformance/_artifacts/: generated output is an artifact, not source. +_artifacts/ diff --git a/infra/ansible/playbooks/node-health.yml b/infra/ansible/playbooks/node-health.yml new file mode 100644 index 0000000..ca84853 --- /dev/null +++ b/infra/ansible/playbooks/node-health.yml @@ -0,0 +1,30 @@ +--- +# Node hardware health — READ-ONLY inspection of a node's disks, memory, CPU, thermals, +# network and pressure-stall state. Installs nothing, changes nothing, safe to run at any +# time against dev or prod (see roles/node_health for the full contract). +# +# ./dev.sh run bash -c 'cd /work/infra/ansible && \ +# ansible-playbook playbooks/node-health.yml -e node_health_hosts=k3s_prod' +# +# TARGET GROUP IS A VARIABLE, unlike the other playbooks here. Every existing node +# playbook (node-baseline.yml, k3s.yml, cilium.yml, kube-vip.yml, verify.yml) hardcodes +# `hosts: k3s_servers` / `k3s_primary` / `k3s_secondary`, which is exactly why they cannot +# select the prod groups today (the gap flagged in inventory/hosts.ini's prod section, and +# the R-ENV-02 reuse promise it threatens). A health check is the one play you want to be +# able to point anywhere, so it takes the pattern as a var and defaults to dev. Run it +# against prod with `-e node_health_hosts=k3s_prod`, or a single box with `--limit`. +# +# BECOME: left at the play level exactly as the inventory sets it. The role itself pins +# `ansible_become: false` on every unprivileged task and probes escalation once before +# attempting the only root-only check (SMART), so this play behaves correctly on dev +# (NOPASSWD sudo) and on prod (vaulted become password, or none at all) without any +# per-environment flags. See roles/node_health/tasks/main.yml for why that has to be a +# variable override rather than `become: false`. +- name: Inspect node hardware health (read-only) + hosts: "{{ node_health_hosts | default('k3s_servers') }}" + # The role's own collector supersedes ansible.builtin.setup for this purpose (setup + # exposes no thermal zones, no EDAC, no PSI, no per-disk error counters), so skip the + # fact gathering round trip entirely. + gather_facts: false + roles: + - node_health diff --git a/infra/ansible/roles/node_health/README.md b/infra/ansible/roles/node_health/README.md new file mode 100644 index 0000000..e2eddd8 --- /dev/null +++ b/infra/ansible/roles/node_health/README.md @@ -0,0 +1,91 @@ +# `node_health` — read-only node hardware health + +Sibling of `roles/cluster_verify`. `cluster_verify` answers *"is the cluster healthy"*; +`node_health` answers *"is the hardware under it healthy"*. Same posture: it **inspects and +reports, it never changes the node**. + +Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-03** +(bare-metal parity with dev) and **R-OBS-04** (node-level observability). + +```bash +# prod (all nodes in the k3s_prod group) +./infra/devcontainer/dev.sh run bash -c \ + 'cd /work/infra/ansible && ansible-playbook playbooks/node-health.yml -e node_health_hosts=k3s_prod' + +# dev (default target), one node +./infra/devcontainer/dev.sh run bash -c \ + 'cd /work/infra/ansible && ansible-playbook playbooks/node-health.yml --limit k3s-dev-1' +``` + +## Contract + +- **Read-only and side-effect-free.** It reads `/sys`, `/proc`, `/etc`, runs `lsblk`/`lscpu`, + and — only when it can escalate — `smartctl`. It writes nothing to the node and + **installs nothing**. `smartmontools` is declared in `node_baseline_packages`; if it is + absent the SMART checks SKIP with that as the stated reason (operator ruling 2026-08-31: + package installation is `node_baseline`'s job, never a health check's, never a manual + `apt-get`). +- **A skip is honest; a fake green is a defect.** Every check is `PASS` / `WARN` / `FAIL` / + `SKIP` / `INFO`, and a `SKIP` always carries *why* it could not be measured. Nothing that + was not read is reported as passing. The summary ends with a dedicated + "NOT READ (and why) — these are UNKNOWNS, not passes" section. +- **Works on bare metal and VMs.** Everything absent in a guest (SMART on virtio, DMI, + EDAC, `coretemp`, CPU throttle counters) skips cleanly instead of erroring. Verified + against both `k3s-prod-1` (HP EliteDesk, bare metal) and `k3s-dev-1` (Proxmox VM). +- **Never `become`s unless it must.** Only SMART needs root. + +## What it checks + +| id | what | +|---|---| +| `access.privilege` | whether `become` works at all (gates the root-only checks) | +| `storage.smart.` | overall-health, power-on hours, lifetime remaining, reallocated + pending sectors, CRC errors, drive temperature — **root only** | +| `storage.io_errors` | kernel SCSI/ATA `ioerr_cnt` / `iotmo_cnt` per disk (no root needed) | +| `storage.capacity`, `storage.inodes` | worst mounted filesystem, space and inodes | +| `storage.layout` | physical disks, and whether any device is genuinely free for `node_baseline_longhorn_device` / `node_baseline_etcd_device` | +| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite, EDAC correctable/uncorrectable counters | +| `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) | +| `thermal.max` | hottest sensor across thermal zones and hwmon chips | +| `pressure.cpu` / `.memory` / `.io` | PSI `some avg60` — dev learned to care about CPU PSI specifically | +| `network.link`, `network.errors`, `network.resolvers` | link state/speed, NIC error counters, DNS upstream redundancy per address family | +| `os.reboot_required`, `os.inventory`, `firmware.inventory` | pending reboot, OS/kernel/uptime, chassis + BIOS | + +Thresholds live in `defaults/main.yml` (`node_health_thresholds`); no verdict is hardcoded +in task logic. + +## Output + +1. A human summary printed in the play output. +2. The same summary and a structured JSON record under + `infra/ansible/_artifacts/node-health/.{txt,json}`. + +`_artifacts/` is in `infra/ansible/.gitignore` — the same convention as +`conformance/_artifacts/`. Generated output is never committed (it also carries drive +serial numbers). + +## The `become` subtlety (read before editing) + +`ansible_become` set as an inventory/group var — which is exactly what +`inventory/group_vars/k3s_prod/vars.yml` does — is a **connection variable**, and +connection variables **outrank** the `become:` play/task keyword. Verified on +ansible-core 2.20: a task saying `become: false` still tries to sudo and, on prod, dies +with `Missing sudo password` before the module runs. So this role overrides the +**variable** (`vars: ansible_become: false`) on the unprivileged work, and probes +escalation exactly once before attempting SMART. + +Equally verified: `failed_when: false` does **not** catch a missing-sudo-password error +(it is raised by the become plugin before the failure expression is evaluated); +`ignore_errors: true` does, and still populates the registered variable. That is why the +escalation probe uses `ignore_errors` — anything else turns "no vault yet" into a hard, +misleading play failure. + +## Known limits + +- Without root: **no SMART** (wear, power-on hours, reallocated/pending sectors, drive + temperature) and no DMI serial numbers. On prod that means creating + `inventory/group_vars/k3s_prod/vault.yml` (see that directory's README) and running + `node-baseline` so `smartmontools` exists — in that order. +- EDAC reports nothing on platforms with no ECC memory controller bound. That is + "ECC status unknown", not "no errors", and the role says so. +- `ioerr_cnt` counts commands the kernel saw fail, which includes benign rejected or + unsupported commands during discovery. Attribution needs `dmesg`, which needs root. diff --git a/infra/ansible/roles/node_health/defaults/main.yml b/infra/ansible/roles/node_health/defaults/main.yml new file mode 100644 index 0000000..65cc7c7 --- /dev/null +++ b/infra/ansible/roles/node_health/defaults/main.yml @@ -0,0 +1,78 @@ +--- +# node_health — READ-ONLY hardware/health inspection of a node (bare metal or VM). +# +# Sibling of roles/cluster_verify: cluster_verify answers "is the CLUSTER healthy", +# node_health answers "is the HARDWARE under it healthy". Same posture — it inspects and +# reports, it never changes the node. Deliberately NOT part of node_baseline: baseline +# installs and configures, health only looks. That separation is why this role must never +# `apt install` anything (smartmontools is declared in node_baseline_packages; if it is +# missing here the SMART checks SKIP with a reason, they do not self-heal). +# +# Serves R-ENV-01 (reproducible node provisioning — you cannot certify a node you have not +# measured), R-ENV-03 (bare-metal parity) and R-OBS-04 (node-level observability). + +# --- Reporting --------------------------------------------------------------------- +# Artifacts land under infra/ansible/_artifacts/node-health/, mirroring the repo's +# conformance/_artifacts/ convention: generated output, regenerated on every run, listed +# in .gitignore, never committed (it also carries drive serial numbers). +node_health_artifact_dir: "{{ playbook_dir }}/../_artifacts/node-health" +node_health_write_artifacts: true +# Print the human summary to the play output as well as writing it to the artifact dir. +node_health_show_summary: true +# Report-only by default: a health check that aborts the play hides the other 20 findings. +# Set true (e.g. in CI or a pre-build gate) to make any FAIL check end the play non-zero. +node_health_fail_on_fail: false +# Same, one step stricter: also fail on WARN. +node_health_fail_on_warn: false + +# --- Root-only checks --------------------------------------------------------------- +# Everything this role can read WITHOUT root, it reads without root. Only SMART needs +# escalation (the ATA/NVMe passthrough ioctls are root-only). The role PROBES become +# first and, if escalation is unavailable (e.g. prod's vaulted become password is not +# present — infra/PROD-PROVISIONING.md §5), those checks report SKIPPED with the exact +# reason. Never a hard failure, and never a fake PASS: a skip is honest. +node_health_try_privileged: true +# --nocheck=standby: never spin up a sleeping disk just to read it. Read-only hygiene. +node_health_smartctl_args: "--json=c --all --nocheck=standby" + +# --- Thresholds --------------------------------------------------------------------- +# All verdicts come from here, so a site can retune without touching task logic. +node_health_thresholds: + # Filesystems + fs_used_pct_warn: 80 + fs_used_pct_fail: 90 + inode_used_pct_warn: 80 + # Memory (percentage of MemTotal still available) + mem_available_pct_warn: 15 + mem_available_pct_fail: 5 + # Temperatures (deg C, any sensor) + temp_warn_c: 70 + temp_fail_c: 85 + # Pressure-stall: "some" avg60, percent of wall time stalled. Dev learned to care about + # CPU PSI specifically (it shows saturation long before load average makes it obvious). + psi_some_avg60_warn: 10.0 + psi_some_avg60_fail: 40.0 + # Network + nic_min_speed_mbps: 1000 + # A single upstream resolver is a known SPOF on this project — it broke containerd + # image pulls and CoreDNS forwarding on dev (DECISIONS-LOG 2026-08-31, finding 5). + min_nameservers: 2 + # SMART (only evaluated when smartctl actually ran) + smart_reallocated_warn: 1 + smart_reallocated_fail: 50 + smart_pending_warn: 1 + smart_pending_fail: 10 + smart_lifetime_remaining_warn_pct: 20 + smart_lifetime_remaining_fail_pct: 10 + smart_temp_warn_c: 55 + smart_temp_fail_c: 70 + # Power-on hours past which a consumer SSD is simply old. Informational — age alone is + # not a failure, but it belongs in the operator's field of view when deciding whether a + # drive may hold production state (PROD-PROVISIONING.md §2). + smart_power_on_hours_notice: 40000 + +# ATA SMART attribute IDs surfaced verbatim in the report. Vendors disagree on names, so +# the report prints whatever the drive calls them and the checks key off the IDs. +node_health_smart_attribute_ids: [5, 9, 177, 179, 181, 182, 183, 184, 187, 188, 190, + 194, 195, 196, 197, 198, 199, 202, 231, 232, 233, + 241, 242, 246] diff --git a/infra/ansible/roles/node_health/files/node_health_collect.py b/infra/ansible/roles/node_health/files/node_health_collect.py new file mode 100644 index 0000000..f627861 --- /dev/null +++ b/infra/ansible/roles/node_health/files/node_health_collect.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python3 +# ============================================================================= +# node_health_collect.py — UNPRIVILEGED hardware/health collector for one node. +# +# Run by roles/node_health via ansible.builtin.script. Prints ONE JSON document on +# stdout and nothing else. It is strictly READ-ONLY: it opens files under /sys, /proc +# and /etc, and shells out only to `lsblk` (a read-only query). It never writes to the +# node, never installs anything, and never needs root. +# +# WHY A COLLECTOR SCRIPT AND NOT 40 ANSIBLE TASKS +# Everything here is "read a sysfs file that may or may not exist and may or may not +# be readable". Expressing that as tasks means ~40 SSH round trips plus fragile Jinja +# parsing of `cat` output. One script is a single round trip, parses in Python where +# it is legible, and — the point that matters for this repo — can express the +# difference between "value is 0" and "value could not be read" HONESTLY, per field, +# instead of collapsing both into an empty string. Anything unreadable is emitted as +# null and appended to the top-level "unreadable" list with the reason. A skip is +# honest; a fake zero is a defect. +# +# WHY NOT ANSIBLE'S OWN setup MODULE +# `setup` gives DMI/CPU/memory/mounts, but none of the things this role exists for: +# thermal zones, EDAC correctable-error counters, CPU throttle counters, PSI +# (/proc/pressure), SCSI per-disk error counters, per-NIC error/drop statistics. The +# role therefore runs with gather_facts disabled and takes everything from here, so +# there is exactly one source of truth for the report and the JSON artifact. +# +# PORTABILITY (bare metal AND VMs — the role must be reusable against the dev nodes) +# Every read is optional. A VM has no DMI product_name worth reading, no coretemp +# hwmon, no EDAC, no SCSI error counters on virtio, and no SMART. All of those come +# back null/absent rather than raising. Nothing in this file assumes an HP EliteDesk. +# +# Python 3.6+, standard library only (no pip, no ansible module deps). +# ============================================================================= + +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import time + +SCHEMA = "node-health/1" + +# Fields that could not be read, with why. Appended to by read_text()/read_int(). +UNREADABLE = [] + + +def read_text(path, optional=True): + """Read a small sysfs/procfs file. Returns stripped str, or None when unreadable. + + `optional=False` records the failure in UNREADABLE so the report can say WHY a + check was skipped (permission denied vs. not present are very different answers — + the first is "give Ansible become", the second is "this hardware has no such + sensor").""" + try: + with open(path, "r", errors="replace") as fh: + return fh.read().strip() + except FileNotFoundError: + if not optional: + UNREADABLE.append({"path": path, "reason": "not present"}) + return None + except PermissionError: + # ALWAYS recorded, even when optional: permission-denied is the signal that a + # value exists but we lack root, which is exactly what the operator needs told. + UNREADABLE.append({"path": path, "reason": "permission denied (needs root)"}) + return None + except OSError as exc: + UNREADABLE.append({"path": path, "reason": "read error: %s" % exc}) + return None + + +def read_int(path, optional=True): + raw = read_text(path, optional=optional) + if raw is None: + return None + try: + return int(raw.split()[0]) + except (ValueError, IndexError): + return None + + +def read_kv(path, sep=":"): + """Parse a `key: value` file (/proc/meminfo, /etc/os-release) into a dict.""" + raw = read_text(path) + if raw is None: + return None + out = {} + for line in raw.splitlines(): + if sep not in line: + continue + key, _, value = line.partition(sep) + out[key.strip()] = value.strip().strip('"') + return out + + +def run(argv, timeout=20): + """Run a read-only query command. Returns (rc, stdout, stderr); rc None if absent.""" + if shutil.which(argv[0]) is None: + return None, "", "%s not installed" % argv[0] + try: + proc = subprocess.run( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout + ) + return ( + proc.returncode, + proc.stdout.decode("utf-8", "replace"), + proc.stderr.decode("utf-8", "replace"), + ) + except (OSError, subprocess.SubprocessError) as exc: + return None, "", str(exc) + + +# --- DMI / firmware ----------------------------------------------------------------- +# /sys/class/dmi/id/* is world-readable EXCEPT the identifying serials +# (product_serial, board_serial, chassis_serial, product_uuid — mode 0400 root). Those +# come back null here and are reported as "needs root", not as "absent". A VM usually +# has the directory but with hypervisor-ish values (QEMU/SeaBIOS), which is fine and +# still worth recording. +DMI_PUBLIC = [ + "sys_vendor", + "product_name", + "product_version", + "product_family", + "product_sku", + "board_vendor", + "board_name", + "board_version", + "bios_vendor", + "bios_version", + "bios_date", + "bios_release", + "ec_firmware_release", + "chassis_type", + "chassis_vendor", +] +DMI_ROOT_ONLY = ["product_serial", "board_serial", "chassis_serial", "product_uuid"] + + +def collect_dmi(): + base = "/sys/class/dmi/id" + if not os.path.isdir(base): + return {"present": False} + dmi = {"present": True} + for field in DMI_PUBLIC: + dmi[field] = read_text(os.path.join(base, field)) + # Read the root-only ones deliberately so PermissionError lands in UNREADABLE and + # the operator can see exactly which fields root would add. + for field in DMI_ROOT_ONLY: + dmi[field] = read_text(os.path.join(base, field), optional=False) + return dmi + + +# --- CPU ---------------------------------------------------------------------------- +def collect_cpu(): + cpu = {} + rc, out, _ = run(["lscpu", "-J"]) + lscpu = {} + if rc == 0: + try: + for entry in json.loads(out).get("lscpu", []): + lscpu[entry.get("field", "").rstrip(":")] = entry.get("data") + except ValueError: + lscpu = {} + cpu["model"] = lscpu.get("Model name") + cpu["vendor"] = lscpu.get("Vendor ID") + cpu["architecture"] = lscpu.get("Architecture") + cpu["cpus"] = lscpu.get("CPU(s)") + cpu["threads_per_core"] = lscpu.get("Thread(s) per core") + cpu["cores_per_socket"] = lscpu.get("Core(s) per socket") + cpu["sockets"] = lscpu.get("Socket(s)") + # "Virtualization: VT-x" on bare metal; "Hypervisor vendor: KVM" inside a VM. Both + # are recorded — the first tells us KubeVirt can run here (R-PLAT-03), the second + # tells us this node IS a guest and that SMART/DMI/EDAC skips are expected. + cpu["virtualization"] = lscpu.get("Virtualization") + cpu["hypervisor_vendor"] = lscpu.get("Hypervisor vendor") + cpu["virtualization_type"] = lscpu.get("Virtualization type") + cpu["is_virtual_guest"] = bool(lscpu.get("Hypervisor vendor")) + cpu["kvm_device"] = os.path.exists("/dev/kvm") + + flags = (read_text("/proc/cpuinfo") or "") + cpu["hardware_virt_flag"] = bool(re.search(r"^flags\s*:.*\b(vmx|svm)\b", flags, re.M)) + + # cpufreq: governor/driver and the frequency window. Absent inside most VMs. + cpufreq = "/sys/devices/system/cpu/cpu0/cpufreq" + if os.path.isdir(cpufreq): + cpu["scaling"] = { + "governor": read_text(cpufreq + "/scaling_governor"), + "driver": read_text(cpufreq + "/scaling_driver"), + "base_khz": read_int(cpufreq + "/base_frequency"), + "min_khz": read_int(cpufreq + "/cpuinfo_min_freq"), + "max_khz": read_int(cpufreq + "/cpuinfo_max_freq"), + "cur_khz": read_int(cpufreq + "/scaling_cur_freq"), + } + else: + cpu["scaling"] = None + + # Thermal throttling. These counters are cumulative since boot: non-zero means the + # package or a core has ACTUALLY been clamped, which on a mini-PC in a cupboard is + # the difference between "it fits" and "it cooks". Absent in VMs. + throttle_dirs = sorted(glob.glob("/sys/devices/system/cpu/cpu*/thermal_throttle")) + if throttle_dirs: + core_total = 0 + pkg_max = 0 + core_time = 0 + for tdir in throttle_dirs: + core_total += read_int(tdir + "/core_throttle_count") or 0 + core_time += read_int(tdir + "/core_throttle_total_time_ms") or 0 + pkg_max = max(pkg_max, read_int(tdir + "/package_throttle_count") or 0) + cpu["throttle"] = { + "core_throttle_count_total": core_total, + "core_throttle_total_time_ms": core_time, + "package_throttle_count": pkg_max, + "cpus_reporting": len(throttle_dirs), + } + else: + cpu["throttle"] = None + + load = read_text("/proc/loadavg") + if load: + parts = load.split() + cpu["loadavg"] = [float(parts[0]), float(parts[1]), float(parts[2])] + cpu["runnable"] = parts[3] if len(parts) > 3 else None + return cpu + + +# --- Memory + EDAC ------------------------------------------------------------------ +def collect_memory(): + meminfo = read_kv("/proc/meminfo") or {} + + def kb(key): + raw = meminfo.get(key) + if not raw: + return None + try: + return int(raw.split()[0]) + except (ValueError, IndexError): + return None + + mem = { + "mem_total_kb": kb("MemTotal"), + "mem_available_kb": kb("MemAvailable"), + "mem_free_kb": kb("MemFree"), + "swap_total_kb": kb("SwapTotal"), + "swap_free_kb": kb("SwapFree"), + "hugepages_total": kb("HugePages_Total"), + } + + # EDAC = the kernel's memory-controller error reporting. Only populated when the + # platform has ECC RAM *and* a driver bound (mc0, mc1, ...). Consumer desktop + # boards and VMs have /sys/devices/system/edac/mc but NO mcN under it — that is + # "ECC not exposed", NOT "zero errors". The distinction is the whole point: we + # report present=false and the check SKIPS rather than claiming a clean bill. + controllers = sorted(glob.glob("/sys/devices/system/edac/mc/mc[0-9]*")) + edac = { + "sysfs_present": os.path.isdir("/sys/devices/system/edac/mc"), + "controllers_present": bool(controllers), + "controllers": [], + "ce_total": None, + "ue_total": None, + } + if controllers: + ce_total = 0 + ue_total = 0 + for mc in controllers: + ce = read_int(mc + "/ce_count") + ue = read_int(mc + "/ue_count") + edac["controllers"].append( + { + "name": os.path.basename(mc), + "mc_name": read_text(mc + "/mc_name"), + "size_mb": read_int(mc + "/size_mb"), + "ce_count": ce, + "ue_count": ue, + "ce_noinfo_count": read_int(mc + "/ce_noinfo_count"), + "ue_noinfo_count": read_int(mc + "/ue_noinfo_count"), + } + ) + ce_total += ce or 0 + ue_total += ue or 0 + edac["ce_total"] = ce_total + edac["ue_total"] = ue_total + mem["edac"] = edac + return mem + + +# --- Thermals ----------------------------------------------------------------------- +def collect_thermal(): + zones = [] + for zdir in sorted(glob.glob("/sys/class/thermal/thermal_zone*")): + milli = read_int(zdir + "/temp") + crit = None + for trip in sorted(glob.glob(zdir + "/trip_point_*_type")): + if (read_text(trip) or "") == "critical": + crit = read_int(trip.replace("_type", "_temp")) + break + zones.append( + { + "zone": os.path.basename(zdir), + "type": read_text(zdir + "/type"), + "temp_c": round(milli / 1000.0, 1) if milli is not None else None, + "critical_c": round(crit / 1000.0, 1) if crit is not None else None, + } + ) + + # hwmon carries the interesting per-chip sensors (coretemp package/cores, PCH, the + # HP embedded controller). Also where FAN inputs live when the platform exposes them. + chips = [] + for hdir in sorted(glob.glob("/sys/class/hwmon/hwmon*")): + chip = {"path": os.path.basename(hdir), "name": read_text(hdir + "/name"), + "temps": [], "fans": []} + for tin in sorted(glob.glob(hdir + "/temp*_input")): + milli = read_int(tin) + base = tin[: -len("_input")] + chip["temps"].append( + { + "label": read_text(base + "_label") or os.path.basename(base), + "temp_c": round(milli / 1000.0, 1) if milli is not None else None, + "max_c": (lambda v: round(v / 1000.0, 1) if v is not None else None)( + read_int(base + "_max") + ), + "crit_c": (lambda v: round(v / 1000.0, 1) if v is not None else None)( + read_int(base + "_crit") + ), + } + ) + for fin in sorted(glob.glob(hdir + "/fan*_input")): + base = fin[: -len("_input")] + chip["fans"].append( + { + "label": read_text(base + "_label") or os.path.basename(base), + "rpm": read_int(fin), + } + ) + if chip["temps"] or chip["fans"]: + chips.append(chip) + return {"zones": zones, "hwmon": chips} + + +# --- Pressure-stall (PSI) ----------------------------------------------------------- +# /proc/pressure/{cpu,memory,io}. Dev learned to care about CPU PSI (saturation shows +# up here long before load average makes it obvious), so it is first-class in this role. +# Requires CONFIG_PSI=y + psi=1 — absent on some kernels, which SKIPS honestly. +def collect_pressure(): + out = {"available": os.path.isdir("/proc/pressure")} + for resource in ("cpu", "memory", "io"): + raw = read_text("/proc/pressure/" + resource) + if raw is None: + out[resource] = None + continue + parsed = {} + for line in raw.splitlines(): + fields = line.split() + if not fields: + continue + kind = fields[0] # "some" | "full" + parsed[kind] = {} + for token in fields[1:]: + key, _, value = token.partition("=") + try: + parsed[kind][key] = float(value) + except ValueError: + parsed[kind][key] = value + out[resource] = parsed + return out + + +# --- Storage ------------------------------------------------------------------------ +PSEUDO_FS = { + "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "cgroup2", "pstore", + "bpf", "securityfs", "debugfs", "tracefs", "configfs", "fusectl", "hugetlbfs", + "mqueue", "efivarfs", "autofs", "binfmt_misc", "ramfs", "nsfs", "squashfs", + "overlay", "fuse.portal", +} + + +def collect_storage(): + disks = [] + + # lsblk is the reliable source for model/serial/transport (it reads udev, and does + # so WITHOUT root). sysfs fills in what lsblk does not expose: the SCSI-layer error + # counters and the queue tunables. + lsblk_map = {} + rc, out, _ = run( + ["lsblk", "-J", "-b", "-o", + "NAME,KNAME,PATH,TYPE,SIZE,ROTA,MODEL,SERIAL,TRAN,REV,FSTYPE,MOUNTPOINTS,PARTLABEL"] + ) + lsblk_tree = [] + if rc == 0: + try: + lsblk_tree = json.loads(out).get("blockdevices", []) + except ValueError: + lsblk_tree = [] + + def walk(nodes): + for node in nodes: + lsblk_map[node.get("kname")] = node + walk(node.get("children") or []) + + walk(lsblk_tree) + + for bdir in sorted(glob.glob("/sys/block/*")): + kname = os.path.basename(bdir) + # Skip pseudo block devices — they carry no hardware health and would only add + # noise to the report (loop mounts, ramdisks, device-mapper stacks, CD-ROM). + if re.match(r"^(loop|ram|sr|fd|dm-|zram|md)", kname): + continue + node = lsblk_map.get(kname, {}) + sectors = read_int(bdir + "/size") + dev = bdir + "/device" + disk = { + "kname": kname, + "path": node.get("path") or "/dev/" + kname, + "model": node.get("model") or read_text(dev + "/model"), + "vendor": read_text(dev + "/vendor"), + "firmware_rev": node.get("rev") or read_text(dev + "/rev"), + "serial": node.get("serial"), + "transport": node.get("tran"), + # A device can carry a filesystem DIRECTLY, with no partition table at all — + # which is exactly how roles/node_baseline formats the Longhorn and etcd data + # disks (mkfs on /dev/vdb, mounted by LABEL). Such a device has zero + # partitions but is emphatically NOT spare, so both facts are recorded and + # the "is there a free device" check keys off all three. + "fstype": node.get("fstype"), + "mountpoints": [m for m in (node.get("mountpoints") or []) if m], + "size_bytes": (sectors * 512) if sectors is not None else node.get("size"), + "rotational": (read_int(bdir + "/queue/rotational") == 1) + if read_int(bdir + "/queue/rotational") is not None + else node.get("rota"), + "scheduler": read_text(bdir + "/queue/scheduler"), + "logical_block_size": read_int(bdir + "/queue/logical_block_size"), + "physical_block_size": read_int(bdir + "/queue/physical_block_size"), + # SCSI/ATA error counters, readable WITHOUT root. Not a substitute for SMART + # (no wear, no power-on hours, no reallocated sectors) but a genuine signal: + # ioerr_cnt > 0 means the kernel saw command failures on this device. + "scsi_counters": { + "iodone_cnt": (lambda v: int(v, 16) if v else None)( + read_text(dev + "/iodone_cnt") + ), + "ioerr_cnt": (lambda v: int(v, 16) if v else None)( + read_text(dev + "/ioerr_cnt") + ), + "iotmo_cnt": (lambda v: int(v, 16) if v else None)( + read_text(dev + "/iotmo_cnt") + ), + } + if (os.path.isdir(dev) and os.path.exists(dev + "/ioerr_cnt")) + else None, + "partitions": [], + } + for pdir in sorted(glob.glob(bdir + "/" + kname + "*")): + pname = os.path.basename(pdir) + if not os.path.exists(pdir + "/partition"): + continue + pnode = lsblk_map.get(pname, {}) + psect = read_int(pdir + "/size") + disk["partitions"].append( + { + "kname": pname, + "path": pnode.get("path") or "/dev/" + pname, + "size_bytes": (psect * 512) if psect is not None else pnode.get("size"), + "fstype": pnode.get("fstype"), + "partlabel": pnode.get("partlabel"), + "mountpoints": [m for m in (pnode.get("mountpoints") or []) if m], + } + ) + disks.append(disk) + + # iSCSI/FC devices are ATTACHED VOLUMES, not hardware in this box: on a k3s node they + # are Longhorn replicas the node has logged into, they come and go with workloads, and + # they have no drive health of their own. Keeping them in `disks` would (a) pad the + # report with churn and (b) — far worse — let a Longhorn volume look like a "spare + # device" that node_baseline could be pointed at and reformatted. Split, never dropped. + attached = [d for d in disks if (d.get("transport") or "") in ("iscsi", "fc", "fcoe")] + disks = [d for d in disks if d not in attached] + + # Mounted filesystems with real free space, from /proc/self/mounts + statvfs. No + # `df` fork, and no root needed. + mounts = [] + raw_mounts = read_text("/proc/self/mounts") or "" + seen = set() + for line in raw_mounts.splitlines(): + fields = line.split() + if len(fields) < 3: + continue + source, target, fstype = fields[0], fields[1].replace("\\040", " "), fields[2] + if fstype in PSEUDO_FS or not source.startswith("/dev/"): + continue + if target in seen: + continue + seen.add(target) + try: + st = os.statvfs(target) + except OSError as exc: + UNREADABLE.append({"path": target, "reason": "statvfs failed: %s" % exc}) + continue + total = st.f_blocks * st.f_frsize + avail = st.f_bavail * st.f_frsize + mounts.append( + { + "source": source, + "mountpoint": target, + "fstype": fstype, + "options": fields[3] if len(fields) > 3 else None, + "size_bytes": total, + "avail_bytes": avail, + "used_pct": round((total - avail) * 100.0 / total, 1) if total else None, + "inodes_total": st.f_files, + "inodes_avail": st.f_favail, + "inodes_used_pct": round( + (st.f_files - st.f_favail) * 100.0 / st.f_files, 1 + ) + if st.f_files + else None, + } + ) + return {"disks": disks, "attached_volumes": attached, "mounts": mounts} + + +# --- Network ------------------------------------------------------------------------ +def collect_network(): + interfaces = [] + for ndir in sorted(glob.glob("/sys/class/net/*")): + name = os.path.basename(ndir) + # Only physical/primary interfaces matter for node health. Virtual plumbing + # (cilium_*, lxc*, veth, cni, flannel, docker/podman bridges, k8s tunnels) is + # noise here and churns constantly. + if re.match(r"^(lo|veth|cilium|lxc|cni|flannel|docker|podman|br-|kube|tunl|" + r"vxlan|virbr|tailscale|wg|ip6tnl|sit|dummy|bond_|nodelocal)", name): + continue + stats = {} + for sfile in sorted(glob.glob(ndir + "/statistics/*")): + stats[os.path.basename(sfile)] = read_int(sfile) + speed = read_int(ndir + "/speed") # -1 / unreadable when the link is down + driver = None + drvlink = ndir + "/device/driver" + if os.path.islink(drvlink): + driver = os.path.basename(os.path.realpath(drvlink)) + interfaces.append( + { + "name": name, + "operstate": read_text(ndir + "/operstate"), + "carrier": read_int(ndir + "/carrier"), + "speed_mbps": speed if (speed is not None and speed > 0) else None, + "duplex": read_text(ndir + "/duplex"), + "mtu": read_int(ndir + "/mtu"), + "mac": read_text(ndir + "/address"), + "driver": driver, + "virtual": not os.path.exists(ndir + "/device"), + "statistics": { + k: stats.get(k) + for k in ( + "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", + "rx_errors", "tx_errors", "rx_dropped", "tx_dropped", + "rx_crc_errors", "rx_frame_errors", "rx_over_errors", + "tx_carrier_errors", "collisions", + ) + if k in stats + }, + } + ) + # Resolver config is a known SPOF class on this project (a single upstream broke + # containerd pulls and CoreDNS forwarding on dev). Cheap to read here, so the health + # report can flag it on prod too. + resolv = read_text("/etc/resolv.conf") or "" + nameservers = re.findall(r"^\s*nameserver\s+(\S+)", resolv, re.M) + return {"interfaces": interfaces, "nameservers": nameservers} + + +# --- OS / kernel / uptime ----------------------------------------------------------- +def collect_os(): + osrel = read_kv("/etc/os-release", sep="=") or {} + uptime_raw = read_text("/proc/uptime") + uptime = None + if uptime_raw: + try: + uptime = float(uptime_raw.split()[0]) + except (ValueError, IndexError): + uptime = None + return { + "distribution": osrel.get("NAME"), + "version": osrel.get("VERSION"), + "version_id": osrel.get("VERSION_ID"), + "pretty_name": osrel.get("PRETTY_NAME"), + "kernel": (read_text("/proc/sys/kernel/osrelease")), + "kernel_version_full": (read_text("/proc/version")), + "hostname": (read_text("/proc/sys/kernel/hostname")), + "uptime_seconds": uptime, + "boot_time_epoch": (time.time() - uptime) if uptime else None, + # Debian drops this file after a kernel/libc upgrade needing a reboot. + "reboot_required": os.path.exists("/var/run/reboot-required") + or os.path.exists("/run/reboot-required"), + "dmesg_restricted": (read_int("/proc/sys/kernel/dmesg_restrict") == 1), + } + + +# --- Tooling presence --------------------------------------------------------------- +# The role must NEVER install anything (a health check is read-only and side-effect +# free — package installation is node_baseline's job, operator ruling 2026-08-31). So +# we only DETECT, and the role skips the SMART checks with an accurate reason. +def collect_tools(): + tools = {} + for tool in ("smartctl", "nvme", "ethtool", "dmidecode", "lsblk", "lscpu"): + tools[tool] = shutil.which(tool) or shutil.which(tool, path="/usr/sbin:/sbin:/usr/bin:/bin") + return tools + + +def main(): + doc = { + "schema": SCHEMA, + "collected_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "collector_uid": os.geteuid(), + "os": collect_os(), + "dmi": collect_dmi(), + "cpu": collect_cpu(), + "memory": collect_memory(), + "thermal": collect_thermal(), + "pressure": collect_pressure(), + "storage": collect_storage(), + "network": collect_network(), + "tools": collect_tools(), + } + doc["unreadable"] = UNREADABLE + json.dump(doc, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/infra/ansible/roles/node_health/tasks/assess.yml b/infra/ansible/roles/node_health/tasks/assess.yml new file mode 100644 index 0000000..8c0dd96 --- /dev/null +++ b/infra/ansible/roles/node_health/tasks/assess.yml @@ -0,0 +1,509 @@ +--- +# Turn raw facts into explicit, named checks. Every check is one of: +# +# PASS measured, and within threshold +# WARN measured, outside the warn threshold — worth a human look +# FAIL measured, outside the fail threshold — do not put production state here +# SKIP NOT measured, with the reason attached (missing root, missing tool, sensor not +# present on this hardware). A SKIP is an honest "unknown" and is NEVER rendered +# as a pass — this repo's discipline is absolute on that point. +# INFO measured, no pass/fail semantics (inventory: firmware, uptime, CPU scaling) +# +# All thresholds come from node_health_thresholds (defaults/main.yml) so retuning never +# means editing logic here. + +- name: Derive intermediate values used by several checks + ansible.builtin.set_fact: + # Every temperature the box exposes, from both thermal zones and hwmon chips. + # `select('number')` drops any sensor that returned null (unreadable/absent). + node_health_temps_c: >- + {{ ((node_health_facts.thermal.zones | map(attribute='temp_c') | list) + + (node_health_facts.thermal.hwmon | map(attribute='temps') | flatten + | map(attribute='temp_c') | list)) | select('number') | list }} + # Filesystems, worst-first by percentage used / inodes used. + node_health_mounts_by_use: >- + {{ node_health_facts.storage.mounts | selectattr('used_pct', 'number') + | sort(attribute='used_pct') | reverse | list }} + node_health_mounts_by_inode: >- + {{ node_health_facts.storage.mounts | selectattr('inodes_used_pct', 'number') + | sort(attribute='inodes_used_pct') | reverse | list }} + # Physical NICs that are actually up. Virtual plumbing was already excluded by the + # collector; `virtual` here catches bonds/VLANs the regex let through. + node_health_nics_up: >- + {{ node_health_facts.network.interfaces + | selectattr('operstate', 'equalto', 'up') | list }} + # Devices that could actually BECOME a dedicated Longhorn or etcd volume: no + # partitions, no whole-device filesystem, nothing mounted from them. All three + # conditions matter — node_baseline formats the data disks with no partition table at + # all (mkfs straight onto /dev/vdb, mounted by LABEL), so "no partitions" alone would + # nominate a node's live Longhorn disk for reformatting. iSCSI/FC volumes were already + # split out by the collector for the same reason. On a single-disk node this list is + # empty — precisely the finding that blocks PROD-PROVISIONING.md §2. + node_health_free_devices: >- + {{ node_health_facts.storage.disks + | selectattr('partitions', 'equalto', []) + | selectattr('fstype', 'none') + | selectattr('mountpoints', 'equalto', []) + | list }} + vars: + ansible_become: false + +- name: Assess — privilege escalation (gates every root-only check below) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'access.privilege', + 'title': 'Privilege escalation (become) for root-only checks', + 'status': ('INFO' if node_health_root_available | bool else 'SKIP'), + 'detail': node_health_root_skip_reason }] }}" + +# --- Storage ------------------------------------------------------------------------ +- name: Assess — SMART per physical disk + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'storage.smart.' ~ item.kname, + 'title': 'SMART health — ' ~ item.path ~ ' (' ~ (item.model | default('unknown model')) ~ ')', + 'status': smart_status, + 'detail': smart_detail }] }}" + loop: "{{ node_health_facts.storage.disks }}" + loop_control: + label: "{{ item.path }}" + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + disk_smart: "{{ (node_health_smart | selectattr('path', 'equalto', item.path) | list | first) | default({}) }}" + # Sub-verdicts. Each is SKIP unless the number was actually read. + s_health: >- + {{ 'SKIP' if (disk_smart.passed is not defined or disk_smart.passed is none) + else ('PASS' if disk_smart.passed else 'FAIL') }} + s_realloc: >- + {{ 'SKIP' if (disk_smart.reallocated_sectors is not defined or disk_smart.reallocated_sectors is not number) + else ('FAIL' if disk_smart.reallocated_sectors | int >= th.smart_reallocated_fail + else ('WARN' if disk_smart.reallocated_sectors | int >= th.smart_reallocated_warn else 'PASS')) }} + s_pending: >- + {{ 'SKIP' if (disk_smart.pending_sectors is not defined or disk_smart.pending_sectors is not number) + else ('FAIL' if disk_smart.pending_sectors | int >= th.smart_pending_fail + else ('WARN' if disk_smart.pending_sectors | int >= th.smart_pending_warn else 'PASS')) }} + s_life: >- + {{ 'SKIP' if (disk_smart.lifetime_remaining_pct is not defined or disk_smart.lifetime_remaining_pct is not number) + else ('FAIL' if disk_smart.lifetime_remaining_pct | int <= th.smart_lifetime_remaining_fail_pct + else ('WARN' if disk_smart.lifetime_remaining_pct | int <= th.smart_lifetime_remaining_warn_pct else 'PASS')) }} + s_temp: >- + {{ 'SKIP' if (disk_smart.temperature_c is not defined or disk_smart.temperature_c is not number) + else ('FAIL' if disk_smart.temperature_c | int >= th.smart_temp_fail_c + else ('WARN' if disk_smart.temperature_c | int >= th.smart_temp_warn_c else 'PASS')) }} + # Worst sub-verdict wins. `select('in', ...)` preserves the order of the left-hand + # list, so the LAST survivor is the most severe present. + smart_status: >- + {{ (['PASS', 'WARN', 'FAIL'] | select('in', [s_health, s_realloc, s_pending, s_life, s_temp]) | list | last) + | default('SKIP', true) }} + smart_detail: >- + {{ ('not read — ' ~ node_health_root_skip_reason) + if not (node_health_root_available | bool) + else ('not read — smartmontools is not installed on this node; it ships with + roles/node_baseline (node_baseline_packages), which has not been run here. + This role never installs packages.') + if not (node_health_facts.tools.smartctl is truthy) + else ('smartctl ran but this device exposes no SMART data (' + ~ (disk_smart.messages | default([]) | join('; ')) ~ ') — normal for virtio/VM disks') + if not (disk_smart.available | default(false)) + else ('overall-health=' ~ ('PASSED' if disk_smart.passed else 'FAILED') + ~ ', power_on_hours=' ~ (disk_smart.power_on_hours | default('?')) + ~ ', lifetime_remaining=' ~ (disk_smart.lifetime_remaining_pct | default('?')) ~ '%' + ~ ', reallocated=' ~ (disk_smart.reallocated_sectors | default('?')) + ~ ', pending=' ~ (disk_smart.pending_sectors | default('?')) + ~ ', temp=' ~ (disk_smart.temperature_c | default('?')) ~ 'C') }} + +- name: Assess — kernel-visible I/O errors per disk (no root needed) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'storage.io_errors', + 'title': 'Kernel I/O error counters on physical disks (SCSI/ATA layer)', + 'status': io_status, + 'detail': io_detail }] }}" + vars: + ansible_become: false + # ioerr_cnt/iotmo_cnt are readable WITHOUT root. They are not a substitute for SMART + # (no wear, no power-on hours, no reallocated sectors) but they are a real signal: a + # non-zero count means the kernel actually saw failed or timed-out commands on that + # device since boot. + counted: "{{ node_health_facts.storage.disks | selectattr('scsi_counters', 'mapping') | list }}" + err_total: >- + {{ (counted | map(attribute='scsi_counters.ioerr_cnt') | select('number') | list | sum) + + (counted | map(attribute='scsi_counters.iotmo_cnt') | select('number') | list | sum) }} + io_status: >- + {{ 'SKIP' if (counted | length) == 0 + else ('PASS' if err_total | int == 0 else 'WARN') }} + io_sum_done: "{{ counted | map(attribute='scsi_counters.iodone_cnt') | select('number') | list | sum }}" + io_sum_err: "{{ counted | map(attribute='scsi_counters.ioerr_cnt') | select('number') | list | sum }}" + io_sum_tmo: "{{ counted | map(attribute='scsi_counters.iotmo_cnt') | select('number') | list | sum }}" + # Aggregate here; the STORAGE section of the summary prints the per-disk counters. + io_detail: >- + {{ 'no disk exposes SCSI/ATA error counters (virtio or NVMe-only node)' + if (counted | length) == 0 + else ((counted | map(attribute='kname') | join(', ')) ~ ': completed=' + ~ io_sum_done | string ~ ', errors=' ~ io_sum_err | string + ~ ', timeouts=' ~ io_sum_tmo | string + ~ ('. Non-zero errors are commands the kernel saw fail on that device + since boot. They are NOT proof of media failure (a rejected or + unsupported command during discovery counts too) and they cannot be + attributed without dmesg, which needs root.' + if err_total | int > 0 else '')) }} + +- name: Assess — filesystem free space + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'storage.capacity', + 'title': 'Filesystem free space (worst mount)', + 'status': fs_status, + 'detail': fs_detail }] }}" + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + worst: "{{ (node_health_mounts_by_use | first) | default({}) }}" + fs_status: >- + {{ 'SKIP' if (node_health_mounts_by_use | length) == 0 + else ('FAIL' if worst.used_pct >= th.fs_used_pct_fail + else ('WARN' if worst.used_pct >= th.fs_used_pct_warn else 'PASS')) }} + fs_detail: >- + {{ 'no block-backed filesystem found' if (node_health_mounts_by_use | length) == 0 + else (worst.mountpoint ~ ' ' ~ worst.used_pct | string ~ '% used (' + ~ (worst.avail_bytes | int / 1073741824) | round(1) | string ~ ' GiB free of ' + ~ (worst.size_bytes | int / 1073741824) | round(1) | string ~ ' GiB)') }} + +- name: Assess — filesystem inode headroom + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'storage.inodes', + 'title': 'Filesystem inode headroom (worst mount)', + 'status': in_status, + 'detail': in_detail }] }}" + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + worst: "{{ (node_health_mounts_by_inode | first) | default({}) }}" + in_status: >- + {{ 'SKIP' if (node_health_mounts_by_inode | length) == 0 + else ('WARN' if worst.inodes_used_pct >= th.inode_used_pct_warn else 'PASS') }} + in_detail: >- + {{ 'no filesystem reported inode usage' if (node_health_mounts_by_inode | length) == 0 + else (worst.mountpoint ~ ' ' ~ worst.inodes_used_pct | string ~ '% of inodes used') }} + +- name: Assess — data-disk layout (feeds PROD-PROVISIONING.md §2) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'storage.layout', + 'title': 'Physical disks and spare block devices for Longhorn/etcd', + 'status': 'INFO', + 'detail': layout_detail }] }}" + vars: + ansible_become: false + disks: "{{ node_health_facts.storage.disks }}" + attached: "{{ node_health_facts.storage.attached_volumes | default([]) }}" + layout_detail: >- + {{ (disks | length) | string ~ ' physical disk(s): ' + ~ (disks | map(attribute='kname') | join(', ')) ~ '. ' + ~ ((attached | length) | string ~ ' attached iSCSI/FC volume(s) excluded (' + ~ (attached | map(attribute='kname') | join(', ')) ~ '). ' + if (attached | length) > 0 else '') + ~ ((node_health_free_devices | length) | string ~ ' device(s) with no partitions (' + ~ (node_health_free_devices | map(attribute='kname') | join(', ')) ~ ')' + if (node_health_free_devices | length) > 0 + else 'NO unpartitioned spare device — node_baseline_longhorn_device / + node_baseline_etcd_device have nothing to claim without repartitioning + or adding a drive') }} + +# --- Memory ------------------------------------------------------------------------- +- name: Assess — memory headroom + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'memory.available', + 'title': 'Available memory', + 'status': mem_status, + 'detail': mem_detail }] }}" + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + m: "{{ node_health_facts.memory }}" + avail_pct: >- + {{ ((m.mem_available_kb | int) * 100.0 / (m.mem_total_kb | int)) | round(1) + if (m.mem_total_kb is number and m.mem_available_kb is number and m.mem_total_kb | int > 0) + else none }} + mem_status: >- + {{ 'SKIP' if avail_pct is not number + else ('FAIL' if avail_pct | float <= th.mem_available_pct_fail + else ('WARN' if avail_pct | float <= th.mem_available_pct_warn else 'PASS')) }} + mem_detail: >- + {{ '/proc/meminfo unreadable' if avail_pct is not number + else ((m.mem_total_kb | int / 1048576) | round(1) | string ~ ' GiB total, ' + ~ (m.mem_available_kb | int / 1048576) | round(1) | string ~ ' GiB available (' + ~ avail_pct | string ~ '%)') }} + +- name: Assess — swap (kubelet requires it off) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'memory.swap', + 'title': 'Swap disabled (kubelet prerequisite)', + 'status': swap_status, + 'detail': swap_detail }] }}" + vars: + ansible_become: false + m: "{{ node_health_facts.memory }}" + swap_status: >- + {{ 'SKIP' if m.swap_total_kb is not number + else ('PASS' if m.swap_total_kb | int == 0 else 'WARN') }} + swap_detail: >- + {{ 'swap state unreadable' if m.swap_total_kb is not number + else ('no swap active' if m.swap_total_kb | int == 0 + else ((m.swap_total_kb | int / 1048576) | round(2) | string + ~ ' GiB swap active — roles/node_baseline runs swapoff and neutralises + the fstab entry; this node has not been baselined yet')) }} + +- name: Assess — ECC / EDAC correctable errors + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'memory.ecc', + 'title': 'ECC memory errors (EDAC correctable/uncorrectable counters)', + 'status': ecc_status, + 'detail': ecc_detail }] }}" + vars: + ansible_become: false + e: "{{ node_health_facts.memory.edac }}" + ecc_status: >- + {{ 'SKIP' if not (e.controllers_present | bool) + else ('FAIL' if (e.ue_total | int) > 0 + else ('WARN' if (e.ce_total | int) > 0 else 'PASS')) }} + ecc_detail: >- + {{ ('no EDAC memory controller is bound (' ~ ('/sys/devices/system/edac/mc exists but is empty' + if e.sysfs_present | bool else 'no EDAC sysfs at all') + ~ ') — this platform does not report ECC status. Non-ECC consumer RAM or a VM; + "no errors" CANNOT be confirmed on this hardware.') + if not (e.controllers_present | bool) + else ((e.controllers | length) | string ~ ' controller(s), correctable=' + ~ e.ce_total | string ~ ', uncorrectable=' ~ e.ue_total | string) }} + +# --- CPU ---------------------------------------------------------------------------- +- name: Assess — CPU thermal throttling + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'cpu.throttle', + 'title': 'CPU thermal throttling since boot', + 'status': thr_status, + 'detail': thr_detail }] }}" + vars: + ansible_become: false + t: "{{ node_health_facts.cpu.throttle }}" + thr_status: >- + {{ 'SKIP' if t is none + else ('PASS' if ((t.core_throttle_count_total | int) + (t.package_throttle_count | int)) == 0 + else 'WARN') }} + thr_detail: >- + {{ 'no thermal_throttle counters (VM guest or unsupported CPU)' if t is none + else ('core throttle events=' ~ t.core_throttle_count_total | string + ~ ' (' ~ t.core_throttle_total_time_ms | string ~ ' ms), package events=' + ~ t.package_throttle_count | string) }} + +- name: Assess — CPU inventory and frequency scaling + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'cpu.inventory', + 'title': 'CPU model, topology and frequency scaling', + 'status': 'INFO', + 'detail': cpu_detail }] }}" + vars: + ansible_become: false + c: "{{ node_health_facts.cpu }}" + cpu_detail: >- + {{ (c.model | default('unknown')) ~ ' — ' ~ (c.cpus | default('?')) ~ ' logical CPU(s), ' + ~ (c.sockets | default('?')) ~ ' socket(s) x ' ~ (c.cores_per_socket | default('?')) + ~ ' core(s) x ' ~ (c.threads_per_core | default('?')) ~ ' thread(s); load ' + ~ (c.loadavg | default([]) | join(' ')) ~ '; ' + ~ ('governor=' ~ c.scaling.governor ~ ' driver=' ~ c.scaling.driver + ~ ' cur=' ~ ((c.scaling.cur_khz | int) / 1000) | round | int | string ~ 'MHz' + ~ ' max=' ~ ((c.scaling.max_khz | int) / 1000) | round | int | string ~ 'MHz' + if c.scaling is mapping else 'no cpufreq scaling exposed') }} + +- name: Assess — hardware virtualisation (KubeVirt prerequisite, R-PLAT-03) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'cpu.virtualisation', + 'title': 'Hardware virtualisation available (VT-x/AMD-V + /dev/kvm)', + 'status': virt_status, + 'detail': virt_detail }] }}" + vars: + ansible_become: false + c: "{{ node_health_facts.cpu }}" + virt_status: >- + {{ 'PASS' if (c.hardware_virt_flag | bool and c.kvm_device | bool) else 'WARN' }} + virt_detail: >- + {{ 'vmx/svm flag=' ~ c.hardware_virt_flag | string ~ ', /dev/kvm=' + ~ c.kvm_device | string + ~ (', lscpu Virtualization=' ~ c.virtualization if c.virtualization else '') + ~ (', running as a guest of ' ~ c.hypervisor_vendor if c.is_virtual_guest | bool else '') }} + +# --- Thermals ----------------------------------------------------------------------- +- name: Assess — temperatures + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'thermal.max', + 'title': 'Hottest reported sensor', + 'status': temp_status, + 'detail': temp_detail }] }}" + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + hottest: "{{ (node_health_temps_c | max) if (node_health_temps_c | length) > 0 else none }}" + temp_status: >- + {{ 'SKIP' if hottest is not number + else ('FAIL' if hottest | float >= th.temp_fail_c + else ('WARN' if hottest | float >= th.temp_warn_c else 'PASS')) }} + temp_detail: >- + {{ 'no thermal sensor exposed (typical in a VM)' if hottest is not number + else (hottest | string ~ 'C hottest of ' ~ (node_health_temps_c | length) | string + ~ ' sensors; zones: ' + ~ (node_health_facts.thermal.zones + | map(attribute='type') | join(', ')) ) }} + +# --- Pressure-stall (PSI) ----------------------------------------------------------- +- name: Assess — pressure stall (CPU, memory, I/O) + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'pressure.' ~ item, + 'title': 'PSI ' ~ item ~ ' — some avg60 (% of wall time stalled)', + 'status': psi_status, + 'detail': psi_detail }] }}" + loop: [cpu, memory, io] + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + psi: "{{ node_health_facts.pressure[item] | default(none) }}" + avg60: "{{ psi.some.avg60 | default(none) if psi is mapping else none }}" + psi_status: >- + {{ 'SKIP' if avg60 is not number + else ('FAIL' if avg60 | float >= th.psi_some_avg60_fail + else ('WARN' if avg60 | float >= th.psi_some_avg60_warn else 'PASS')) }} + psi_detail: >- + {{ '/proc/pressure/' ~ item ~ ' unavailable (kernel without CONFIG_PSI, or psi=0)' + if avg60 is not number + else ('some avg10=' ~ psi.some.avg10 | string ~ ' avg60=' ~ psi.some.avg60 | string + ~ ' avg300=' ~ psi.some.avg300 | string + ~ (' | full avg60=' ~ psi.full.avg60 | string if psi.full is defined else '')) }} + +# --- Network ------------------------------------------------------------------------ +- name: Assess — link state and speed + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'network.link', + 'title': 'Physical NIC link state and negotiated speed', + 'status': link_status, + 'detail': link_detail }] }}" + vars: + ansible_become: false + th: "{{ node_health_thresholds }}" + up: "{{ node_health_nics_up }}" + slow: >- + {{ up | selectattr('speed_mbps', 'number') + | selectattr('speed_mbps', 'lt', th.nic_min_speed_mbps | int) | list }} + link_status: >- + {{ 'FAIL' if (up | length) == 0 + else ('WARN' if (slow | length) > 0 else 'PASS') }} + link_detail: >- + {{ 'no physical interface is up' if (up | length) == 0 + else (up | map(attribute='name') | zip(up | map(attribute='speed_mbps')) + | map('join', '=') | join('Mb/s, ') ~ 'Mb/s' + ~ ' (driver ' ~ (up | map(attribute='driver') | join(', ')) ~ ')') }} + +- name: Assess — link error counters + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'network.errors', + 'title': 'NIC error counters since boot (rx/tx errors, CRC, carrier)', + 'status': ('PASS' if err_total | int == 0 else 'WARN'), + 'detail': err_detail }] }}" + vars: + ansible_become: false + up: "{{ node_health_nics_up }}" + err_keys: [rx_errors, tx_errors, rx_crc_errors, rx_frame_errors, rx_over_errors, tx_carrier_errors] + err_total: >- + {{ up | map(attribute='statistics') | map('dict2items') | flatten + | selectattr('key', 'in', err_keys) | map(attribute='value') + | select('number') | list | sum }} + drop_total: >- + {{ up | map(attribute='statistics') | map('dict2items') | flatten + | selectattr('key', 'in', ['rx_dropped', 'tx_dropped']) | map(attribute='value') + | select('number') | list | sum }} + err_detail: >- + {{ err_total | string ~ ' error(s), ' ~ drop_total | string ~ ' drop(s) across ' + ~ (up | map(attribute='name') | join(', ')) }} + +- name: Assess — DNS resolver redundancy + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'network.resolvers', + 'title': 'Configured DNS upstreams (single upstream is a known SPOF here)', + 'status': ('PASS' if (ns4 | length) >= (node_health_thresholds.min_nameservers | int) else 'WARN'), + 'detail': ns_detail }] }}" + vars: + ansible_become: false + ns: "{{ node_health_facts.network.nameservers }}" + # Counted PER FAMILY on purpose. The outage this check exists for was a single IPv4 + # upstream; IPv6 upstreams do not cover for it unless the node actually has working + # IPv6 egress, which this role cannot prove from sysfs alone. So the verdict keys off + # the IPv4 count and the detail shows the full picture. + ns4: "{{ ns | select('search', '^[0-9.]+$') | list }}" + ns6: "{{ ns | reject('search', '^[0-9.]+$') | list }}" + ns_detail: >- + {{ (ns | length) | string ~ ' nameserver(s) — IPv4: ' + ~ (ns4 | join(', ') | default('none', true)) + ~ '; IPv6: ' ~ (ns6 | join(', ') | default('none', true)) + ~ ('. Only ' ~ (ns4 | length) | string ~ ' IPv4 upstream: a single IPv4 upstream + already broke containerd image pulls and CoreDNS forwarding on dev, and the + fix is stranded on fix/node-dns-resilient-upstream (DECISIONS-LOG + 2026-08-31, finding 5). IPv6 upstreams only compensate if the node has + working IPv6 egress, which this check cannot verify.' + if (ns4 | length) < (node_health_thresholds.min_nameservers | int) else '') }} + +# --- OS / firmware ------------------------------------------------------------------ +- name: Assess — pending reboot + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'os.reboot_required', + 'title': 'Pending reboot flag', + 'status': ('WARN' if node_health_facts.os.reboot_required | bool else 'PASS'), + 'detail': ('/run/reboot-required present — a package upgrade needs a restart' + if node_health_facts.os.reboot_required | bool + else 'no reboot flagged') }] }}" + vars: + ansible_become: false + +- name: Assess — OS, kernel and uptime + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'os.inventory', + 'title': 'OS, kernel and uptime', + 'status': 'INFO', + 'detail': os_detail }] }}" + vars: + ansible_become: false + o: "{{ node_health_facts.os }}" + os_detail: >- + {{ (o.pretty_name | default('unknown OS')) ~ ', kernel ' ~ (o.kernel | default('?')) + ~ ', up ' ~ ((o.uptime_seconds | float / 86400) | round(2) | string) ~ ' day(s)' }} + +- name: Assess — firmware / DMI inventory + ansible.builtin.set_fact: + node_health_checks: "{{ node_health_checks + [{ + 'id': 'firmware.inventory', + 'title': 'Chassis, board and BIOS firmware', + 'status': ('INFO' if node_health_facts.dmi.present | bool else 'SKIP'), + 'detail': dmi_detail }] }}" + vars: + ansible_become: false + d: "{{ node_health_facts.dmi }}" + dmi_detail: >- + {{ 'no DMI table exposed (typical for some VMs/containers)' if not (d.present | bool) + else ((d.sys_vendor | default('?')) ~ ' ' ~ (d.product_name | default('?')) + ~ ' / board ' ~ (d.board_name | default('?')) + ~ ' / BIOS ' ~ (d.bios_vendor | default('?')) ~ ' ' + ~ (d.bios_version | default('?')) ~ ' (' ~ (d.bios_date | default('?')) ~ ')' + ~ (' — serial numbers not read (root-only DMI fields)' + if d.product_serial is none else '')) }} diff --git a/infra/ansible/roles/node_health/tasks/collect_smart.yml b/infra/ansible/roles/node_health/tasks/collect_smart.yml new file mode 100644 index 0000000..ccabe84 --- /dev/null +++ b/infra/ansible/roles/node_health/tasks/collect_smart.yml @@ -0,0 +1,83 @@ +--- +# SMART — the ONLY part of this role that needs root, and the highest-value part. +# +# Reached only when probe_privilege.yml proved escalation works AND the collector found a +# smartctl binary. Both conditions are checked by the caller (main.yml); if either is +# false the SMART checks SKIP with the precise reason rather than silently passing. +# +# smartmontools is NOT installed from here. Package installation is node_baseline's job +# (node_baseline_packages carries smartmontools) — operator ruling 2026-08-31, the same +# click-ops-never principle as the rest of the repo. A health check must be read-only and +# side-effect-free, so this role only ever DETECTS the tool. +# +# `--nocheck=standby` (defaults/main.yml) means a sleeping disk is never spun up just to +# be measured. smartctl's exit status is a BITMASK (bit 0 = usage error, bit 1 = open +# failed, bit 3 = disk failing, ...), not a simple success/failure, so `failed_when: false` +# is correct here: a non-zero exit still carries a useful JSON payload, and bit 3 in +# particular is a finding, not an error to abort on. + +- name: Read SMART data for each physical disk + ansible.builtin.command: + cmd: "{{ node_health_facts.tools.smartctl }} {{ node_health_smartctl_args }} {{ item.path }}" + become: true + vars: + ansible_become: true + loop: "{{ node_health_facts.storage.disks }}" + loop_control: + label: "{{ item.path }}" + register: node_health_smart_raw + changed_when: false + failed_when: false + +- name: Parse SMART payloads into per-disk facts + ansible.builtin.set_fact: + node_health_smart: "{{ node_health_smart + [smart_disk] }}" + loop: "{{ node_health_smart_raw.results | default([]) }}" + loop_control: + loop_var: smart_result + label: "{{ smart_result.item.path }}" + vars: + ansible_become: false + # An unopenable device (virtio in a VM, a USB bridge without passthrough) yields no + # stdout at all — default to an empty object so parsing never explodes on a VM. + smart_json: "{{ (smart_result.stdout | default('') | trim) | default('{}', true) | from_json }}" + smart_attrs: "{{ smart_json.ata_smart_attributes.table | default([]) }}" + smart_nvme: "{{ smart_json.nvme_smart_health_information_log | default({}) }}" + # Normalised ("value") wear indicators, in vendor-preference order. Micron/Crucial use + # 202 Percent_Lifetime_Remaining; many others use 231 SSD_Life_Left or 233. The RAW + # value is kept alongside in `attributes` so nothing is hidden by this normalisation. + smart_life_202: "{{ smart_attrs | selectattr('id', 'equalto', 202) | map(attribute='value') | first | default(none) }}" + smart_life_231: "{{ smart_attrs | selectattr('id', 'equalto', 231) | map(attribute='value') | first | default(none) }}" + smart_life_233: "{{ smart_attrs | selectattr('id', 'equalto', 233) | map(attribute='value') | first | default(none) }}" + smart_disk: + path: "{{ smart_result.item.path }}" + # smartctl ran, but did it understand the device? + available: "{{ (smart_json.smart_status is defined) or (smart_nvme | length > 0) }}" + exit_status: "{{ smart_json.smartctl.exit_status | default(smart_result.rc) }}" + messages: "{{ smart_json.smartctl.messages | default([]) | map(attribute='string') | list }}" + protocol: "{{ smart_json.device.protocol | default(none) }}" + model: "{{ smart_json.model_name | default(none) }}" + serial: "{{ smart_json.serial_number | default(none) }}" + firmware: "{{ smart_json.firmware_version | default(none) }}" + # The headline: the drive's own overall self-assessment. + passed: "{{ smart_json.smart_status.passed | default(none) }}" + power_on_hours: "{{ smart_json.power_on_time.hours | default(none) }}" + power_cycles: "{{ smart_json.power_cycle_count | default(none) }}" + temperature_c: "{{ smart_json.temperature.current | default(none) }}" + reallocated_sectors: "{{ smart_attrs | selectattr('id', 'equalto', 5) | map(attribute='raw.value') | first | default(none) }}" + pending_sectors: "{{ smart_attrs | selectattr('id', 'equalto', 197) | map(attribute='raw.value') | first | default(none) }}" + offline_uncorrectable: "{{ smart_attrs | selectattr('id', 'equalto', 198) | map(attribute='raw.value') | first | default(none) }}" + crc_errors: "{{ smart_attrs | selectattr('id', 'equalto', 199) | map(attribute='raw.value') | first | default(none) }}" + reported_uncorrect: "{{ smart_attrs | selectattr('id', 'equalto', 187) | map(attribute='raw.value') | first | default(none) }}" + host_writes_raw: "{{ smart_attrs | selectattr('id', 'equalto', 241) | map(attribute='raw.value') | first | default(none) }}" + # NVMe reports wear directly as "percentage_used"; ATA SSDs report a countdown. + lifetime_remaining_pct: >- + {{ (100 - smart_nvme.percentage_used) if (smart_nvme.percentage_used is defined) + else (smart_life_202 if smart_life_202 is not none + else (smart_life_231 if smart_life_231 is not none else smart_life_233)) }} + nvme_media_errors: "{{ smart_nvme.media_errors | default(none) }}" + nvme_critical_warning: "{{ smart_nvme.critical_warning | default(none) }}" + # Every attribute the operator asked to see, verbatim, names and all — vendors + # disagree on naming, so the checks key off IDs and the report shows the drive's + # own labels. + attributes: "{{ smart_attrs | selectattr('id', 'in', node_health_smart_attribute_ids) | list }}" diff --git a/infra/ansible/roles/node_health/tasks/collect_unprivileged.yml b/infra/ansible/roles/node_health/tasks/collect_unprivileged.yml new file mode 100644 index 0000000..2b645d6 --- /dev/null +++ b/infra/ansible/roles/node_health/tasks/collect_unprivileged.yml @@ -0,0 +1,30 @@ +--- +# Everything obtainable without root, in ONE round trip. See +# files/node_health_collect.py for the full field-by-field rationale — including why the +# collector is a script rather than 40 sysfs `cat` tasks, and why unreadable fields come +# back as null + a recorded reason instead of an empty string. +# +# NOTE: the role runs with gather_facts irrelevant — the collector supersedes +# ansible.builtin.setup for this purpose (setup exposes no thermal zones, no EDAC, no CPU +# throttle counters, no PSI, no per-disk SCSI error counters, no per-NIC error stats), so +# there is exactly one source of truth behind both the summary and the JSON artifact. + +- name: Run the unprivileged node health collector + ansible.builtin.script: + cmd: node_health_collect.py + executable: /usr/bin/python3 + register: node_health_collect_raw + changed_when: false # strictly read-only; it can never change the node + +- name: Parse the collector payload + ansible.builtin.set_fact: + node_health_facts: "{{ node_health_collect_raw.stdout | from_json }}" + +- name: Assert the collector schema is the one this role understands + ansible.builtin.assert: + that: + - node_health_facts.schema == 'node-health/1' + fail_msg: >- + Collector returned schema {{ node_health_facts.schema | default('') }}; + this role understands node-health/1. Role and collector are out of step. + quiet: true diff --git a/infra/ansible/roles/node_health/tasks/main.yml b/infra/ansible/roles/node_health/tasks/main.yml new file mode 100644 index 0000000..a57ffc0 --- /dev/null +++ b/infra/ansible/roles/node_health/tasks/main.yml @@ -0,0 +1,60 @@ +--- +# node_health — read-only hardware health inspection. See defaults/main.yml for the why. +# +# ORDER OF OPERATIONS +# 1. Collect everything readable WITHOUT root (one script, one round trip). +# 2. Probe whether privilege escalation actually works on this host. +# 3. Only if it does AND smartmontools is installed: read SMART. +# 4. Turn facts into a list of explicit PASS/WARN/FAIL/SKIP checks. +# 5. Render the human summary + write the JSON artifact. +# +# BECOME HANDLING — the subtle bit, read before editing. +# `ansible_become` set as an inventory/group var (which is exactly what +# inventory/group_vars/k3s_prod/vars.yml does for prod) is a CONNECTION VARIABLE, and +# connection variables OUTRANK the `become:` play/task keyword. Verified on this repo's +# ansible-core 2.20: a play or task saying `become: false` still tries to sudo, and on +# prod (no NOPASSWD, vaulted password) dies with "Missing sudo password" before the +# module ever runs. So the unprivileged tasks here must override the VARIABLE, not the +# keyword — hence `vars: ansible_become: false` on the include below. That makes the role +# self-contained: it behaves identically whatever the calling play sets. +# +# Equally verified: `failed_when: false` does NOT catch a missing-sudo-password error +# (it is raised by the become plugin before the task's failure expression is evaluated). +# `ignore_errors: true` DOES, and still populates the registered variable. That is why +# the escalation probe below uses ignore_errors — anything else turns "no vault yet" into +# a hard, misleading play failure. + +- name: Initialise the health record + ansible.builtin.set_fact: + node_health_checks: [] + node_health_smart: [] + node_health_root_available: false + node_health_root_skip_reason: "privileged checks not attempted (node_health_try_privileged=false)" + vars: + ansible_become: false + +- name: Collect unprivileged hardware facts + ansible.builtin.include_tasks: collect_unprivileged.yml + # Applies to every task in the included file (dynamic includes pass their vars down). + vars: + ansible_become: false + +- name: Probe privilege escalation + ansible.builtin.include_tasks: probe_privilege.yml + when: node_health_try_privileged | bool + +- name: Read SMART data (privileged) + ansible.builtin.include_tasks: collect_smart.yml + when: + - node_health_root_available | bool + - node_health_facts.tools.smartctl is truthy + +- name: Assess collected facts into explicit checks + ansible.builtin.include_tasks: assess.yml + vars: + ansible_become: false + +- name: Report (human summary + JSON artifact) + ansible.builtin.include_tasks: report.yml + vars: + ansible_become: false diff --git a/infra/ansible/roles/node_health/tasks/probe_privilege.yml b/infra/ansible/roles/node_health/tasks/probe_privilege.yml new file mode 100644 index 0000000..ad9a71f --- /dev/null +++ b/infra/ansible/roles/node_health/tasks/probe_privilege.yml @@ -0,0 +1,35 @@ +--- +# Does `become` actually work on this host, right now? +# +# We must know BEFORE attempting SMART, because a failed escalation is a hard task error +# (see the become note in main.yml): `failed_when: false` cannot swallow it, only +# `ignore_errors` can. Probing once with a trivial, side-effect-free command means the +# rest of the role can branch on a boolean instead of littering ignore_errors everywhere — +# and lets the report state the exact reason a check was skipped. +# +# On prod today this probe is EXPECTED to fail: prod deliberately has no NOPASSWD sudo and +# its ansible_become_password lives in an ansible-vault file that does not exist yet +# (infra/PROD-PROVISIONING.md §5, DECISIONS-LOG 2026-08-31 finding 3). That is a SKIP, not +# a defect, and the summary says so in those words. + +- name: Probe whether privilege escalation is available + ansible.builtin.command: + cmd: id -u + become: true + vars: + ansible_become: true + register: node_health_root_probe + changed_when: false + ignore_errors: true + +- name: Record the escalation result + ansible.builtin.set_fact: + node_health_root_available: "{{ (node_health_root_probe.stdout | default('') | trim) == '0' }}" + node_health_root_skip_reason: >- + {{ 'privilege escalation OK' + if (node_health_root_probe.stdout | default('') | trim) == '0' + else 'become unavailable: ' ~ (node_health_root_probe.msg | default('unknown error')) + ~ ' (prod uses a vaulted ansible_become_password —' + ~ ' see infra/PROD-PROVISIONING.md §5)' }} + vars: + ansible_become: false diff --git a/infra/ansible/roles/node_health/tasks/report.yml b/infra/ansible/roles/node_health/tasks/report.yml new file mode 100644 index 0000000..7cb1bca --- /dev/null +++ b/infra/ansible/roles/node_health/tasks/report.yml @@ -0,0 +1,84 @@ +--- +# Two outputs, deliberately: +# +# 1. A human summary the operator can read at a glance (printed in the play output AND +# written next to the JSON, so it survives the terminal scrollback). +# 2. A structured JSON artifact — the machine-readable record, and the thing a future +# R-OBS-04 exporter or a prod-readiness gate can consume without re-parsing prose. +# +# Artifacts follow the repo's existing convention (conformance/_artifacts/): generated +# output under a `_artifacts/` directory that .gitignore excludes. Never committed — they +# are regenerated on every run and they carry drive serial numbers. + +- name: Count check outcomes + ansible.builtin.set_fact: + node_health_counts: + FAIL: "{{ node_health_checks | selectattr('status', 'equalto', 'FAIL') | list | length }}" + WARN: "{{ node_health_checks | selectattr('status', 'equalto', 'WARN') | list | length }}" + PASS: "{{ node_health_checks | selectattr('status', 'equalto', 'PASS') | list | length }}" + SKIP: "{{ node_health_checks | selectattr('status', 'equalto', 'SKIP') | list | length }}" + INFO: "{{ node_health_checks | selectattr('status', 'equalto', 'INFO') | list | length }}" + +# Rendered once, then used for BOTH the on-screen report and the .txt artifact, so the +# two can never drift. +- name: Render the human summary + ansible.builtin.set_fact: + node_health_summary: "{{ lookup('ansible.builtin.template', 'summary.txt.j2') }}" + +- name: Show the summary + ansible.builtin.debug: + msg: "{{ node_health_summary.split('\n') }}" + when: node_health_show_summary | bool + +- name: Ensure the artifact directory exists (on the controller) + ansible.builtin.file: + path: "{{ node_health_artifact_dir }}" + state: directory + mode: "0755" + delegate_to: localhost + become: false + when: node_health_write_artifacts | bool + +- name: Write the structured health record (JSON) + ansible.builtin.copy: + dest: "{{ node_health_artifact_dir }}/{{ inventory_hostname }}.json" + mode: "0644" + content: "{{ node_health_record | to_nice_json }}" + delegate_to: localhost + become: false + when: node_health_write_artifacts | bool + vars: + node_health_record: + schema: node-health/1 + host: "{{ inventory_hostname }}" + address: "{{ ansible_host | default(inventory_hostname) }}" + collected_at: "{{ node_health_facts.collected_at }}" + privileged: + available: "{{ node_health_root_available }}" + reason: "{{ node_health_root_skip_reason }}" + counts: "{{ node_health_counts }}" + checks: "{{ node_health_checks }}" + smart: "{{ node_health_smart }}" + facts: "{{ node_health_facts }}" + +- name: Write the human summary alongside it + ansible.builtin.copy: + dest: "{{ node_health_artifact_dir }}/{{ inventory_hostname }}.txt" + mode: "0644" + content: "{{ node_health_summary }}" + delegate_to: localhost + become: false + when: node_health_write_artifacts | bool + +# Report-only by default (see defaults/main.yml): a health check that aborts the play +# hides the other twenty findings. Opt in when using this role as a pre-build gate. +- name: Fail the play on health findings (opt-in) + ansible.builtin.assert: + that: + - (node_health_counts.FAIL | int) == 0 + - not (node_health_fail_on_warn | bool) or (node_health_counts.WARN | int) == 0 + fail_msg: >- + node_health: {{ node_health_counts.FAIL }} FAIL / {{ node_health_counts.WARN }} WARN + on {{ inventory_hostname }} — see {{ node_health_artifact_dir }}/{{ inventory_hostname }}.txt + quiet: true + when: node_health_fail_on_fail | bool diff --git a/infra/ansible/roles/node_health/templates/summary.txt.j2 b/infra/ansible/roles/node_health/templates/summary.txt.j2 new file mode 100644 index 0000000..248f98b --- /dev/null +++ b/infra/ansible/roles/node_health/templates/summary.txt.j2 @@ -0,0 +1,147 @@ +{# =========================================================================== + node_health human summary. Rendered ONCE (tasks/report.yml) and used for both + the on-screen report and the .txt artifact, so the two can never drift. + + Detail strings are normalised with regex_replace('\s+',' ') because the check + details are authored as YAML folded scalars in assess.yml and can carry stray + newlines/indentation from that folding. + =========================================================================== #} +{% macro fmt(text, width=74, pad=10) -%} +{{ text | regex_replace('\s+', ' ') | trim | wordwrap(width) | indent(pad, first=True) }} +{%- endmacro %} +{% macro gib(n) -%} +{{ (n | int / 1073741824) | round(1) }} GiB +{%- endmacro %} +================================================================================ + NODE HEALTH — {{ inventory_hostname }} ({{ ansible_host | default('n/a') }}) + collected {{ node_health_facts.collected_at }} as uid {{ node_health_facts.collector_uid }} +================================================================================ + + VERDICT {{ node_health_counts.FAIL }} FAIL · {{ node_health_counts.WARN }} WARN · {{ node_health_counts.PASS }} PASS · {{ node_health_counts.SKIP }} SKIP · {{ node_health_counts.INFO }} INFO + {{ 'privileged checks RAN' if node_health_root_available | bool else 'privileged checks SKIPPED — root not available' }} + +-------------------------------------------------------------------------------- + CHECKS +-------------------------------------------------------------------------------- +{% for c in node_health_checks %} + [{{ '%-4s' | format(c.status) }}] {{ c.id }} — {{ c.title }} +{{ fmt(c.detail) }} +{% endfor %} + +-------------------------------------------------------------------------------- + MACHINE — {{ node_health_facts.dmi.sys_vendor | default('?') }} {{ node_health_facts.dmi.product_name | default('?') }} +-------------------------------------------------------------------------------- + OS {{ node_health_facts.os.pretty_name | default('?') }} + Kernel {{ node_health_facts.os.kernel | default('?') }} + Uptime {{ (node_health_facts.os.uptime_seconds | float / 86400) | round(2) }} days + BIOS {{ node_health_facts.dmi.bios_vendor | default('?') }} {{ node_health_facts.dmi.bios_version | default('?') }} ({{ node_health_facts.dmi.bios_date | default('?') }}){% if node_health_facts.dmi.ec_firmware_release %}, EC {{ node_health_facts.dmi.ec_firmware_release }}{% endif %} + + Board {{ node_health_facts.dmi.board_vendor | default('?') }} {{ node_health_facts.dmi.board_name | default('?') }} + CPU {{ node_health_facts.cpu.model | default('?') }} + {{ node_health_facts.cpu.cpus | default('?') }} logical / {{ node_health_facts.cpu.sockets | default('?') }} socket(s) x {{ node_health_facts.cpu.cores_per_socket | default('?') }} core(s) x {{ node_health_facts.cpu.threads_per_core | default('?') }} thread(s) + virtualisation: {{ node_health_facts.cpu.virtualization | default('none reported') }}, /dev/kvm={{ node_health_facts.cpu.kvm_device }} +{% if node_health_facts.cpu.scaling %} + scaling: {{ node_health_facts.cpu.scaling.driver }}/{{ node_health_facts.cpu.scaling.governor }}, {{ (node_health_facts.cpu.scaling.min_khz | int / 1000) | round | int }}–{{ (node_health_facts.cpu.scaling.max_khz | int / 1000) | round | int }} MHz (now {{ (node_health_facts.cpu.scaling.cur_khz | int / 1000) | round | int }} MHz) +{% endif %} + Memory {{ (node_health_facts.memory.mem_total_kb | int / 1048576) | round(1) }} GiB total, {{ (node_health_facts.memory.mem_available_kb | int / 1048576) | round(1) }} GiB available, swap {{ (node_health_facts.memory.swap_total_kb | int / 1048576) | round(2) }} GiB + ECC/EDAC {{ 'controllers: ' ~ (node_health_facts.memory.edac.controllers | map(attribute='name') | join(', ')) ~ ' CE=' ~ node_health_facts.memory.edac.ce_total | string ~ ' UE=' ~ node_health_facts.memory.edac.ue_total | string if node_health_facts.memory.edac.controllers_present else 'not exposed (no EDAC controller bound — ECC status UNKNOWN on this platform)' }} + Load {{ node_health_facts.cpu.loadavg | default([]) | join(' ') }} + +-------------------------------------------------------------------------------- + STORAGE +-------------------------------------------------------------------------------- +{% for d in node_health_facts.storage.disks %} + {{ d.path }} {{ gib(d.size_bytes) }} {{ 'rotational' if d.rotational else 'non-rotating' }} {{ d.transport | default('?', true) }}{% if d.fstype %} whole-device {{ d.fstype }}{% endif %}{% if d.mountpoints %} mounted at {{ d.mountpoints | join(',') }}{% endif %} + + model {{ d.model | default('(none reported)', true) }}{% if d.firmware_rev %} fw {{ d.firmware_rev }}{% endif %}{% if d.serial %} s/n {{ d.serial }}{% endif %} + +{% if d.scsi_counters %} + kernel I/O completed={{ d.scsi_counters.iodone_cnt }} errors={{ d.scsi_counters.ioerr_cnt }} timeouts={{ d.scsi_counters.iotmo_cnt }} +{% endif %} +{% set sm = (node_health_smart | selectattr('path', 'equalto', d.path) | list | first) | default(none) %} +{% if sm and sm.available %} + SMART overall-health={{ 'PASSED' if sm.passed else 'FAILED' }} power_on_hours={{ sm.power_on_hours | default('?') }} power_cycles={{ sm.power_cycles | default('?') }} + lifetime_remaining={{ sm.lifetime_remaining_pct | default('?') }}% temp={{ sm.temperature_c | default('?') }}C + reallocated={{ sm.reallocated_sectors | default('?') }} pending={{ sm.pending_sectors | default('?') }} offline_uncorrectable={{ sm.offline_uncorrectable | default('?') }} crc_errors={{ sm.crc_errors | default('?') }} +{% for a in sm.attributes | default([]) %} + [{{ '%3d' | format(a.id) }}] {{ '%-26s' | format(a.name) }} value={{ a.value }} worst={{ a.worst }} thresh={{ a.thresh }} raw={{ a.raw.string | default(a.raw.value) }} +{% endfor %} +{% else %} + SMART NOT READ — {{ (node_health_root_skip_reason if not node_health_root_available else ('smartmontools not installed on this node (ships with roles/node_baseline; this role never installs packages)' if not node_health_facts.tools.smartctl else 'device exposes no SMART data')) | regex_replace('\s+', ' ') | trim }} +{% endif %} +{% for p in d.partitions %} + {{ '%-12s' | format(p.kname) }} {{ '%9s' | format(gib(p.size_bytes)) }} {{ '%-6s' | format(p.fstype | default('-', true)) }} {{ (p.mountpoints[:2] | join(', ') ~ (' (+%d more)' | format(p.mountpoints | length - 2) if (p.mountpoints | length) > 2 else '')) if p.mountpoints else '(unmounted)' }} +{% endfor %} + +{% endfor %} +{% if node_health_facts.storage.attached_volumes | default([]) %} + ATTACHED VOLUMES (iSCSI/FC — Longhorn replicas etc., not hardware in this box) +{% for a in node_health_facts.storage.attached_volumes %} + {{ '%-10s' | format(a.path) }} {{ '%9s' | format(gib(a.size_bytes)) }} {{ a.transport }} {{ a.model | default('-', true) }} +{% endfor %} + +{% endif %} + MOUNTED FILESYSTEMS +{% for m in node_health_facts.storage.mounts | sort(attribute='mountpoint') %} + {{ '%-14s' | format(m.mountpoint) }} {{ '%-6s' | format(m.fstype) }} {{ '%9s' | format(gib(m.size_bytes)) }} total, {{ '%9s' | format(gib(m.avail_bytes)) }} free ({{ m.used_pct }}% used, inodes {{ (m.inodes_used_pct | string ~ '%') if m.inodes_used_pct is not none else 'n/a' }}) +{% endfor %} + +-------------------------------------------------------------------------------- + THERMAL +-------------------------------------------------------------------------------- +{% for z in node_health_facts.thermal.zones %} + zone {{ '%-18s' | format(z.type | default(z.zone)) }} {{ (z.temp_c | string ~ 'C') if z.temp_c is not none else 'unreadable (sensor returned no data)' }}{% if z.critical_c %} (critical {{ z.critical_c }}C){% endif %} + +{% endfor %} +{% for chip in node_health_facts.thermal.hwmon %} +{% for t in chip.temps %} + {{ '%-16s' | format(chip.name | default('hwmon')) }} {{ '%-22s' | format(t.label) }} {{ (t.temp_c | string ~ 'C') if t.temp_c is not none else 'unreadable (sensor returned no data)' }}{% if t.crit_c %} (crit {{ t.crit_c }}C){% endif %} + +{% endfor %} +{% for f in chip.fans %} + {{ '%-16s' | format(chip.name | default('hwmon')) }} {{ '%-22s' | format(f.label) }} {{ f.rpm }} rpm +{% endfor %} +{% endfor %} + +-------------------------------------------------------------------------------- + PRESSURE STALL (/proc/pressure — % of wall time stalled) +-------------------------------------------------------------------------------- +{% for res in ['cpu', 'memory', 'io'] %} +{% set p = node_health_facts.pressure[res] %} +{% if p %} + {{ '%-7s' | format(res) }} some avg10={{ p.some.avg10 }} avg60={{ p.some.avg60 }} avg300={{ p.some.avg300 }}{% if p.full is defined %} | full avg60={{ p.full.avg60 }}{% endif %} + +{% else %} + {{ '%-7s' | format(res) }} unavailable +{% endif %} +{% endfor %} + +-------------------------------------------------------------------------------- + NETWORK +-------------------------------------------------------------------------------- +{% for n in node_health_facts.network.interfaces %} + {{ '%-10s' | format(n.name) }} {{ '%-6s' | format(n.operstate) }} {{ '%-8s' | format((n.speed_mbps | string ~ 'Mb/s') if n.speed_mbps else 'no-link') }} {{ '%-9s' | format(n.duplex | default('-')) }} mtu {{ n.mtu }} driver {{ n.driver | default('-') }} + rx {{ n.statistics.rx_packets | default('?') }} pkts / errors {{ n.statistics.rx_errors | default('?') }} / dropped {{ n.statistics.rx_dropped | default('?') }} + tx {{ n.statistics.tx_packets | default('?') }} pkts / errors {{ n.statistics.tx_errors | default('?') }} / dropped {{ n.statistics.tx_dropped | default('?') }} +{% endfor %} + nameservers {{ node_health_facts.network.nameservers | join(', ') | default('none configured', true) }} + +-------------------------------------------------------------------------------- + NOT READ (and why) — these are UNKNOWNS, not passes +-------------------------------------------------------------------------------- +{% for c in node_health_checks if c.status == 'SKIP' %} + - {{ c.id }}: {{ c.detail | regex_replace('\s+', ' ') | trim | wordwrap(70) | indent(3) }} +{% else %} + (nothing skipped) +{% endfor %} +{% if node_health_facts.unreadable %} + + Unreadable paths reported by the collector: +{% for u in node_health_facts.unreadable %} + - {{ u.path }} — {{ u.reason }} +{% endfor %} +{% endif %} + + tools present: {% for t, p in node_health_facts.tools.items() %}{{ t }}={{ p | default('MISSING', true) }}{{ ', ' if not loop.last }}{% endfor %} + +================================================================================