---
# node_health — READ-ONLY hardware/health inspection of a node (bare metal or VM).
#
# Sibling of roles/cluster_verify: cluster_verify answers "is the CLUSTER healthy",
# node_health answers "is the HARDWARE under it healthy". Same posture — it inspects and
# reports, it never changes the node. Deliberately NOT part of node_baseline: baseline
# installs and configures, health only looks. That separation is why this role must never
# `apt install` anything (smartmontools is declared in node_baseline_packages; if it is
# missing here the SMART checks SKIP with a reason, they do not self-heal).
#
# Serves R-ENV-01 (reproducible node provisioning — you cannot certify a node you have not
# measured), R-ENV-03 (bare-metal parity) and R-OBS-04 (node-level observability).

# --- Reporting ---------------------------------------------------------------------
# Artifacts land under infra/ansible/_artifacts/node-health/, mirroring the repo's
# conformance/_artifacts/ convention: generated output, regenerated on every run, listed
# in .gitignore, never committed (it also carries drive serial numbers).
node_health_artifact_dir: "{{ playbook_dir }}/../_artifacts/node-health"
node_health_write_artifacts: true
# Print the human summary to the play output as well as writing it to the artifact dir.
node_health_show_summary: true
# Report-only by default: a health check that aborts the play hides the other 20 findings.
# Set true (e.g. in CI or a pre-build gate) to make any FAIL check end the play non-zero.
node_health_fail_on_fail: false
# Same, one step stricter: also fail on WARN.
node_health_fail_on_warn: false

# --- Root-only checks ---------------------------------------------------------------
# Everything this role can read WITHOUT root, it reads without root. Only SMART needs
# escalation (the ATA/NVMe passthrough ioctls are root-only). The role PROBES become
# first and, if escalation is unavailable (e.g. prod's vaulted become password is not
# present — infra/PROD-PROVISIONING.md §5), those checks report SKIPPED with the exact
# reason. Never a hard failure, and never a fake PASS: a skip is honest.
node_health_try_privileged: true
# --nocheck=standby: never spin up a sleeping disk just to read it. Read-only hygiene.
node_health_smartctl_args: "--json=c --all --nocheck=standby"

# --- Graphics / render device ---------------------------------------------------------
# The DRM driver this site EXPECTS to be bound to its display device. Measured on all three
# prod nodes 2026-09-30: `i915` on the CoffeeLake-S GT2 [UHD Graphics 630] iGPU of every HP
# EliteDesk 800 G5 Mini. The newer `xe` module is present on disk as xe.ko.xz but is bound to
# nothing, and that difference matters enough to surface: the two drivers do not share a
# userspace media stack (a libva/iHD build that works against i915 is not automatically the
# right one for xe), so a silent switch would change Plex's transcoding path. A site on AMD
# or NVIDIA sets this to `amdgpu` / `nouveau` / `nvidia` rather than editing task logic.
#
# WHAT IS DELIBERATELY *NOT* A SETTING HERE: the numeric GID of the render node's owning
# group. `render` is a systemd DYNAMIC allocation (systemd-sysusers), not a Debian static
# one, so a rebuilt node may legitimately land on a different number — it was 992 on all
# three prod nodes on 2026-09-30, and an assertion against that literal would be a false
# alarm waiting to happen. Plex's own container entrypoint stats the device and joins
# whichever group owns it, so drift genuinely is not a fault. The observed group name and GID
# are therefore REPORTED in the check detail and never graded.
node_health_graphics_expected_driver: i915

# --- Thresholds ---------------------------------------------------------------------
# All verdicts come from here, so a site can retune without touching task logic.
node_health_thresholds:
  # Filesystems
  fs_used_pct_warn: 80
  fs_used_pct_fail: 90
  inode_used_pct_warn: 80
  # Memory (percentage of MemTotal still available)
  mem_available_pct_warn: 15
  mem_available_pct_fail: 5
  # --- MEMORY THE MACHINE HAS THAT THE OS WILL NEVER USE ----------------------------
  # Two independent measurements of the same question, both fed by memory.physical_total:
  #
  #   unmanaged  /proc/zoneinfo present pages MINUS managed pages — RAM the kernel's own
  #              memory map says exists, that never reached the allocator. Needs no root
  #              and no tools, works identically on metal and in a VM.
  #   shortfall  MemTotal against the DIMM capacity SMBIOS (dmidecode -t 17) says is
  #              installed. Needs root, and only bare metal reports it. Independent of the
  #              kernel's memory map entirely, so it also catches RAM the firmware never
  #              handed the kernel in the first place.
  #
  # MEASURED BASELINES for the unmanaged figure, so these are tuned against numbers and
  # not taste: 2.0% k3s-prod-1 (32 GiB metal), 2.1% (32 GiB Fedora workstation), 2.2%
  # k3s-dev-1 (24 GiB VM) — versus 83.4% on k3s-prod-2 in the EFI-memattr failure state.
  # The healthy figure is the kernel image, memmap and firmware reservations; it is a
  # near-constant few hundred MB, so a small machine sits higher in PERCENT terms and 10%
  # keeps clearance for that. There is no plausible tuning of a threshold in this band
  # that changes any of those five verdicts.
  #
  # The DIMM shortfall runs a little higher than the unmanaged figure on the same node
  # (2.7% on k3s-prod-1) because installed capacity also covers e820-reserved regions and
  # the 64 MiB of graphics stolen memory — legitimately missing RAM that was never the
  # kernel's to manage. Same bands: the signal being caught is 83%, not 3%.
  mem_unmanaged_pct_warn: 10
  mem_unmanaged_pct_fail: 25
  mem_dimm_shortfall_pct_warn: 10
  mem_dimm_shortfall_pct_fail: 25
  # Temperatures (deg C, any sensor)
  temp_warn_c: 70
  temp_fail_c: 85
  # Pressure-stall: "some" avg60, percent of wall time stalled. Dev learned to care about
  # CPU PSI specifically (it shows saturation long before load average makes it obvious).
  psi_some_avg60_warn: 10.0
  psi_some_avg60_fail: 40.0
  # Network
  nic_min_speed_mbps: 1000
  # A single upstream resolver is a known SPOF on this project — it broke containerd
  # image pulls and CoreDNS forwarding on dev (DECISIONS-LOG 2026-08-31, finding 5).
  # Counted against the node's REAL upstreams: where systemd-resolved is in use the
  # collector looks behind the 127.0.0.53 stub rather than counting it as one server.
  # The fix side of this is the `dns`-tagged block in roles/node_baseline.
  min_nameservers: 2
  # --- SMART (only evaluated when smartctl actually ran) -----------------------------
  #
  # WHY THESE THRESHOLDS AND NOT "reallocated sectors" ANY MORE.
  # The first real run of this role FAILED k3s-prod-1's Crucial/Micron C400 on
  # `reallocated=61440`, read as a plain bad-sector count. That verdict was WRONG, and it
  # was wrong for a structural reason, not a tuning one: SMART *raw* values are vendor
  # defined. smartctl(8) says so outright -- "starting with ATA/ATAPI-4, revision 4, the
  # meaning of these Attribute fields has been made entirely vendor-specific". Micron's own
  # TN-FD-20 (SMART Attributes for the C400 SSD) defines ID 5 not as a sector count but as
  # "Reallocated Block Count", raw = "the grown NAND Flash bad-block count multiplied by the
  # Block_Sector_Count, which is 256 x 8 for 64Gb NAND ... or 256 x 16 for 128Gb NAND (256GB
  # and 512GB)". 256 x 16 = 4096, and 61440 / 4096 = 15 -- corroborated independently by
  # ID 170 Grown_Failing_Block_Ct = 15 and ID 196 Reallocated_Event_Count = 15. Fifteen
  # grown bad blocks, not sixty-one thousand bad sectors.
  #
  # So no threshold below is ever applied to attribute 5's raw value. The verdict comes from
  # the signals smartctl itself defines as vendor-INDEPENDENT:
  #   * the drive's own overall-health self-assessment (smart_status.passed);
  #   * NORMALISED value vs THRESHOLD, i.e. smartctl's when_failed ("now"/"past"), which is
  #     the manufacturer's OWN trip point for that attribute on that drive;
  #   * the ATA-standard error counters whose raw meaning is a plain event count everywhere:
  #     197 pending, 198 offline-uncorrectable, 187 reported-uncorrectable, 199 CRC;
  #   * NVMe's structured health log (critical_warning, media_errors, percentage_used).
  # Vendor raw counts are still collected and printed -- they are real information -- but
  # they can only ever raise a WARN with the number shown, never a FAIL on their own.
  #
  # Hard-error counters (197/198/187 raw, NVMe media_errors). A plain event count on every
  # drive that implements them. NOTE the converse is not evidence: some drives (the C400
  # among them, per TN-FD-20) hard-code 197 to 0 because reallocation happens on the fly, so
  # "pending = 0" proves nothing. Nonzero is a finding; zero is not a clean bill of health.
  smart_pending_warn: 1
  smart_pending_fail: 10
  smart_uncorrectable_warn: 1
  smart_uncorrectable_fail: 10
  # 199 UDMA_CRC_Error_Count -- SATA link/cable integrity, not media. WARN only: it is a
  # lifetime count that never resets, so a historic cable reseat should not FAIL a node.
  smart_crc_warn: 1
  # Wear, from the NORMALISED lifetime indicator (ATA 202/231/233 value, or NVMe
  # percentage_used) -- a percentage by definition, so portable in a way raw wear counts
  # are not.  ..._remaining_* are the "this drive is nearly used up" bands; ..._used_pct_warn
  # is the softer "this is not a fresh drive, know that before you put etcd on it" notice
  # that PROD-PROVISIONING.md Section 2 actually needs.
  smart_lifetime_remaining_warn_pct: 20
  smart_lifetime_remaining_fail_pct: 10
  smart_life_used_pct_warn: 25
  # Reallocation ACTIVITY. 196 Reallocated_Event_Count is an ATA-standard *event* count
  # (one per remap operation) and is the portable stand-in for attribute 5's unportable raw.
  # WARN-only by design: remapping is what a drive is supposed to do, and how many events
  # are "too many" is drive-specific -- the manufacturer already encodes that answer in
  # attribute 5's normalised threshold, which the when_failed check above honours.
  smart_realloc_events_warn: 1
  # Endurance-stress counters, matched by the NAME smartctl's drive database resolved for
  # this model rather than by a bare vendor ID -- drivedb IS the portability layer. On a
  # drive smartctl does not recognise these simply do not match and nothing is claimed.
  smart_erase_fail_warn: 1
  smart_program_fail_warn: 1
  # Unexpected power loss on a consumer SSD with no power-loss protection: each one is a
  # chance of a torn/lost in-flight write. Directly relevant to etcd's fsync contract.
  smart_unexpected_power_loss_warn: 10
  smart_temp_warn_c: 55
  smart_temp_fail_c: 70
  # Power-on hours past which a consumer SSD is simply old. Informational — age alone is
  # not a failure, but it belongs in the operator's field of view when deciding whether a
  # drive may hold production state (PROD-PROVISIONING.md §2).
  smart_power_on_hours_notice: 40000

# ATA SMART attribute IDs surfaced verbatim in the report. Vendors disagree on names, so
# the report prints whatever the drive calls them and the checks key off the IDs.
# 170-174 and 189 are the Micron/Crucial (and, by convention, most SSD controllers')
# endurance and grown-bad-block counters; they are the CONTEXT that made the C400 verdict
# above provable, and leaving them out of the report is what let a single misread raw value
# stand unchallenged. 1 and 12 are included so the error-rate and power-cycle baselines are
# visible next to them.
node_health_smart_attribute_ids: [1, 5, 9, 12, 170, 171, 172, 173, 174, 177, 179, 181,
                                  182, 183, 184, 187, 188, 189, 190, 194, 195, 196, 197,
                                  198, 199, 202, 206, 231, 232, 233, 241, 242, 246]