home-network-k3s-impl/infra/ansible/roles/node_health/tasks/assess.yml

---
# Turn raw facts into explicit, named checks. Every check is one of:
#
#   PASS  measured, and within threshold
#   WARN  measured, outside the warn threshold — worth a human look
#   FAIL  measured, outside the fail threshold — do not put production state here
#   SKIP  NOT measured, with the reason attached (missing root, missing tool, sensor not
#         present on this hardware). A SKIP is an honest "unknown" and is NEVER rendered
#         as a pass — this repo's discipline is absolute on that point.
#   INFO  measured, no pass/fail semantics (inventory: firmware, uptime, CPU scaling)
#
# All thresholds come from node_health_thresholds (defaults/main.yml) so retuning never
# means editing logic here.

- name: Derive intermediate values used by several checks
  ansible.builtin.set_fact:
    # Every temperature the box exposes, from both thermal zones and hwmon chips.
    # `select('number')` drops any sensor that returned null (unreadable/absent).
    node_health_temps_c: >-
      {{ ((node_health_facts.thermal.zones | map(attribute='temp_c') | list)
          + (node_health_facts.thermal.hwmon | map(attribute='temps') | flatten
             | map(attribute='temp_c') | list)) | select('number') | list }}
    # Filesystems, worst-first by percentage used / inodes used.
    node_health_mounts_by_use: >-
      {{ node_health_facts.storage.mounts | selectattr('used_pct', 'number')
         | sort(attribute='used_pct') | reverse | list }}
    node_health_mounts_by_inode: >-
      {{ node_health_facts.storage.mounts | selectattr('inodes_used_pct', 'number')
         | sort(attribute='inodes_used_pct') | reverse | list }}
    # Physical NICs that are actually up. Virtual plumbing was already excluded by the
    # collector; `virtual` here catches bonds/VLANs the regex let through.
    node_health_nics_up: >-
      {{ node_health_facts.network.interfaces
         | selectattr('operstate', 'equalto', 'up') | list }}
    # Devices that could actually BECOME a dedicated Longhorn or etcd volume: no
    # partitions, no whole-device filesystem, nothing mounted from them. All three
    # conditions matter — node_baseline formats the data disks with no partition table at
    # all (mkfs straight onto /dev/vdb, mounted by LABEL), so "no partitions" alone would
    # nominate a node's live Longhorn disk for reformatting. iSCSI/FC volumes were already
    # split out by the collector for the same reason. On a single-disk node this list is
    # empty — precisely the finding that blocks PROD-PROVISIONING.md §2.
    node_health_free_devices: >-
      {{ node_health_facts.storage.disks
         | selectattr('partitions', 'equalto', [])
         | selectattr('fstype', 'none')
         | selectattr('mountpoints', 'equalto', [])
         | list }}
  vars:
    ansible_become: false

- name: Assess — privilege escalation (gates every root-only check below)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'access.privilege',
      'title': 'Privilege escalation (become) for root-only checks',
      'status': ('INFO' if node_health_root_available | bool else 'SKIP'),
      'detail': node_health_root_skip_reason }] }}"

# --- Storage ------------------------------------------------------------------------
- name: Assess — SMART per physical disk
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'storage.smart.' ~ item.kname,
      'title': 'SMART health — ' ~ item.path ~ ' (' ~ (item.model | default('unknown model')) ~ ')',
      'status': smart_status,
      'detail': smart_detail }] }}"
  loop: "{{ node_health_facts.storage.disks }}"
  loop_control:
    label: "{{ item.path }}"
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    disk_smart: "{{ (node_health_smart | selectattr('path', 'equalto', item.path) | list | first) | default({}) }}"
    # Sub-verdicts. Each is SKIP unless the number was actually read.
    s_health: >-
      {{ 'SKIP' if (disk_smart.passed is not defined or disk_smart.passed is none)
         else ('PASS' if disk_smart.passed else 'FAIL') }}
    s_realloc: >-
      {{ 'SKIP' if (disk_smart.reallocated_sectors is not defined or disk_smart.reallocated_sectors is not number)
         else ('FAIL' if disk_smart.reallocated_sectors | int >= th.smart_reallocated_fail
               else ('WARN' if disk_smart.reallocated_sectors | int >= th.smart_reallocated_warn else 'PASS')) }}
    s_pending: >-
      {{ 'SKIP' if (disk_smart.pending_sectors is not defined or disk_smart.pending_sectors is not number)
         else ('FAIL' if disk_smart.pending_sectors | int >= th.smart_pending_fail
               else ('WARN' if disk_smart.pending_sectors | int >= th.smart_pending_warn else 'PASS')) }}
    s_life: >-
      {{ 'SKIP' if (disk_smart.lifetime_remaining_pct is not defined or disk_smart.lifetime_remaining_pct is not number)
         else ('FAIL' if disk_smart.lifetime_remaining_pct | int <= th.smart_lifetime_remaining_fail_pct
               else ('WARN' if disk_smart.lifetime_remaining_pct | int <= th.smart_lifetime_remaining_warn_pct else 'PASS')) }}
    s_temp: >-
      {{ 'SKIP' if (disk_smart.temperature_c is not defined or disk_smart.temperature_c is not number)
         else ('FAIL' if disk_smart.temperature_c | int >= th.smart_temp_fail_c
               else ('WARN' if disk_smart.temperature_c | int >= th.smart_temp_warn_c else 'PASS')) }}
    # Worst sub-verdict wins. `select('in', ...)` preserves the order of the left-hand
    # list, so the LAST survivor is the most severe present.
    smart_status: >-
      {{ (['PASS', 'WARN', 'FAIL'] | select('in', [s_health, s_realloc, s_pending, s_life, s_temp]) | list | last)
         | default('SKIP', true) }}
    smart_detail: >-
      {{ ('not read — ' ~ node_health_root_skip_reason)
           if not (node_health_root_available | bool)
         else ('not read — smartmontools is not installed on this node; it ships with
                roles/node_baseline (node_baseline_packages), which has not been run here.
                This role never installs packages.')
           if not (node_health_facts.tools.smartctl is truthy)
         else ('smartctl ran but this device exposes no SMART data ('
                ~ (disk_smart.messages | default([]) | join('; ')) ~ ') — normal for virtio/VM disks')
           if not (disk_smart.available | default(false))
         else ('overall-health=' ~ ('PASSED' if disk_smart.passed else 'FAILED')
               ~ ', power_on_hours=' ~ (disk_smart.power_on_hours | default('?'))
               ~ ', lifetime_remaining=' ~ (disk_smart.lifetime_remaining_pct | default('?')) ~ '%'
               ~ ', reallocated=' ~ (disk_smart.reallocated_sectors | default('?'))
               ~ ', pending=' ~ (disk_smart.pending_sectors | default('?'))
               ~ ', temp=' ~ (disk_smart.temperature_c | default('?')) ~ 'C') }}

- name: Assess — kernel-visible I/O errors per disk (no root needed)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'storage.io_errors',
      'title': 'Kernel I/O error counters on physical disks (SCSI/ATA layer)',
      'status': io_status,
      'detail': io_detail }] }}"
  vars:
    ansible_become: false
    # ioerr_cnt/iotmo_cnt are readable WITHOUT root. They are not a substitute for SMART
    # (no wear, no power-on hours, no reallocated sectors) but they are a real signal: a
    # non-zero count means the kernel actually saw failed or timed-out commands on that
    # device since boot.
    counted: "{{ node_health_facts.storage.disks | selectattr('scsi_counters', 'mapping') | list }}"
    err_total: >-
      {{ (counted | map(attribute='scsi_counters.ioerr_cnt') | select('number') | list | sum)
         + (counted | map(attribute='scsi_counters.iotmo_cnt') | select('number') | list | sum) }}
    io_status: >-
      {{ 'SKIP' if (counted | length) == 0
         else ('PASS' if err_total | int == 0 else 'WARN') }}
    io_sum_done: "{{ counted | map(attribute='scsi_counters.iodone_cnt') | select('number') | list | sum }}"
    io_sum_err: "{{ counted | map(attribute='scsi_counters.ioerr_cnt') | select('number') | list | sum }}"
    io_sum_tmo: "{{ counted | map(attribute='scsi_counters.iotmo_cnt') | select('number') | list | sum }}"
    # Aggregate here; the STORAGE section of the summary prints the per-disk counters.
    io_detail: >-
      {{ 'no disk exposes SCSI/ATA error counters (virtio or NVMe-only node)'
           if (counted | length) == 0
         else ((counted | map(attribute='kname') | join(', ')) ~ ': completed='
               ~ io_sum_done | string ~ ', errors=' ~ io_sum_err | string
               ~ ', timeouts=' ~ io_sum_tmo | string
               ~ ('. Non-zero errors are commands the kernel saw fail on that device
                   since boot. They are NOT proof of media failure (a rejected or
                   unsupported command during discovery counts too) and they cannot be
                   attributed without dmesg, which needs root.'
                  if err_total | int > 0 else '')) }}

- name: Assess — filesystem free space
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'storage.capacity',
      'title': 'Filesystem free space (worst mount)',
      'status': fs_status,
      'detail': fs_detail }] }}"
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    worst: "{{ (node_health_mounts_by_use | first) | default({}) }}"
    fs_status: >-
      {{ 'SKIP' if (node_health_mounts_by_use | length) == 0
         else ('FAIL' if worst.used_pct >= th.fs_used_pct_fail
               else ('WARN' if worst.used_pct >= th.fs_used_pct_warn else 'PASS')) }}
    fs_detail: >-
      {{ 'no block-backed filesystem found' if (node_health_mounts_by_use | length) == 0
         else (worst.mountpoint ~ ' ' ~ worst.used_pct | string ~ '% used ('
               ~ (worst.avail_bytes | int / 1073741824) | round(1) | string ~ ' GiB free of '
               ~ (worst.size_bytes | int / 1073741824) | round(1) | string ~ ' GiB)') }}

- name: Assess — filesystem inode headroom
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'storage.inodes',
      'title': 'Filesystem inode headroom (worst mount)',
      'status': in_status,
      'detail': in_detail }] }}"
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    worst: "{{ (node_health_mounts_by_inode | first) | default({}) }}"
    in_status: >-
      {{ 'SKIP' if (node_health_mounts_by_inode | length) == 0
         else ('WARN' if worst.inodes_used_pct >= th.inode_used_pct_warn else 'PASS') }}
    in_detail: >-
      {{ 'no filesystem reported inode usage' if (node_health_mounts_by_inode | length) == 0
         else (worst.mountpoint ~ ' ' ~ worst.inodes_used_pct | string ~ '% of inodes used') }}

- name: Assess — data-disk layout (feeds PROD-PROVISIONING.md §2)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'storage.layout',
      'title': 'Physical disks and spare block devices for Longhorn/etcd',
      'status': 'INFO',
      'detail': layout_detail }] }}"
  vars:
    ansible_become: false
    disks: "{{ node_health_facts.storage.disks }}"
    attached: "{{ node_health_facts.storage.attached_volumes | default([]) }}"
    layout_detail: >-
      {{ (disks | length) | string ~ ' physical disk(s): '
         ~ (disks | map(attribute='kname') | join(', ')) ~ '. '
         ~ ((attached | length) | string ~ ' attached iSCSI/FC volume(s) excluded ('
            ~ (attached | map(attribute='kname') | join(', ')) ~ '). '
            if (attached | length) > 0 else '')
         ~ ((node_health_free_devices | length) | string ~ ' device(s) with no partitions ('
            ~ (node_health_free_devices | map(attribute='kname') | join(', ')) ~ ')'
            if (node_health_free_devices | length) > 0
            else 'NO unpartitioned spare device — node_baseline_longhorn_device /
                  node_baseline_etcd_device have nothing to claim without repartitioning
                  or adding a drive') }}

# --- Memory -------------------------------------------------------------------------
- name: Assess — memory headroom
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'memory.available',
      'title': 'Available memory',
      'status': mem_status,
      'detail': mem_detail }] }}"
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    m: "{{ node_health_facts.memory }}"
    avail_pct: >-
      {{ ((m.mem_available_kb | int) * 100.0 / (m.mem_total_kb | int)) | round(1)
         if (m.mem_total_kb is number and m.mem_available_kb is number and m.mem_total_kb | int > 0)
         else none }}
    mem_status: >-
      {{ 'SKIP' if avail_pct is not number
         else ('FAIL' if avail_pct | float <= th.mem_available_pct_fail
               else ('WARN' if avail_pct | float <= th.mem_available_pct_warn else 'PASS')) }}
    mem_detail: >-
      {{ '/proc/meminfo unreadable' if avail_pct is not number
         else ((m.mem_total_kb | int / 1048576) | round(1) | string ~ ' GiB total, '
               ~ (m.mem_available_kb | int / 1048576) | round(1) | string ~ ' GiB available ('
               ~ avail_pct | string ~ '%)') }}

- name: Assess — swap (kubelet requires it off)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'memory.swap',
      'title': 'Swap disabled (kubelet prerequisite)',
      'status': swap_status,
      'detail': swap_detail }] }}"
  vars:
    ansible_become: false
    m: "{{ node_health_facts.memory }}"
    swap_status: >-
      {{ 'SKIP' if m.swap_total_kb is not number
         else ('PASS' if m.swap_total_kb | int == 0 else 'WARN') }}
    swap_detail: >-
      {{ 'swap state unreadable' if m.swap_total_kb is not number
         else ('no swap active' if m.swap_total_kb | int == 0
               else ((m.swap_total_kb | int / 1048576) | round(2) | string
                     ~ ' GiB swap active — roles/node_baseline runs swapoff and neutralises
                       the fstab entry; this node has not been baselined yet')) }}

- name: Assess — ECC / EDAC correctable errors
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'memory.ecc',
      'title': 'ECC memory errors (EDAC correctable/uncorrectable counters)',
      'status': ecc_status,
      'detail': ecc_detail }] }}"
  vars:
    ansible_become: false
    e: "{{ node_health_facts.memory.edac }}"
    ecc_status: >-
      {{ 'SKIP' if not (e.controllers_present | bool)
         else ('FAIL' if (e.ue_total | int) > 0
               else ('WARN' if (e.ce_total | int) > 0 else 'PASS')) }}
    ecc_detail: >-
      {{ ('no EDAC memory controller is bound (' ~ ('/sys/devices/system/edac/mc exists but is empty'
          if e.sysfs_present | bool else 'no EDAC sysfs at all')
          ~ ') — this platform does not report ECC status. Non-ECC consumer RAM or a VM;
            "no errors" CANNOT be confirmed on this hardware.')
         if not (e.controllers_present | bool)
         else ((e.controllers | length) | string ~ ' controller(s), correctable='
               ~ e.ce_total | string ~ ', uncorrectable=' ~ e.ue_total | string) }}

# --- CPU ----------------------------------------------------------------------------
- name: Assess — CPU thermal throttling
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'cpu.throttle',
      'title': 'CPU thermal throttling since boot',
      'status': thr_status,
      'detail': thr_detail }] }}"
  vars:
    ansible_become: false
    t: "{{ node_health_facts.cpu.throttle }}"
    thr_status: >-
      {{ 'SKIP' if t is none
         else ('PASS' if ((t.core_throttle_count_total | int) + (t.package_throttle_count | int)) == 0
               else 'WARN') }}
    thr_detail: >-
      {{ 'no thermal_throttle counters (VM guest or unsupported CPU)' if t is none
         else ('core throttle events=' ~ t.core_throttle_count_total | string
               ~ ' (' ~ t.core_throttle_total_time_ms | string ~ ' ms), package events='
               ~ t.package_throttle_count | string) }}

- name: Assess — CPU inventory and frequency scaling
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'cpu.inventory',
      'title': 'CPU model, topology and frequency scaling',
      'status': 'INFO',
      'detail': cpu_detail }] }}"
  vars:
    ansible_become: false
    c: "{{ node_health_facts.cpu }}"
    cpu_detail: >-
      {{ (c.model | default('unknown')) ~ ' — ' ~ (c.cpus | default('?')) ~ ' logical CPU(s), '
         ~ (c.sockets | default('?')) ~ ' socket(s) x ' ~ (c.cores_per_socket | default('?'))
         ~ ' core(s) x ' ~ (c.threads_per_core | default('?')) ~ ' thread(s); load '
         ~ (c.loadavg | default([]) | join(' ')) ~ '; '
         ~ ('governor=' ~ c.scaling.governor ~ ' driver=' ~ c.scaling.driver
            ~ ' cur=' ~ ((c.scaling.cur_khz | int) / 1000) | round | int | string ~ 'MHz'
            ~ ' max=' ~ ((c.scaling.max_khz | int) / 1000) | round | int | string ~ 'MHz'
            if c.scaling is mapping else 'no cpufreq scaling exposed') }}

- name: Assess — hardware virtualisation (KubeVirt prerequisite, R-PLAT-03)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'cpu.virtualisation',
      'title': 'Hardware virtualisation available (VT-x/AMD-V + /dev/kvm)',
      'status': virt_status,
      'detail': virt_detail }] }}"
  vars:
    ansible_become: false
    c: "{{ node_health_facts.cpu }}"
    virt_status: >-
      {{ 'PASS' if (c.hardware_virt_flag | bool and c.kvm_device | bool) else 'WARN' }}
    virt_detail: >-
      {{ 'vmx/svm flag=' ~ c.hardware_virt_flag | string ~ ', /dev/kvm='
         ~ c.kvm_device | string
         ~ (', lscpu Virtualization=' ~ c.virtualization if c.virtualization else '')
         ~ (', running as a guest of ' ~ c.hypervisor_vendor if c.is_virtual_guest | bool else '') }}

# --- Thermals -----------------------------------------------------------------------
- name: Assess — temperatures
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'thermal.max',
      'title': 'Hottest reported sensor',
      'status': temp_status,
      'detail': temp_detail }] }}"
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    hottest: "{{ (node_health_temps_c | max) if (node_health_temps_c | length) > 0 else none }}"
    temp_status: >-
      {{ 'SKIP' if hottest is not number
         else ('FAIL' if hottest | float >= th.temp_fail_c
               else ('WARN' if hottest | float >= th.temp_warn_c else 'PASS')) }}
    temp_detail: >-
      {{ 'no thermal sensor exposed (typical in a VM)' if hottest is not number
         else (hottest | string ~ 'C hottest of ' ~ (node_health_temps_c | length) | string
               ~ ' sensors; zones: '
               ~ (node_health_facts.thermal.zones
                  | map(attribute='type') | join(', ')) ) }}

# --- Pressure-stall (PSI) -----------------------------------------------------------
- name: Assess — pressure stall (CPU, memory, I/O)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'pressure.' ~ item,
      'title': 'PSI ' ~ item ~ ' — some avg60 (% of wall time stalled)',
      'status': psi_status,
      'detail': psi_detail }] }}"
  loop: [cpu, memory, io]
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    psi: "{{ node_health_facts.pressure[item] | default(none) }}"
    avg60: "{{ psi.some.avg60 | default(none) if psi is mapping else none }}"
    psi_status: >-
      {{ 'SKIP' if avg60 is not number
         else ('FAIL' if avg60 | float >= th.psi_some_avg60_fail
               else ('WARN' if avg60 | float >= th.psi_some_avg60_warn else 'PASS')) }}
    psi_detail: >-
      {{ '/proc/pressure/' ~ item ~ ' unavailable (kernel without CONFIG_PSI, or psi=0)'
         if avg60 is not number
         else ('some avg10=' ~ psi.some.avg10 | string ~ ' avg60=' ~ psi.some.avg60 | string
               ~ ' avg300=' ~ psi.some.avg300 | string
               ~ (' | full avg60=' ~ psi.full.avg60 | string if psi.full is defined else '')) }}

# --- Network ------------------------------------------------------------------------
- name: Assess — link state and speed
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'network.link',
      'title': 'Physical NIC link state and negotiated speed',
      'status': link_status,
      'detail': link_detail }] }}"
  vars:
    ansible_become: false
    th: "{{ node_health_thresholds }}"
    up: "{{ node_health_nics_up }}"
    slow: >-
      {{ up | selectattr('speed_mbps', 'number')
         | selectattr('speed_mbps', 'lt', th.nic_min_speed_mbps | int) | list }}
    link_status: >-
      {{ 'FAIL' if (up | length) == 0
         else ('WARN' if (slow | length) > 0 else 'PASS') }}
    link_detail: >-
      {{ 'no physical interface is up' if (up | length) == 0
         else (up | map(attribute='name') | zip(up | map(attribute='speed_mbps'))
               | map('join', '=') | join('Mb/s, ') ~ 'Mb/s'
               ~ ' (driver ' ~ (up | map(attribute='driver') | join(', ')) ~ ')') }}

- name: Assess — link error counters
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'network.errors',
      'title': 'NIC error counters since boot (rx/tx errors, CRC, carrier)',
      'status': ('PASS' if err_total | int == 0 else 'WARN'),
      'detail': err_detail }] }}"
  vars:
    ansible_become: false
    up: "{{ node_health_nics_up }}"
    err_keys: [rx_errors, tx_errors, rx_crc_errors, rx_frame_errors, rx_over_errors, tx_carrier_errors]
    err_total: >-
      {{ up | map(attribute='statistics') | map('dict2items') | flatten
         | selectattr('key', 'in', err_keys) | map(attribute='value')
         | select('number') | list | sum }}
    drop_total: >-
      {{ up | map(attribute='statistics') | map('dict2items') | flatten
         | selectattr('key', 'in', ['rx_dropped', 'tx_dropped']) | map(attribute='value')
         | select('number') | list | sum }}
    err_detail: >-
      {{ err_total | string ~ ' error(s), ' ~ drop_total | string ~ ' drop(s) across '
         ~ (up | map(attribute='name') | join(', ')) }}

- name: Assess — DNS resolver redundancy
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'network.resolvers',
      'title': 'Configured DNS upstreams (single upstream is a known SPOF here)',
      'status': ('PASS' if (ns4 | length) >= (node_health_thresholds.min_nameservers | int) else 'WARN'),
      'detail': ns_detail }] }}"
  vars:
    ansible_become: false
    ns: "{{ node_health_facts.network.nameservers }}"
    # Counted PER FAMILY on purpose. The outage this check exists for was a single IPv4
    # upstream; IPv6 upstreams do not cover for it unless the node actually has working
    # IPv6 egress, which this role cannot prove from sysfs alone. So the verdict keys off
    # the IPv4 count and the detail shows the full picture.
    ns4: "{{ ns | select('search', '^[0-9.]+$') | list }}"
    ns6: "{{ ns | reject('search', '^[0-9.]+$') | list }}"
    ns_detail: >-
      {{ (ns | length) | string ~ ' nameserver(s) — IPv4: '
         ~ (ns4 | join(', ') | default('none', true))
         ~ '; IPv6: ' ~ (ns6 | join(', ') | default('none', true))
         ~ ('. Only ' ~ (ns4 | length) | string ~ ' IPv4 upstream: a single IPv4 upstream
             already broke containerd image pulls and CoreDNS forwarding on dev, and the
             fix is stranded on fix/node-dns-resilient-upstream (DECISIONS-LOG
             2026-08-31, finding 5). IPv6 upstreams only compensate if the node has
             working IPv6 egress, which this check cannot verify.'
            if (ns4 | length) < (node_health_thresholds.min_nameservers | int) else '') }}

# --- OS / firmware ------------------------------------------------------------------
- name: Assess — pending reboot
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'os.reboot_required',
      'title': 'Pending reboot flag',
      'status': ('WARN' if node_health_facts.os.reboot_required | bool else 'PASS'),
      'detail': ('/run/reboot-required present — a package upgrade needs a restart'
                 if node_health_facts.os.reboot_required | bool
                 else 'no reboot flagged') }] }}"
  vars:
    ansible_become: false

- name: Assess — OS, kernel and uptime
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'os.inventory',
      'title': 'OS, kernel and uptime',
      'status': 'INFO',
      'detail': os_detail }] }}"
  vars:
    ansible_become: false
    o: "{{ node_health_facts.os }}"
    os_detail: >-
      {{ (o.pretty_name | default('unknown OS')) ~ ', kernel ' ~ (o.kernel | default('?'))
         ~ ', up ' ~ ((o.uptime_seconds | float / 86400) | round(2) | string) ~ ' day(s)' }}

- name: Assess — firmware / DMI inventory
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'firmware.inventory',
      'title': 'Chassis, board and BIOS firmware',
      'status': ('INFO' if node_health_facts.dmi.present | bool else 'SKIP'),
      'detail': dmi_detail }] }}"
  vars:
    ansible_become: false
    d: "{{ node_health_facts.dmi }}"
    dmi_detail: >-
      {{ 'no DMI table exposed (typical for some VMs/containers)' if not (d.present | bool)
         else ((d.sys_vendor | default('?')) ~ ' ' ~ (d.product_name | default('?'))
               ~ ' / board ' ~ (d.board_name | default('?'))
               ~ ' / BIOS ' ~ (d.bios_vendor | default('?')) ~ ' '
               ~ (d.bios_version | default('?')) ~ ' (' ~ (d.bios_date | default('?')) ~ ')'
               ~ (' — serial numbers not read (root-only DMI fields)'
                  if d.product_serial is none else '')) }}