--- # Turn raw facts into explicit, named checks. Every check is one of: # # PASS measured, and within threshold # WARN measured, outside the warn threshold — worth a human look # FAIL measured, outside the fail threshold — do not put production state here # SKIP NOT measured, with the reason attached (missing root, missing tool, sensor not # present on this hardware). A SKIP is an honest "unknown" and is NEVER rendered # as a pass — this repo's discipline is absolute on that point. # INFO measured, no pass/fail semantics (inventory: firmware, uptime, CPU scaling) # # All thresholds come from node_health_thresholds (defaults/main.yml) so retuning never # means editing logic here. - name: Derive intermediate values used by several checks ansible.builtin.set_fact: # Every temperature the box exposes, from both thermal zones and hwmon chips. # `select('number')` drops any sensor that returned null (unreadable/absent). node_health_temps_c: >- {{ ((node_health_facts.thermal.zones | map(attribute='temp_c') | list) + (node_health_facts.thermal.hwmon | map(attribute='temps') | flatten | map(attribute='temp_c') | list)) | select('number') | list }} # Filesystems, worst-first by percentage used / inodes used. node_health_mounts_by_use: >- {{ node_health_facts.storage.mounts | selectattr('used_pct', 'number') | sort(attribute='used_pct') | reverse | list }} node_health_mounts_by_inode: >- {{ node_health_facts.storage.mounts | selectattr('inodes_used_pct', 'number') | sort(attribute='inodes_used_pct') | reverse | list }} # Physical NICs that are actually up. Virtual plumbing was already excluded by the # collector; `virtual` here catches bonds/VLANs the regex let through. node_health_nics_up: >- {{ node_health_facts.network.interfaces | selectattr('operstate', 'equalto', 'up') | list }} # Devices that could actually BECOME a dedicated Longhorn or etcd volume: no # partitions, no whole-device filesystem, nothing mounted from them. All three # conditions matter — node_baseline formats the data disks with no partition table at # all (mkfs straight onto /dev/vdb, mounted by LABEL), so "no partitions" alone would # nominate a node's live Longhorn disk for reformatting. iSCSI/FC volumes were already # split out by the collector for the same reason. On a single-disk node this list is # empty — precisely the finding that blocks PROD-PROVISIONING.md §2. node_health_free_devices: >- {{ node_health_facts.storage.disks | selectattr('partitions', 'equalto', []) | selectattr('fstype', 'none') | selectattr('mountpoints', 'equalto', []) | list }} vars: ansible_become: false - name: Assess — privilege escalation (gates every root-only check below) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'access.privilege', 'title': 'Privilege escalation (become) for root-only checks', 'status': ('INFO' if node_health_root_available | bool else 'SKIP'), 'detail': node_health_root_skip_reason }] }}" # --- Storage ------------------------------------------------------------------------ # SMART VERDICTS — READ defaults/main.yml's SMART threshold block BEFORE EDITING THIS. # Short version: this check judges a drive on the signals smartctl(8) itself defines as # vendor-independent (the drive's own overall-health verdict; NORMALISED value vs the # MANUFACTURER'S OWN threshold, via smartctl's `when_failed`; and the ATA-standard error # counters 197/198/187/199 whose raw meaning is a plain event count everywhere, plus NVMe's # structured health log). Vendor raw counts are reported with their numbers and can raise a # WARN, but they can never produce a FAIL — because their units are vendor-defined and # reading one as if it were a portable sector count is exactly the defect this replaces. - name: Assess — SMART per physical disk ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'storage.smart.' ~ item.kname, 'title': 'SMART health — ' ~ item.path ~ ' (' ~ (item.model | default('unknown model')) ~ ')', 'status': smart_status, 'detail': smart_detail }] }}" loop: "{{ node_health_facts.storage.disks }}" loop_control: label: "{{ item.path }}" vars: ansible_become: false th: "{{ node_health_thresholds }}" disk_smart: "{{ (node_health_smart | selectattr('path', 'equalto', item.path) | list | first) | default({}) }}" # --- Sub-verdicts. Each is SKIP unless the number behind it was actually read; a SKIP is # an honest "unknown" and never contributes a PASS. ------------------------------------ # 1. The drive's own overall-health self-assessment. Vendor-independent by construction. s_health: >- {{ 'SKIP' if (disk_smart.passed is not defined or disk_smart.passed is none) else ('PASS' if disk_smart.passed else 'FAIL') }} # 2. NVMe critical warning bitfield (spare below threshold, media in read-only, etc.). s_nvme_crit: >- {{ 'SKIP' if (disk_smart.nvme_critical_warning is not defined or disk_smart.nvme_critical_warning is not number) else ('PASS' if disk_smart.nvme_critical_warning | int == 0 else 'FAIL') }} # 3. THE portable per-attribute failure signal: normalised value at or below the # threshold the MANUFACTURER set for that attribute on that model. Pre-fail failing # now = FAIL. Old_age failing now, or anything that failed in the past, = WARN. s_thresh: >- {{ 'SKIP' if not (disk_smart.thresholds_present | default(false)) else ('FAIL' if (disk_smart.failing_now_prefail | default([]) | length) > 0 else ('WARN' if ((disk_smart.failing_now | default([]) | length) > 0 or (disk_smart.failed_past | default([]) | length) > 0) else 'PASS')) }} # 4. ATA-standard hard-error counters. 197 pending; 198/187 + NVMe media_errors are # uncorrectable reads — data the drive could NOT return. Same meaning on every drive. s_pending: >- {{ 'SKIP' if (disk_smart.pending_sectors is not defined or disk_smart.pending_sectors is not number) else ('FAIL' if disk_smart.pending_sectors | int >= th.smart_pending_fail else ('WARN' if disk_smart.pending_sectors | int >= th.smart_pending_warn else 'PASS')) }} uncorr_values: >- {{ [disk_smart.offline_uncorrectable | default(none), disk_smart.reported_uncorrect | default(none), disk_smart.nvme_media_errors | default(none)] | select('number') | list }} s_uncorr: >- {{ 'SKIP' if (uncorr_values | length) == 0 else ('FAIL' if (uncorr_values | max) >= th.smart_uncorrectable_fail else ('WARN' if (uncorr_values | max) >= th.smart_uncorrectable_warn else 'PASS')) }} # 5. 199 CRC — SATA link integrity (cable/backplane), not media. WARN only: the counter # never resets, so one historic bad seating must not condemn the node forever. s_crc: >- {{ 'SKIP' if (disk_smart.crc_errors is not defined or disk_smart.crc_errors is not number) else ('WARN' if disk_smart.crc_errors | int >= th.smart_crc_warn else 'PASS') }} # 6. Wear, from the NORMALISED lifetime indicator — a percentage by definition. life_rem: "{{ disk_smart.lifetime_remaining_pct | default(none) }}" life_used: "{{ (100 - (life_rem | int)) if (life_rem is number) else none }}" s_life: >- {{ 'SKIP' if (life_rem is not number) else ('FAIL' if life_rem | int <= th.smart_lifetime_remaining_fail_pct else ('WARN' if (life_rem | int <= th.smart_lifetime_remaining_warn_pct or life_used | int >= th.smart_life_used_pct_warn) else 'PASS')) }} # 7. Reallocation ACTIVITY. Portable evidence first (196 event count, and the grown # bad-block counter smartctl's drivedb named for this model). Only if neither exists # do we fall back to attribute 5's raw — and then ONLY to say "nonzero, so something # was remapped", which is portable, never to interpret its magnitude, which is not. remap_evidence: >- {{ [disk_smart.realloc_events | default(none), disk_smart.grown_bad_blocks | default(none)] | select('number') | list }} realloc_raw: "{{ (disk_smart.realloc_attr | default({}, true)).get('raw', {}).get('value', none) }}" s_remap: >- {{ ('WARN' if (remap_evidence | max) >= th.smart_realloc_events_warn else 'PASS') if (remap_evidence | length) > 0 else (('WARN' if realloc_raw | int > 0 else 'PASS') if realloc_raw is number else 'SKIP') }} # 8. Endurance stress — erase/program failures are NAND wearing out under the FTL. s_erase: >- {{ 'SKIP' if (disk_smart.erase_fail_count is not defined or disk_smart.erase_fail_count is not number) else ('WARN' if disk_smart.erase_fail_count | int >= th.smart_erase_fail_warn else 'PASS') }} s_program: >- {{ 'SKIP' if (disk_smart.program_fail_count is not defined or disk_smart.program_fail_count is not number) else ('WARN' if disk_smart.program_fail_count | int >= th.smart_program_fail_warn else 'PASS') }} # 9. Unexpected power loss. On a consumer SSD with no power-loss protection each one is # an opportunity for a torn or lost in-flight write — which is precisely the guarantee # etcd's fsync contract depends on. Directly relevant to PROD-PROVISIONING.md Section 2. s_ploss: >- {{ 'SKIP' if (disk_smart.unexpected_power_loss is not defined or disk_smart.unexpected_power_loss is not number) else ('WARN' if disk_smart.unexpected_power_loss | int >= th.smart_unexpected_power_loss_warn else 'PASS') }} s_temp: >- {{ 'SKIP' if (disk_smart.temperature_c is not defined or disk_smart.temperature_c is not number) else ('FAIL' if disk_smart.temperature_c | int >= th.smart_temp_fail_c else ('WARN' if disk_smart.temperature_c | int >= th.smart_temp_warn_c else 'PASS')) }} # Worst sub-verdict wins. `select('in', ...)` preserves the order of the left-hand # list, so the LAST survivor is the most severe present. smart_status: >- {{ (['PASS', 'WARN', 'FAIL'] | select('in', [s_health, s_nvme_crit, s_thresh, s_pending, s_uncorr, s_crc, s_life, s_remap, s_erase, s_program, s_ploss, s_temp]) | list | last) | default('SKIP', true) }} # --- Detail. Every WARN/FAIL must arrive WITH the number that caused it; a verdict the # operator cannot check is not much better than a wrong one. ------------------------- smart_findings: >- {%- set n = namespace(l=[]) -%} {%- if s_health == 'FAIL' -%} {%- set _ = n.l.append('the drive reports its OWN overall-health self-assessment as FAILED') -%} {%- endif -%} {%- if s_nvme_crit == 'FAIL' -%} {%- set _ = n.l.append('NVMe critical_warning=' ~ disk_smart.nvme_critical_warning ~ ' (non-zero)') -%} {%- endif -%} {%- for a in (disk_smart.failing_now_prefail | default([])) -%} {%- set _ = n.l.append('Pre-fail attribute ' ~ a.id ~ ' ' ~ a.name ~ ' is AT OR BELOW its manufacturer threshold NOW (value=' ~ a.value ~ ', thresh=' ~ a.thresh ~ ')') -%} {%- endfor -%} {%- for a in (disk_smart.failed_past | default([])) -%} {%- set _ = n.l.append('attribute ' ~ a.id ~ ' ' ~ a.name ~ ' crossed its manufacturer threshold IN THE PAST (worst=' ~ a.worst ~ ', thresh=' ~ a.thresh ~ ')') -%} {%- endfor -%} {%- if s_pending in ['WARN', 'FAIL'] -%} {%- set _ = n.l.append('current pending sectors (197)=' ~ disk_smart.pending_sectors) -%} {%- endif -%} {%- if s_uncorr in ['WARN', 'FAIL'] -%} {%- set _ = n.l.append('UNCORRECTABLE read errors — offline(198)=' ~ (disk_smart.offline_uncorrectable if disk_smart.offline_uncorrectable is number else 'n/a') ~ ', reported(187)=' ~ (disk_smart.reported_uncorrect if disk_smart.reported_uncorrect is number else 'n/a') ~ ', nvme media_errors=' ~ (disk_smart.nvme_media_errors if disk_smart.nvme_media_errors is number else 'n/a')) -%} {%- endif -%} {%- if s_crc == 'WARN' -%} {%- set _ = n.l.append('SATA link CRC errors (199)=' ~ disk_smart.crc_errors ~ ' — cable/backplane, not media') -%} {%- endif -%} {%- if s_life in ['WARN', 'FAIL'] -%} {%- set _ = n.l.append(life_used ~ '% of rated life CONSUMED (' ~ life_rem ~ '% remaining, from the normalised wear indicator)') -%} {%- endif -%} {%- if s_remap == 'WARN' and (remap_evidence | length) > 0 -%} {%- set _ = n.l.append((remap_evidence | max) ~ ' grown bad block(s)/reallocation event(s)' ~ (' [196 events=' ~ disk_smart.realloc_events ~ ']' if disk_smart.realloc_events is number else '') ~ (' [grown failing blocks=' ~ disk_smart.grown_bad_blocks ~ ']' if disk_smart.grown_bad_blocks is number else '')) -%} {%- elif s_remap == 'WARN' -%} {%- set _ = n.l.append('the drive reports a non-zero reallocation attribute (5 raw=' ~ realloc_raw ~ '); its UNIT is vendor-defined, so the count is not interpreted here') -%} {%- endif -%} {%- if s_erase == 'WARN' -%} {%- set _ = n.l.append('NAND erase failures=' ~ disk_smart.erase_fail_count) -%} {%- endif -%} {%- if s_program == 'WARN' -%} {%- set _ = n.l.append('NAND program failures=' ~ disk_smart.program_fail_count) -%} {%- endif -%} {%- if s_ploss == 'WARN' -%} {%- set _ = n.l.append('unexpected power losses=' ~ disk_smart.unexpected_power_loss ~ ' — a consumer SSD has no power-loss protection, so each one risks a torn or lost in-flight write') -%} {%- endif -%} {%- if s_temp in ['WARN', 'FAIL'] -%} {%- set _ = n.l.append('temperature=' ~ disk_smart.temperature_c ~ 'C') -%} {%- endif -%} {{ n.l | join('; ') }} # The raw value that started this. Shown ALWAYS when it is non-zero and contradicts the # normalised reading, explicitly labelled as vendor-encoded and explicitly excluded from # the verdict — so the next operator to see 61440 sees why it is not 61440 bad sectors. realloc_raw_note: >- {%- if realloc_raw is number and realloc_raw | int > 0 -%} {%- set a = disk_smart.realloc_attr -%} attribute 5 ({{ a.name }}) raw={{ realloc_raw }} normalised value={{ a.value }} vs thresh={{ a.thresh }} {%- if (remap_evidence | length) > 0 and (remap_evidence | max) > 0 and realloc_raw | int != (remap_evidence | max) | int and realloc_raw | int % (remap_evidence | max) | int == 0 -%} {{ ' (= ' ~ (remap_evidence | max) ~ ' x ' ~ (realloc_raw | int // (remap_evidence | max) | int) ~ ')' }} {%- endif -%} . SMART raw values are VENDOR-DEFINED (smartctl(8): "entirely vendor-specific") and this one is NOT a portable bad-sector count — it is reported as context and is NOT used for the verdict. The normalised value/threshold pair beside it is the manufacturer's own trip point and IS. {%- endif -%} smart_detail: >- {{ ('not read — ' ~ node_health_root_skip_reason) if not (node_health_root_available | bool) else ('not read — smartmontools is not installed on this node; it ships with roles/node_baseline (node_baseline_packages), which has not been run here. This role never installs packages.') if not (node_health_facts.tools.smartctl is truthy) else ('smartctl ran but this device exposes no SMART data (' ~ (disk_smart.messages | default([]) | join('; ')) ~ ') — normal for virtio/VM disks') if not (disk_smart.available | default(false)) else ('overall-health=' ~ ('PASSED' if disk_smart.passed else 'FAILED') ~ ', power_on_hours=' ~ (disk_smart.power_on_hours | default('?')) ~ ', power_cycles=' ~ (disk_smart.power_cycles | default('?')) ~ ', life_remaining=' ~ ((life_rem | string ~ '%') if life_rem is number else 'not reported') ~ ', pending(197)=' ~ (disk_smart.pending_sectors if disk_smart.pending_sectors is number else 'n/a') ~ ', offline_uncorrectable(198)=' ~ (disk_smart.offline_uncorrectable if disk_smart.offline_uncorrectable is number else 'n/a') ~ ', reported_uncorrect(187)=' ~ (disk_smart.reported_uncorrect if disk_smart.reported_uncorrect is number else 'n/a') ~ ', crc(199)=' ~ (disk_smart.crc_errors if disk_smart.crc_errors is number else 'n/a') ~ (('. NO attribute is at or below its manufacturer threshold; closest is ' ~ disk_smart.prefail_closest.id ~ ' ' ~ disk_smart.prefail_closest.name ~ ' value=' ~ disk_smart.prefail_closest.value ~ ' vs thresh=' ~ disk_smart.prefail_closest.thresh) if (s_thresh == 'PASS' and (disk_smart.prefail_closest | default(none)) is mapping) else '') ~ (('. FINDINGS: ' ~ smart_findings) if smart_findings | trim else '') ~ (('. CONTEXT: ' ~ realloc_raw_note) if realloc_raw_note | trim else '')) }} - name: Assess — kernel-visible I/O errors per disk (no root needed) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'storage.io_errors', 'title': 'Kernel I/O error counters on physical disks (SCSI/ATA layer)', 'status': io_status, 'detail': io_detail }] }}" vars: ansible_become: false # ioerr_cnt/iotmo_cnt are readable WITHOUT root. They are not a substitute for SMART # (no wear, no power-on hours, no reallocated sectors) but they are a real signal: a # non-zero count means the kernel actually saw failed or timed-out commands on that # device since boot. counted: "{{ node_health_facts.storage.disks | selectattr('scsi_counters', 'mapping') | list }}" err_total: >- {{ (counted | map(attribute='scsi_counters.ioerr_cnt') | select('number') | list | sum) + (counted | map(attribute='scsi_counters.iotmo_cnt') | select('number') | list | sum) }} io_status: >- {{ 'SKIP' if (counted | length) == 0 else ('PASS' if err_total | int == 0 else 'WARN') }} io_sum_done: "{{ counted | map(attribute='scsi_counters.iodone_cnt') | select('number') | list | sum }}" io_sum_err: "{{ counted | map(attribute='scsi_counters.ioerr_cnt') | select('number') | list | sum }}" io_sum_tmo: "{{ counted | map(attribute='scsi_counters.iotmo_cnt') | select('number') | list | sum }}" # Aggregate here; the STORAGE section of the summary prints the per-disk counters. io_detail: >- {{ 'no disk exposes SCSI/ATA error counters (virtio or NVMe-only node)' if (counted | length) == 0 else ((counted | map(attribute='kname') | join(', ')) ~ ': completed=' ~ io_sum_done | string ~ ', errors=' ~ io_sum_err | string ~ ', timeouts=' ~ io_sum_tmo | string ~ ('. Non-zero errors are commands the kernel saw fail on that device since boot. They are NOT proof of media failure (a rejected or unsupported command during discovery counts too) and they cannot be attributed without dmesg, which needs root.' if err_total | int > 0 else '')) }} - name: Assess — filesystem free space ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'storage.capacity', 'title': 'Filesystem free space (worst mount)', 'status': fs_status, 'detail': fs_detail }] }}" vars: ansible_become: false th: "{{ node_health_thresholds }}" worst: "{{ (node_health_mounts_by_use | first) | default({}) }}" fs_status: >- {{ 'SKIP' if (node_health_mounts_by_use | length) == 0 else ('FAIL' if worst.used_pct >= th.fs_used_pct_fail else ('WARN' if worst.used_pct >= th.fs_used_pct_warn else 'PASS')) }} fs_detail: >- {{ 'no block-backed filesystem found' if (node_health_mounts_by_use | length) == 0 else (worst.mountpoint ~ ' ' ~ worst.used_pct | string ~ '% used (' ~ (worst.avail_bytes | int / 1073741824) | round(1) | string ~ ' GiB free of ' ~ (worst.size_bytes | int / 1073741824) | round(1) | string ~ ' GiB)') }} - name: Assess — filesystem inode headroom ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'storage.inodes', 'title': 'Filesystem inode headroom (worst mount)', 'status': in_status, 'detail': in_detail }] }}" vars: ansible_become: false th: "{{ node_health_thresholds }}" worst: "{{ (node_health_mounts_by_inode | first) | default({}) }}" in_status: >- {{ 'SKIP' if (node_health_mounts_by_inode | length) == 0 else ('WARN' if worst.inodes_used_pct >= th.inode_used_pct_warn else 'PASS') }} in_detail: >- {{ 'no filesystem reported inode usage' if (node_health_mounts_by_inode | length) == 0 else (worst.mountpoint ~ ' ' ~ worst.inodes_used_pct | string ~ '% of inodes used') }} - name: Assess — data-disk layout (feeds PROD-PROVISIONING.md §2) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'storage.layout', 'title': 'Physical disks and spare block devices for Longhorn/etcd', 'status': 'INFO', 'detail': layout_detail }] }}" vars: ansible_become: false disks: "{{ node_health_facts.storage.disks }}" attached: "{{ node_health_facts.storage.attached_volumes | default([]) }}" layout_detail: >- {{ (disks | length) | string ~ ' physical disk(s): ' ~ (disks | map(attribute='kname') | join(', ')) ~ '. ' ~ ((attached | length) | string ~ ' attached iSCSI/FC volume(s) excluded (' ~ (attached | map(attribute='kname') | join(', ')) ~ '). ' if (attached | length) > 0 else '') ~ ((node_health_free_devices | length) | string ~ ' device(s) with no partitions (' ~ (node_health_free_devices | map(attribute='kname') | join(', ')) ~ ')' if (node_health_free_devices | length) > 0 else 'NO unpartitioned spare device — node_baseline_longhorn_device / node_baseline_etcd_device have nothing to claim without repartitioning or adding a drive') }} # --- Memory ------------------------------------------------------------------------- - name: Assess — memory headroom ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'memory.available', 'title': 'Available memory', 'status': mem_status, 'detail': mem_detail }] }}" vars: ansible_become: false th: "{{ node_health_thresholds }}" m: "{{ node_health_facts.memory }}" avail_pct: >- {{ ((m.mem_available_kb | int) * 100.0 / (m.mem_total_kb | int)) | round(1) if (m.mem_total_kb is number and m.mem_available_kb is number and m.mem_total_kb | int > 0) else none }} mem_status: >- {{ 'SKIP' if avail_pct is not number else ('FAIL' if avail_pct | float <= th.mem_available_pct_fail else ('WARN' if avail_pct | float <= th.mem_available_pct_warn else 'PASS')) }} mem_detail: >- {{ '/proc/meminfo unreadable' if avail_pct is not number else ((m.mem_total_kb | int / 1048576) | round(1) | string ~ ' GiB total, ' ~ (m.mem_available_kb | int / 1048576) | round(1) | string ~ ' GiB available (' ~ avail_pct | string ~ '%)') }} - name: Assess — swap (kubelet requires it off) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'memory.swap', 'title': 'Swap disabled (kubelet prerequisite)', 'status': swap_status, 'detail': swap_detail }] }}" vars: ansible_become: false m: "{{ node_health_facts.memory }}" swap_status: >- {{ 'SKIP' if m.swap_total_kb is not number else ('PASS' if m.swap_total_kb | int == 0 else 'WARN') }} swap_detail: >- {{ 'swap state unreadable' if m.swap_total_kb is not number else ('no swap active' if m.swap_total_kb | int == 0 else ((m.swap_total_kb | int / 1048576) | round(2) | string ~ ' GiB swap active — roles/node_baseline runs swapoff and neutralises the fstab entry; this node has not been baselined yet')) }} - name: Assess — ECC / EDAC correctable errors ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'memory.ecc', 'title': 'ECC memory errors (EDAC correctable/uncorrectable counters)', 'status': ecc_status, 'detail': ecc_detail }] }}" vars: ansible_become: false e: "{{ node_health_facts.memory.edac }}" ecc_status: >- {{ 'SKIP' if not (e.controllers_present | bool) else ('FAIL' if (e.ue_total | int) > 0 else ('WARN' if (e.ce_total | int) > 0 else 'PASS')) }} ecc_detail: >- {{ ('no EDAC memory controller is bound (' ~ ('/sys/devices/system/edac/mc exists but is empty' if e.sysfs_present | bool else 'no EDAC sysfs at all') ~ ') — this platform does not report ECC status. Non-ECC consumer RAM or a VM; "no errors" CANNOT be confirmed on this hardware.') if not (e.controllers_present | bool) else ((e.controllers | length) | string ~ ' controller(s), correctable=' ~ e.ce_total | string ~ ', uncorrectable=' ~ e.ue_total | string) }} # --- CPU ---------------------------------------------------------------------------- - name: Assess — CPU thermal throttling ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'cpu.throttle', 'title': 'CPU thermal throttling since boot', 'status': thr_status, 'detail': thr_detail }] }}" vars: ansible_become: false t: "{{ node_health_facts.cpu.throttle }}" thr_status: >- {{ 'SKIP' if t is none else ('PASS' if ((t.core_throttle_count_total | int) + (t.package_throttle_count | int)) == 0 else 'WARN') }} thr_detail: >- {{ 'no thermal_throttle counters (VM guest or unsupported CPU)' if t is none else ('core throttle events=' ~ t.core_throttle_count_total | string ~ ' (' ~ t.core_throttle_total_time_ms | string ~ ' ms), package events=' ~ t.package_throttle_count | string) }} - name: Assess — CPU inventory and frequency scaling ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'cpu.inventory', 'title': 'CPU model, topology and frequency scaling', 'status': 'INFO', 'detail': cpu_detail }] }}" vars: ansible_become: false c: "{{ node_health_facts.cpu }}" cpu_detail: >- {{ (c.model | default('unknown')) ~ ' — ' ~ (c.cpus | default('?')) ~ ' logical CPU(s), ' ~ (c.sockets | default('?')) ~ ' socket(s) x ' ~ (c.cores_per_socket | default('?')) ~ ' core(s) x ' ~ (c.threads_per_core | default('?')) ~ ' thread(s); load ' ~ (c.loadavg | default([]) | join(' ')) ~ '; ' ~ ('governor=' ~ c.scaling.governor ~ ' driver=' ~ c.scaling.driver ~ ' cur=' ~ ((c.scaling.cur_khz | int) / 1000) | round | int | string ~ 'MHz' ~ ' max=' ~ ((c.scaling.max_khz | int) / 1000) | round | int | string ~ 'MHz' if c.scaling is mapping else 'no cpufreq scaling exposed') }} - name: Assess — hardware virtualisation (KubeVirt prerequisite, R-PLAT-03) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'cpu.virtualisation', 'title': 'Hardware virtualisation available (VT-x/AMD-V + /dev/kvm)', 'status': virt_status, 'detail': virt_detail }] }}" vars: ansible_become: false c: "{{ node_health_facts.cpu }}" virt_status: >- {{ 'PASS' if (c.hardware_virt_flag | bool and c.kvm_device | bool) else 'WARN' }} virt_detail: >- {{ 'vmx/svm flag=' ~ c.hardware_virt_flag | string ~ ', /dev/kvm=' ~ c.kvm_device | string ~ (', lscpu Virtualization=' ~ c.virtualization if c.virtualization else '') ~ (', running as a guest of ' ~ c.hypervisor_vendor if c.is_virtual_guest | bool else '') }} # --- Thermals ----------------------------------------------------------------------- - name: Assess — temperatures ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'thermal.max', 'title': 'Hottest reported sensor', 'status': temp_status, 'detail': temp_detail }] }}" vars: ansible_become: false th: "{{ node_health_thresholds }}" hottest: "{{ (node_health_temps_c | max) if (node_health_temps_c | length) > 0 else none }}" temp_status: >- {{ 'SKIP' if hottest is not number else ('FAIL' if hottest | float >= th.temp_fail_c else ('WARN' if hottest | float >= th.temp_warn_c else 'PASS')) }} temp_detail: >- {{ 'no thermal sensor exposed (typical in a VM)' if hottest is not number else (hottest | string ~ 'C hottest of ' ~ (node_health_temps_c | length) | string ~ ' sensors; zones: ' ~ (node_health_facts.thermal.zones | map(attribute='type') | join(', ')) ) }} # --- Pressure-stall (PSI) ----------------------------------------------------------- - name: Assess — pressure stall (CPU, memory, I/O) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'pressure.' ~ item, 'title': 'PSI ' ~ item ~ ' — some avg60 (% of wall time stalled)', 'status': psi_status, 'detail': psi_detail }] }}" loop: [cpu, memory, io] vars: ansible_become: false th: "{{ node_health_thresholds }}" psi: "{{ node_health_facts.pressure[item] | default(none) }}" avg60: "{{ psi.some.avg60 | default(none) if psi is mapping else none }}" psi_status: >- {{ 'SKIP' if avg60 is not number else ('FAIL' if avg60 | float >= th.psi_some_avg60_fail else ('WARN' if avg60 | float >= th.psi_some_avg60_warn else 'PASS')) }} psi_detail: >- {{ '/proc/pressure/' ~ item ~ ' unavailable (kernel without CONFIG_PSI, or psi=0)' if avg60 is not number else ('some avg10=' ~ psi.some.avg10 | string ~ ' avg60=' ~ psi.some.avg60 | string ~ ' avg300=' ~ psi.some.avg300 | string ~ (' | full avg60=' ~ psi.full.avg60 | string if psi.full is defined else '')) }} # --- Network ------------------------------------------------------------------------ - name: Assess — link state and speed ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'network.link', 'title': 'Physical NIC link state and negotiated speed', 'status': link_status, 'detail': link_detail }] }}" vars: ansible_become: false th: "{{ node_health_thresholds }}" up: "{{ node_health_nics_up }}" slow: >- {{ up | selectattr('speed_mbps', 'number') | selectattr('speed_mbps', 'lt', th.nic_min_speed_mbps | int) | list }} link_status: >- {{ 'FAIL' if (up | length) == 0 else ('WARN' if (slow | length) > 0 else 'PASS') }} link_detail: >- {{ 'no physical interface is up' if (up | length) == 0 else (up | map(attribute='name') | zip(up | map(attribute='speed_mbps')) | map('join', '=') | join('Mb/s, ') ~ 'Mb/s' ~ ' (driver ' ~ (up | map(attribute='driver') | join(', ')) ~ ')') }} - name: Assess — link error counters ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'network.errors', 'title': 'NIC error counters since boot (rx/tx errors, CRC, carrier)', 'status': ('PASS' if err_total | int == 0 else 'WARN'), 'detail': err_detail }] }}" vars: ansible_become: false up: "{{ node_health_nics_up }}" err_keys: [rx_errors, tx_errors, rx_crc_errors, rx_frame_errors, rx_over_errors, tx_carrier_errors] err_total: >- {{ up | map(attribute='statistics') | map('dict2items') | flatten | selectattr('key', 'in', err_keys) | map(attribute='value') | select('number') | list | sum }} drop_total: >- {{ up | map(attribute='statistics') | map('dict2items') | flatten | selectattr('key', 'in', ['rx_dropped', 'tx_dropped']) | map(attribute='value') | select('number') | list | sum }} err_detail: >- {{ err_total | string ~ ' error(s), ' ~ drop_total | string ~ ' drop(s) across ' ~ (up | map(attribute='name') | join(', ')) }} # The collector reports the node's REAL upstreams: where /etc/resolv.conf holds only # systemd-resolved's 127.0.0.53 stub (dev's shape) it asks resolved for the servers behind # it, because counting the stub would be a permanent FALSE single-upstream WARN that no # correct configuration could ever clear. When that lookup fails it reports null WITH a # reason rather than a number — so the check SKIPs. A skip is honest; a WARN derived from # an unknown is a fabricated verdict, and a PASS would be worse. - name: Assess — DNS resolver redundancy ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'network.resolvers', 'title': 'Configured DNS upstreams (single upstream is a known SPOF here)', 'status': ('SKIP' if node_health_facts.network.nameservers is none else ('PASS' if (ns4 | length) >= (node_health_thresholds.min_nameservers | int) else 'WARN')), 'detail': ns_detail }] }}" vars: ansible_become: false # `| default([], true)` only ever fires on the null (unreadable) path, where the # counts below are unused because the status is already SKIP. ns: "{{ node_health_facts.network.nameservers | default([], true) }}" # Counted PER FAMILY on purpose. The outage this check exists for was a single IPv4 # upstream; IPv6 upstreams do not cover for it unless the node actually has working # IPv6 egress, which this role cannot prove from sysfs alone. So the verdict keys off # the IPv4 count and the detail shows the full picture. ns4: "{{ ns | select('search', '^[0-9.]+$') | list }}" ns6: "{{ ns | reject('search', '^[0-9.]+$') | list }}" # Where the numbers came from — resolv.conf directly, or resolved behind the stub. ns_src: "{{ node_health_facts.network.resolver_source | default('', true) }}" ns_detail: >- {{ ('not read — ' ~ (node_health_facts.network.resolver_unavailable_reason | default('the collector recorded no reason', true)) ~ '. The upstream count is UNKNOWN here — not zero, and not one.') if node_health_facts.network.nameservers is none else ((ns | length) | string ~ ' nameserver(s) — IPv4: ' ~ (ns4 | join(', ') | default('none', true)) ~ '; IPv6: ' ~ (ns6 | join(', ') | default('none', true)) ~ (' (read via ' ~ ns_src ~ ')' if ns_src else '') ~ ('. Only ' ~ (ns4 | length) | string ~ ' IPv4 upstream: a single IPv4 upstream already broke containerd image pulls and CoreDNS forwarding on dev. The fix is the `dns`-tagged block in roles/node_baseline — set this environment node_baseline_dns_servers (role defaults for dev, group_vars/k3s_prod for prod) and re-run node-baseline.yml --tags dns. IPv6 upstreams only compensate if the node has working IPv6 egress, which this check cannot verify.' if (ns4 | length) < (node_health_thresholds.min_nameservers | int) else '')) }} # --- OS / firmware ------------------------------------------------------------------ - name: Assess — pending reboot ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'os.reboot_required', 'title': 'Pending reboot flag', 'status': ('WARN' if node_health_facts.os.reboot_required | bool else 'PASS'), 'detail': ('/run/reboot-required present — a package upgrade needs a restart' if node_health_facts.os.reboot_required | bool else 'no reboot flagged') }] }}" vars: ansible_become: false - name: Assess — OS, kernel and uptime ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'os.inventory', 'title': 'OS, kernel and uptime', 'status': 'INFO', 'detail': os_detail }] }}" vars: ansible_become: false o: "{{ node_health_facts.os }}" os_detail: >- {{ (o.pretty_name | default('unknown OS')) ~ ', kernel ' ~ (o.kernel | default('?')) ~ ', up ' ~ ((o.uptime_seconds | float / 86400) | round(2) | string) ~ ' day(s)' }} - name: Assess — firmware / DMI inventory ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'firmware.inventory', 'title': 'Chassis, board and BIOS firmware', 'status': ('INFO' if node_health_facts.dmi.present | bool else 'SKIP'), 'detail': dmi_detail }] }}" vars: ansible_become: false d: "{{ node_health_facts.dmi }}" dmi_detail: >- {{ 'no DMI table exposed (typical for some VMs/containers)' if not (d.present | bool) else ((d.sys_vendor | default('?')) ~ ' ' ~ (d.product_name | default('?')) ~ ' / board ' ~ (d.board_name | default('?')) ~ ' / BIOS ' ~ (d.bios_vendor | default('?')) ~ ' ' ~ (d.bios_version | default('?')) ~ ' (' ~ (d.bios_date | default('?')) ~ ')' ~ (' — serial numbers not read (root-only DMI fields)' if d.product_serial is none else '')) }}