--- # ONE round on ONE node: reboot, wait for it back, measure the new boot, record it twice. # Included in a loop from tasks/main.yml with loop_var `memattr_survey_round`. # # FAIL-SAFE, PER NODE. The block/rescue below is what keeps one dead node from ending the # survey: on any failure this host sets memattr_survey_aborted and every later round is # skipped FOR THIS HOST ONLY, while the other nodes keep going (Ansible runs the loop for # all hosts in lockstep, so the reboots are genuinely concurrent). The abort is reported # loudly here AND again in the final analysis, which counts rounds from the artifacts # rather than from Ansible's host state — so a node that vanishes so hard that it is # marked unreachable (which rescue cannot catch) still shows up as a short arm. - name: "Round {{ memattr_survey_round }} — survey step" block: - name: "Round {{ memattr_survey_round }} — reboot and wait for the node to return" ansible.builtin.reboot: reboot_timeout: "{{ memattr_survey_reboot_timeout | int }}" connect_timeout: "{{ memattr_survey_connect_timeout | int }}" post_reboot_delay: "{{ memattr_survey_post_reboot_delay | int }}" msg: "memattr survey round {{ memattr_survey_round }} of {{ memattr_survey_rounds }}" - name: "Round {{ memattr_survey_round }} — measure the new boot and append to the ledger" ansible.builtin.script: cmd: >- memattr_probe.py --round {{ memattr_survey_round }} --append {{ memattr_survey_state_file | quote }} executable: /usr/bin/python3 register: memattr_survey_round_raw changed_when: false # see the note in preflight.yml — the ledger is not the node - name: "Round {{ memattr_survey_round }} — parse the record" ansible.builtin.set_fact: memattr_survey_record: "{{ memattr_survey_round_raw.stdout | from_json }}" - name: "Round {{ memattr_survey_round }} — result" ansible.builtin.debug: msg: >- [{{ inventory_hostname }} r{{ memattr_survey_round }}/{{ memattr_survey_rounds }}] MEMATTR={{ memattr_survey_record.memattr_address | default('none') }} read={{ memattr_survey_record.kernel_read }} {%- if memattr_survey_record.memattr_table_version is not none %} (version {{ memattr_survey_record.memattr_table_version }}) {%- endif %} MemTotal={{ (memattr_survey_record.mem_total_kb | default(0) | int / 1048576) | round(2) }}G of {{ (memattr_survey_record.smbios_installed_kb | default(0) | int / 1048576) | round(0) | int }}G => {{ memattr_survey_record.verdict | upper }} boot={{ memattr_survey_record.boot_id | default('?') | truncate(9, true, '') }} when: memattr_survey_show_progress | bool # BRACES. The on-node ledger already survives a reboot; re-fetching it every round # means the controller also holds every completed round if the DRIVER dies, without # waiting for the end-of-run harvest. One tiny file per host, per-host destination # paths, so three concurrent fetches cannot collide. - name: "Round {{ memattr_survey_round }} — mirror the ledger to the controller" ansible.builtin.fetch: src: "{{ memattr_survey_state_file }}" dest: "{{ memattr_survey_artifact_dir }}/{{ inventory_hostname }}.rounds.jsonl" flat: true changed_when: false when: memattr_survey_write_artifacts | bool rescue: - name: "Round {{ memattr_survey_round }} — ABORT this node, continue the others" ansible.builtin.set_fact: memattr_survey_aborted: true memattr_survey_abort_reason: >- round {{ memattr_survey_round }} failed: {{ ansible_failed_result.msg | default('unknown error') | trim }} # The abort must survive the run, not live only in terminal scrollback. Without # this the report can see that an arm is SHORT but not WHY, and "0 rounds" reads # like a targeting mistake instead of a node that failed to come back. - name: "Round {{ memattr_survey_round }} — record the abort as an artifact" ansible.builtin.copy: dest: "{{ memattr_survey_artifact_dir }}/{{ inventory_hostname }}.aborted.json" mode: "0644" content: "{{ memattr_survey_abort_record | to_nice_json }}" vars: memattr_survey_abort_record: schema: "memattr-survey/1" host: "{{ inventory_hostname }}" aborted_at_round: "{{ memattr_survey_round }}" target_rounds: "{{ memattr_survey_rounds }}" reason: "{{ memattr_survey_abort_reason | trim }}" failed_task: "{{ ansible_failed_task.name | default('unknown') }}" delegate_to: localhost become: false changed_when: false when: memattr_survey_write_artifacts | bool - name: "Round {{ memattr_survey_round }} — announce the abort" ansible.builtin.debug: msg: >- *** SURVEY ABORTED ON {{ inventory_hostname }} at round {{ memattr_survey_round }}/{{ memattr_survey_rounds }}: {{ memattr_survey_abort_reason }} *** The other nodes continue. This node's arm is SHORT — the analysis will say so, and its p(bad) interval will be correspondingly wider.