home-network-k3s-impl/infra/ansible/roles/memattr_survey/tasks/round.yml

---
# ONE round on ONE node: reboot, wait for it back, measure the new boot, record it twice.
# Included in a loop from tasks/main.yml with loop_var `memattr_survey_round`.
#
# FAIL-SAFE, PER NODE. The block/rescue below is what keeps one dead node from ending the
# survey: on any failure this host sets memattr_survey_aborted and every later round is
# skipped FOR THIS HOST ONLY, while the other nodes keep going (Ansible runs the loop for
# all hosts in lockstep, so the reboots are genuinely concurrent). The abort is reported
# loudly here AND again in the final analysis, which counts rounds from the artifacts
# rather than from Ansible's host state — so a node that vanishes so hard that it is
# marked unreachable (which rescue cannot catch) still shows up as a short arm.

- name: "Round {{ memattr_survey_round }} — survey step"
  block:
    - name: "Round {{ memattr_survey_round }} — reboot and wait for the node to return"
      ansible.builtin.reboot:
        reboot_timeout: "{{ memattr_survey_reboot_timeout | int }}"
        connect_timeout: "{{ memattr_survey_connect_timeout | int }}"
        post_reboot_delay: "{{ memattr_survey_post_reboot_delay | int }}"
        msg: "memattr survey round {{ memattr_survey_round }} of {{ memattr_survey_rounds }}"

    - name: "Round {{ memattr_survey_round }} — measure the new boot and append to the ledger"
      ansible.builtin.script:
        cmd: >-
          memattr_probe.py --round {{ memattr_survey_round }}
          --append {{ memattr_survey_state_file | quote }}
        executable: /usr/bin/python3
      register: memattr_survey_round_raw
      changed_when: false   # see the note in preflight.yml — the ledger is not the node

    - name: "Round {{ memattr_survey_round }} — parse the record"
      ansible.builtin.set_fact:
        memattr_survey_record: "{{ memattr_survey_round_raw.stdout | from_json }}"

    - name: "Round {{ memattr_survey_round }} — result"
      ansible.builtin.debug:
        msg: >-
          [{{ inventory_hostname }} r{{ memattr_survey_round }}/{{ memattr_survey_rounds }}]
          MEMATTR={{ memattr_survey_record.memattr_address | default('none') }}
          read={{ memattr_survey_record.kernel_read }}
          {%- if memattr_survey_record.memattr_table_version is not none %}
          (version {{ memattr_survey_record.memattr_table_version }})
          {%- endif %}
          MemTotal={{ (memattr_survey_record.mem_total_kb | default(0) | int / 1048576) | round(2) }}G
          of {{ (memattr_survey_record.smbios_installed_kb | default(0) | int / 1048576) | round(0) | int }}G
          => {{ memattr_survey_record.verdict | upper }}
          boot={{ memattr_survey_record.boot_id | default('?') | truncate(9, true, '') }}
      when: memattr_survey_show_progress | bool

    # BRACES. The on-node ledger already survives a reboot; re-fetching it every round
    # means the controller also holds every completed round if the DRIVER dies, without
    # waiting for the end-of-run harvest. One tiny file per host, per-host destination
    # paths, so three concurrent fetches cannot collide.
    - name: "Round {{ memattr_survey_round }} — mirror the ledger to the controller"
      ansible.builtin.fetch:
        src: "{{ memattr_survey_state_file }}"
        dest: "{{ memattr_survey_artifact_dir }}/{{ inventory_hostname }}.rounds.jsonl"
        flat: true
      changed_when: false
      when: memattr_survey_write_artifacts | bool

  rescue:
    - name: "Round {{ memattr_survey_round }} — ABORT this node, continue the others"
      ansible.builtin.set_fact:
        memattr_survey_aborted: true
        memattr_survey_abort_reason: >-
          round {{ memattr_survey_round }} failed:
          {{ ansible_failed_result.msg | default('unknown error') | trim }}

    # The abort must survive the run, not live only in terminal scrollback. Without
    # this the report can see that an arm is SHORT but not WHY, and "0 rounds" reads
    # like a targeting mistake instead of a node that failed to come back.
    - name: "Round {{ memattr_survey_round }} — record the abort as an artifact"
      ansible.builtin.copy:
        dest: "{{ memattr_survey_artifact_dir }}/{{ inventory_hostname }}.aborted.json"
        mode: "0644"
        content: "{{ memattr_survey_abort_record | to_nice_json }}"
      vars:
        memattr_survey_abort_record:
          schema: "memattr-survey/1"
          host: "{{ inventory_hostname }}"
          aborted_at_round: "{{ memattr_survey_round }}"
          target_rounds: "{{ memattr_survey_rounds }}"
          reason: "{{ memattr_survey_abort_reason | trim }}"
          failed_task: "{{ ansible_failed_task.name | default('unknown') }}"
      delegate_to: localhost
      become: false
      changed_when: false
      when: memattr_survey_write_artifacts | bool

    - name: "Round {{ memattr_survey_round }} — announce the abort"
      ansible.builtin.debug:
        msg: >-
          *** SURVEY ABORTED ON {{ inventory_hostname }} at round
          {{ memattr_survey_round }}/{{ memattr_survey_rounds }}:
          {{ memattr_survey_abort_reason }}
          *** The other nodes continue. This node's arm is SHORT — the analysis will
          say so, and its p(bad) interval will be correspondingly wider.