--- # Safety gate. Runs with any_errors_fatal at the play level (playbooks/memattr-survey.yml), # so a violation on ANY node stops the entire playbook before a single reboot — the # correct posture when the failure mode is "rebooted a machine that was doing something". # # VERIFY, DO NOT ASSUME. The premise that makes these reboots safe is that prod carries no # k3s and runs nothing. That premise is checked here on every run rather than trusted from # a document, because the document and the machine drift independently. - name: Assert the target host is in the prod group # The dev cluster is LIVE. This is the one check that a mistyped --limit or a copied # command line cannot slip past: membership is read from the inventory, not the pattern. ansible.builtin.assert: that: - memattr_survey_required_group in (group_names | default([])) fail_msg: >- REFUSING TO SURVEY {{ inventory_hostname }}: it is not a member of '{{ memattr_survey_required_group }}' (groups: {{ group_names | default([]) | join(', ') }}). This role reboots its targets. Point it at prod, or override memattr_survey_required_group deliberately. quiet: true - name: Check whether any cluster or container runtime unit is active # `is-active` returns non-zero for inactive/unknown units, which is the expected answer # here, so the rc is data and not a failure. ansible.builtin.command: argv: "{{ ['systemctl', 'is-active'] + memattr_survey_forbidden_units }}" register: memattr_survey_units changed_when: false failed_when: false - name: Check whether k3s has been installed on this node ansible.builtin.stat: path: "{{ item }}" loop: "{{ memattr_survey_forbidden_paths }}" register: memattr_survey_paths changed_when: false - name: Assert nothing is running on this node ansible.builtin.assert: that: - memattr_survey_units.stdout_lines | reject('equalto', 'inactive') | reject('equalto', 'unknown') | list | length == 0 - memattr_survey_paths.results | selectattr('stat.exists') | list | length == 0 fail_msg: >- REFUSING TO REBOOT {{ inventory_hostname }} — it is not idle. units {{ memattr_survey_forbidden_units | zip(memattr_survey_units.stdout_lines) | map('join', '=') | join(' ') }}; present paths: {{ memattr_survey_paths.results | selectattr('stat.exists') | map(attribute='item') | list | join(', ') | default('none', true) }}. The survey is only sanctioned because these boxes carry no k3s and run nothing. quiet: true - name: Clear any abort marker from a previous run # A marker left by a run that has since been resumed successfully would make this run's # report describe the LAST failure rather than this one. The marker means "this run # aborted", so it is cleared at the start of every run and re-written only if it does. ansible.builtin.file: path: "{{ memattr_survey_artifact_dir }}/{{ inventory_hostname }}.aborted.json" state: absent delegate_to: localhost become: false changed_when: false when: memattr_survey_write_artifacts | bool - name: Assert forks allows every node to be surveyed concurrently # The survey play runs `strategy: free` so the nodes reboot in parallel, but forks is a # hard cap on that: with fewer forks than hosts the survey quietly serialises and takes # a multiple of the wall clock it should. Cheap to satisfy (`-f `), so assert rather # than discover it three hours later. ansible.builtin.assert: that: - (lookup('ansible.builtin.config', 'DEFAULT_FORKS') | int) >= (ansible_play_hosts_all | length) fail_msg: >- forks={{ lookup('ansible.builtin.config', 'DEFAULT_FORKS') }} but this survey targets {{ ansible_play_hosts_all | length }} hosts. The rounds would serialise. Re-run with -f {{ ansible_play_hosts_all | length }} (or higher). quiet: true run_once: true - name: Read the node's resume state and baseline boot # One call does double duty: it reports how far a previous run got (so this one resumes) # and, with --round 0, records the boot we found the node on. Round 0 is a free data # point — it is a boot nobody chose, so it is not selected for by the survey. ansible.builtin.script: cmd: >- memattr_probe.py --status {{ memattr_survey_state_file | quote }} executable: /usr/bin/python3 register: memattr_survey_status_raw changed_when: false - name: Record the resume point ansible.builtin.set_fact: memattr_survey_status: "{{ memattr_survey_status_raw.stdout | from_json }}" memattr_survey_aborted: false memattr_survey_abort_reason: "" - name: Assert the probe schema is the one this role understands ansible.builtin.assert: that: - memattr_survey_status.schema == 'memattr-survey/1' fail_msg: >- Probe returned schema {{ memattr_survey_status.schema | default('') }}; this role understands memattr-survey/1. Role and probe are out of step. quiet: true - name: Probe the boot the node is currently on # On a FIRST run this is stamped round 0 and appended: the boot the node was already # sitting on is a free, unselected data point. On a RESUME it is read but NOT appended, # because that boot is already round N of the interrupted run and double-counting it # would bias the sample toward whatever outcome happened to survive the interruption. ansible.builtin.script: cmd: >- memattr_probe.py {% if memattr_survey_status.completed_rounds | int == 0 %} --round 0 --append {{ memattr_survey_state_file | quote }} {% endif %} executable: /usr/bin/python3 register: memattr_survey_baseline_raw # The append is the experiment's own ledger, not a change to the node under test: this # role reports `changed` only for things that could affect a boot, and a file written # under /var/lib long after the kernel parsed the table cannot. changed_when: false - name: Parse the baseline record ansible.builtin.set_fact: memattr_survey_baseline: "{{ memattr_survey_baseline_raw.stdout | from_json }}" - name: Assert the journal is PERSISTENT # The end-of-run harvest is the authoritative record precisely because it re-derives # every boot from the journal. On a volatile journal (/run/log/journal only) each # reboot erases the evidence and the harvest would silently return one boot — a survey # that looks complete and holds nothing. Verified, not assumed. ansible.builtin.assert: that: - memattr_survey_baseline.journal_storage.persistent | bool fail_msg: >- {{ inventory_hostname }} has NO persistent journal (/var/log/journal missing). Every reboot would erase the record this survey depends on. Refusing to run. quiet: true - name: Report the starting state ansible.builtin.debug: msg: >- {{ inventory_hostname }}: idle, prod, persistent journal. kernel {{ memattr_survey_baseline.kernel_release | default('?') }}, installed {{ (memattr_survey_baseline.smbios_installed_kb | default(0) | int / 1048576) | round(1) }} GiB. baseline boot MEMATTR={{ memattr_survey_baseline.memattr_address | default('none') }} kernel_read={{ memattr_survey_baseline.kernel_read | default('?') }} verdict={{ memattr_survey_baseline.verdict | default('?') | upper }}. resuming from round {{ memattr_survey_status.completed_rounds }} of {{ memattr_survey_rounds }}.