--- # memattr_survey — a BASELINE reboot survey of the EFI Memory Attributes Table lottery # described in infra/PROD-PROVISIONING.md §0b. Read that section first: the mechanism is # settled and is NOT re-derived here. # # THE QUESTION THIS ROLE ANSWERS # §0b establishes WHAT goes wrong (Linux 6.12 reserves everything above a corrupt EFI # Memory Attributes Table, fixed upstream in 6.13) but leaves two things unmeasured: # 1. Is the boot outcome PREDICTED BY WHERE THE TABLE LANDED? Both of node 2's # observed failures were at exactly MEMATTR=0xbeeb0018 and all five observed # successes fleet-wide were at other addresses. If bad outcomes map onto a narrow # address range, the causal chain is closed with no hardware or firmware touched. # 2. Is the fault node-2-specific, or does any node reach the bad path given enough # boots? Nodes 1 and 3 have each been observed exactly twice. # # WHY 20 ROUNDS AND NOT FIVE # Node 2's failure rate is known only to n = 3 (2 bad of 3). The 95% Clopper-Pearson # interval on p(bad) is roughly 0.09-0.99 — against a null that loose a handful of # boots proves nothing at all. §0b's own power note: detecting ELIMINATION of the fault # needs ~20 boots per arm, detecting a mere odds shift ~50. 20 is therefore the floor, # not a round number. # # THIS RUN IS THE BASELINE ARM: NOTHING IS CHANGED. # No BIOS writes, no kernel change, no package installs, no hardware moves. The role # installs nothing and configures nothing; its only write to a node is its own append- # only ledger under /var/lib. Anything that alters the boot chain's allocation # footprint — and per §0b that includes the initrd, the module set and the boot device — # changes the variable under test and voids the arm. # --- Sample size -------------------------------------------------------------------- # Reboot rounds per node. Rounds run in PARALLEL across nodes (they are independent # machines), so 20 rounds costs the wall clock of 20 reboots, not 60. memattr_survey_rounds: 20 # --- Safety interlock ---------------------------------------------------------------- # REBOOTS ARE OPT-IN. Left false the role does everything EXCEPT reboot: safety # preflight, a baseline probe of the current boot, the journal harvest and the analysis. # That is a genuinely useful dry run and it is also how you prove the pipeline works # before committing 45 minutes of wall clock to it. memattr_survey_reboots_authorised: false # The role refuses to run against a host that is not in this group. The prod boxes carry # no k3s and nothing is running on them, which is the entire reason rebooting them is # sanctioned; the dev cluster is live and must never be caught by a stray --limit. memattr_survey_required_group: k3s_prod # Unit names whose presence or activity means something IS running on this node. Any hit # aborts the whole playbook before the first reboot (see tasks/preflight.yml). memattr_survey_forbidden_units: - k3s - k3s-agent - kubelet - containerd - docker # Paths whose existence means k3s has been installed here, even if not currently running. memattr_survey_forbidden_paths: - /etc/rancher/k3s - /var/lib/rancher/k3s - /usr/local/bin/k3s # --- Reboot behaviour ----------------------------------------------------------------- # A node that does not answer within this is declared lost: the role stops rebooting THAT # node and the survey continues on the others (tasks/round.yml). 10 minutes is ~6x the # observed ~90 s POST-to-SSH time on this hardware, so a trip is a real failure and not a # slow boot. memattr_survey_reboot_timeout: 600 memattr_survey_connect_timeout: 20 # Let journald settle before reading the new boot's kernel log. The kernel log itself is # complete long before SSH is up; this is purely margin. memattr_survey_post_reboot_delay: 15 # --- On-node ledger (belt) ------------------------------------------------------------- # Appended one JSON line per round, fsync'd, so a round survives the next reboot even if # the controller dies. This is what makes the survey resumable: re-running skips rounds # already recorded here rather than starting over. memattr_survey_state_dir: /var/lib/memattr-survey memattr_survey_state_file: "{{ memattr_survey_state_dir }}/rounds.jsonl" # --- Controller artifacts (braces) ------------------------------------------------------ # Same convention as roles/node_health and conformance/_artifacts/: generated output under # a `_artifacts/` directory that .gitignore excludes, never committed. # .rounds.jsonl the per-round ledger, re-fetched AFTER EVERY ROUND so a crash # mid-survey still leaves every completed round on the controller # .boots.json the journal harvest — every retained boot re-derived in one # pass. THIS is the authoritative record; the ledger is the backup. # summary.md the analysis (also printed) memattr_survey_artifact_dir: "{{ playbook_dir }}/../_artifacts/memattr-survey" memattr_survey_write_artifacts: true # Print each round's one-line result as it happens. A 45-minute unattended run with no # output is indistinguishable from a hung one. memattr_survey_show_progress: true