#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated # Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the # replicated tier." — spec docs/04-storage-and-state.md) # # WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY # ---------------------------------------------------------------------------------------- # `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with # `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2). # Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is # resolved", and each of them said, correctly, that adding `class: bulk` before then would # leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing # behind it, and the only thing preventing a multi-terabyte media library from being # authored onto the 3-replica `longhorn` class was four hand-written comments. # # That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk # storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the # replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on # `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the # tier fills up. Distinctness is therefore a property worth asserting statically, on every # gate run, rather than trusting to review. # # It also pins the four decisions that make the STATIC form of this tier safe, each of # which has a plausible-looking "simplification" that silently breaks something: # # * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has # consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an # inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl # delete pvc` a data-loss operation. # * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The # pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of # existing data was never attached. This is the single most deceptive failure available # in this tier, so `storageClassName` must be explicitly empty and binding must be # pinned from BOTH ends (claimRef + volumeName). # * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or # EIO handed to the application. For a media library that is silent corruption. `hard` # hangs instead — recoverable, where truncated is not. # * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port # (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is # a FIREWALL fact as much as a storage one: prod reaches this server through pfSense, # and the number of ports the operator must open is a consequence of this field. # # AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6). # Sonarr and Radarr import by hardlinking or atomically renaming a finished download into # the library. Both are same-filesystem operations, and to the kernel "same filesystem" # means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`) # looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no # matter that the server knows they are one dataset. The arrs do not fail loudly on that: # they degrade to copy-then-delete, rewriting every imported file over NFS and destroying # the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is # gone". A reviewer cannot see it, so this check does: one export, at most one mount, per # workload. # # GIVEN the repository as authored, # WHEN every `class: bulk` storage entry in catalog/ is compared against the manifests # render.py produces for it (read from the committed apps/ goldens, i.e. what Flux # actually reconciles — not from the renderer, which would only agree with itself), # THEN (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC # storageClassName is explicitly empty — never `longhorn`, never any class the # repo declares with the Longhorn provisioner — and its PV's source is an # EXTERNAL one (nfs), not a Longhorn volume; # (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the # bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own # replicated volume. This is the requirement stated as arithmetic: the two tiers # exist because the data has two magnitudes, so a "bulk" volume sized like a # config PVC means the strategy was named and not actually applied; # (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its # PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and # the two agree — so no other claim can win a race, and nothing can provision an # empty replacement; # (4) RECLAIM: every bulk PV is `Retain`; # (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an # `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`; # (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above); # (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH # `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount, # and its access mode is ReadOnlyMany; a writable one renders NEITHER and is # ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed # by several namespaces and several nodes, which ReadWriteOnce cannot express; # (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk # PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that # knob has a base default; and EVERY environment overlay supplies it — so the two # clusters, which reach this storage by different addresses, cannot be served by # one hardcoded value; # AND (9) LOAD-BEARING + NON-VACUOUS: this check DECLARES `conformance.home.k3s/scope: # whole-repo`, so ci/l0.sh runs it unconditionally via --include-scope, and # ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT # empty — a repo with zero bulk volumes fails here rather than passing trivially, # because a check that is satisfiable by deleting the feature proves nothing. # # WHAT THIS DOES *NOT* PROVE (honest gaps) # - Nothing about RUNTIME. Not that the server is reachable, not that the export permits # these clients, not that the PVC binds, not that a write succeeds. The uid analysis in # docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the # in-cluster proof — they need a cluster and are not run here. # - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While # prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md # §7); catching that needs the placeholder guard, which is absent from main. # - Read-only for the media tree is a CLIENT-side guarantee, and this check can only # assert the client side. The export serves a reader and two writers from the same node # addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5). # - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It # catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a # static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit. # - Assertion (6) reasons over ONE workload's mounts. It cannot see two different # services hardlinking between each other's mounts, which is not a thing that happens. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root PYTHON="${PYTHON:-python3}" cd "${ROOT}" if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2 exit 3 fi "${PYTHON}" - "${ROOT}" <<'PY' import re import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) FAILURES: list[str] = [] CHECKS = 0 def check(cond, msg): global CHECKS CHECKS += 1 if not cond: FAILURES.append(msg) def bail(msg): """A failure that makes the rest of the check meaningless: report and stop.""" print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr) raise SystemExit(1) def qty_to_bytes(q): """Kubernetes quantity -> bytes. Only the binary suffixes the schema permits.""" m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip()) if not m: return None return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)] # ===================================================================================== # The replicated tier, named by the repo rather than by this file. Any StorageClass the # repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1); # deriving it means adding a new Longhorn class cannot create a blind spot here. # ===================================================================================== REPLICATED_CLASSES: set[str] = set() LONGHORN_DIR = ROOT / "platform" / "longhorn" if LONGHORN_DIR.is_dir(): for f in sorted(LONGHORN_DIR.rglob("*.yaml")): for doc in yaml.safe_load_all(f.read_text()): if isinstance(doc, dict) and doc.get("kind") == "StorageClass": if "longhorn" in str(doc.get("provisioner", "")): REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", "")) # `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in # the set even if platform/longhorn/ is not present in this checkout. REPLICATED_CLASSES.add("longhorn") REPLICATED_CLASSES.discard("") # ===================================================================================== # The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired # with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling # the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is # a separate gate stage. Asking the renderer would only prove it agrees with itself. # ===================================================================================== sds: dict[str, dict] = {} for f in sorted((ROOT / "catalog").glob("*.yaml")): doc = yaml.safe_load(f.read_text()) if isinstance(doc, dict) and doc.get("name"): sds[doc["name"]] = doc if not sds: bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier") # Every rendered app-tier document, indexed by (kind, name). rendered: dict[tuple, dict] = {} for f in sorted((ROOT / "apps").rglob("*.yaml")): if f.name == "kustomization.yaml" or ".sops." in f.name: continue try: docs = list(yaml.safe_load_all(f.read_text())) except yaml.YAMLError: continue for doc in docs: if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"): rendered[(doc["kind"], doc["metadata"]["name"])] = doc # (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2). corpus: list[tuple[str, dict]] = [] replicated_bytes: dict[str, list[int]] = {} for svc, sd in sds.items(): for vol in sd.get("storage") or []: cls = vol.get("class", "longhorn") if cls == "bulk": corpus.append((svc, vol)) else: b = qty_to_bytes(vol.get("size")) if b: replicated_bytes.setdefault(svc, []).append(b) # ------------------------------------------------------------------- (9) NON-VACUOUS # A corpus of zero would make every loop below trivially satisfied. The bulk tier is # DELIVERED; a change that removes the last bulk volume must fail here and be argued for, # not sail through on an empty iteration. check(len(corpus) > 0, "NO `class: bulk` storage entry exists in catalog/ — this check would then pass " "vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME " "stub for six roadmap phases. If the tier is genuinely being retired, retire this " "check in the same change and say so; do not let it pass on an empty corpus") if not corpus: bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)") inventory: list[str] = [] for svc, vol in corpus: vname = vol.get("name", "") label = f"{svc}/{vname}" obj = f"{svc}-{vname}" ro = bool(vol.get("readOnly")) # `export` is what the schema requires of a bulk entry; without it there is nothing to # attach and the rest of the assertions have no subject. export = vol.get("export") check(bool(export) and str(export).startswith("/"), f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an " f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)") pv = rendered.get(("PersistentVolume", obj)) pvc = rendered.get(("PersistentVolumeClaim", obj)) check(pv is not None, f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk " f"entry that renders no PV has NO external storage behind it — the pod would get " f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory") check(pvc is not None, f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens") if pv is None or pvc is None: continue pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {} # ------------------------------------------------------ (1) DISTINCT FROM REPLICATED for what, spec in (("PV", pvs), ("PVC", pvcs)): sc = spec.get("storageClassName") check(sc not in REPLICATED_CLASSES, f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares " f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier " f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not " f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)") check(sc == "", f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY " f"string. Empty is what disables dynamic provisioning AND the default-class " f"annotation. Any named class risks a provisioner handing out a fresh EMPTY " f"directory that mounts and binds cleanly while the real data is never attached") check("nfs" in pvs, f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole " f"premise is that durability lives OUTSIDE the cluster (docs/04)") # ------------------------------------------------------------ (4) RECLAIM POLICY check(pvs.get("persistentVolumeReclaimPolicy") == "Retain", f"{label}: bulk PV reclaim policy is " f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists " f"the cluster and has consumers outside it; anything but Retain makes `kubectl " f"delete pvc` a data-loss operation") # ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS check(pvcs.get("volumeName") == obj, f"{label}: bulk PVC does not name its PV via volumeName (got " f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be " f"satisfied by some OTHER volume of matching size and mode") cref = pvs.get("claimRef") or {} check(cref.get("name") == obj and cref.get("namespace") == svc, f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} " f"— pre-binding must be pinned from BOTH ends or another PVC can win the race") # ------------------------------------------------------------- (5) MOUNT SAFETY mopts = [str(o) for o in (pvs.get("mountOptions") or [])] check(bool(mopts), f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS " f"version are whatever the client happens to default to, and both are decisions " f"with consequences (silent short reads; how many firewall ports must open)") check("hard" in mopts, f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount " f"hands a server hiccup to the application as a short read or EIO — silent " f"corruption for a media library. `hard` hangs instead, which is recoverable") check("soft" not in mopts, f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the " f"one option that turns an outage into damaged data") vers = [o for o in mopts if o.startswith("nfsvers=")] check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]), f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). " f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port " f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the " f"operator has to open in pfSense for prod to reach this server at all") # ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT want_mode = "ReadOnlyMany" if ro else "ReadWriteMany" for what, spec in (("PV", pvs), ("PVC", pvcs)): modes = spec.get("accessModes") or [] check(modes == [want_mode], f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk " f"volume is a `...Many` (one export, several namespaces, several nodes — " f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write " f"posture stays legible in `kubectl get pv`") nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly")) check(nfs_ro == ro, f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. " f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared " f"posture is not the one the kernel enforces") # The volumeMount half, read out of the rendered Deployment. dep = rendered.get(("Deployment", svc)) check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against") if dep is not None: containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or [] # The app container is containers[0] by the renderer's contract (a forward-auth # sidecar is appended AFTER it). mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else [] check(len(mounts) == 1, f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, " f"expected exactly 1 — a bulk volume that is declared and never mounted is a " f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)") if len(mounts) == 1: m = mounts[0] check(bool(m.get("readOnly")) == ro, f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the " f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, " f"so that dropping one still leaves the other") check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"), f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the " f"declared subPath {vol.get('subPath')!r}") # ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV server = (pvs.get("nfs") or {}).get("server") m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or "")) check(m is not None, f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars " f"token, not a literal address. dev and prod reach this storage host by DIFFERENT " f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct " f"in at most one environment and silently wrong in the other (R-ENV-02)") cap = (pvs.get("capacity") or {}).get("storage") req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage") check(cap is not None and qty_to_bytes(cap) is not None, f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, " f"and it is what makes `kubectl get pv` report something true about this tier") check(cap == req, f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the " f"capacity never binds") inventory.append( f" {label:<28} export={export} {'ro' if ro else 'rw'} {want_mode:<14} " f"cap={cap} server={server} opts={','.join(mopts)}" ) # ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes cap_b = qty_to_bytes(cap) reps = replicated_bytes.get(svc) or [] if cap_b and reps: biggest = max(reps) # ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed: # the media library is ~460x its service's config volume, and the Nextcloud archive # ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate # bulk volume, and this check would be teaching people to weaken it. 10x cannot be # met by accident and still catches the regression that matters: a "bulk" volume # sized like a config PVC, i.e. the strategy named but never applied. check(cap_b >= 10 * biggest, f"{label}: bulk capacity {cap} is less than 10x this service's largest " f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the " f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps " f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A " f"bulk volume sized like a config volume means the tier was named, not applied") # ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT # The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod. by_service: dict[str, list[dict]] = {} for svc, vol in corpus: by_service.setdefault(svc, []).append(vol) for svc, vols in sorted(by_service.items()): seen: dict[str, str] = {} for vol in vols: exp = vol.get("export") prev = seen.get(exp) check(prev is None, f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and " f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the " f"kernel, so link() between them fails EXDEV even though the server knows " f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they " f"degrade to copy-then-delete, rewriting every imported file over NFS and " f"destroying the seeding copy. Mount the common ancestor ONCE instead " f"(docs/BULK-NFS-STORAGE.md §4)") seen[exp] = vol.get("name") # ------------------------------------- (8b) the knob has a base default AND every overlay tokens = set() for _svc, vol in corpus: pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}")) if pv: mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or "")) if mm: tokens.add(mm.group(1)) cv = ROOT / "clusters" / "base" / "cluster-vars.yaml" base_keys: set[str] = set() if cv.is_file(): doc = yaml.safe_load(cv.read_text()) or {} base_keys = set((doc.get("data") or {}).keys()) overlays = sorted( d for d in (ROOT / "clusters").iterdir() if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file() ) if (ROOT / "clusters").is_dir() else [] for tok in sorted(tokens): check(tok in base_keys, f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no " f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without " f"it the token ships into the cluster as a literal server name") for ov in overlays: text = (ov / "kustomization.yaml").read_text() check(f"/data/{tok}" in text, f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV " f"value by this repo's convention, so an overlay that omits it silently " f"inherits an address on the WRONG SIDE of pfSense — a mount that either " f"never connects or connects to something unintended (R-ENV-02)") # --------------------------------------------------------------- (9) LOAD-BEARING L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else "" GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else "" # THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism. # It used to assert `"bulk-tier-distinct-from-replicated/check.sh" in L0`: ci/l0.sh carried a hand-written # "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone # (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED # to write it). The same property is now asserted two ways: this test DECLARES # `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into # the run via `--include-scope`. Neither half alone is enough, so both are checked here; the # mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than # passing) is proven end-to-end and falsifiably by # conformance/tests/l0-static/whole-repo-invariants-wired. SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "bulk-tier-distinct-from-replicated" / "chainsaw-test.yaml" SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else "" check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None, "bulk-tier-distinct-from-replicated/chainsaw-test.yaml does not declare " 'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only ' "when a change happens to touch one of the requirements it is labelled with. " "The path map keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement, so moving a multi-terabyte bulk volume onto the 3-replica tier could attribute no requirement at all and be gated by nothing") check("--include-scope" in L0, "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is " "a purely requirement-scoped run again and the scope annotation is inert decoration — " "this check would then run only by coincidence") check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE), "ci/gate.sh does not set the merge block when the l0 stage fails") # ------------------------------------------------------------------------- report print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):") for line in inventory: print(line) print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): " f"{', '.join(sorted(REPLICATED_CLASSES))}") print(f"bulk volumes: {len(corpus)} distinct exports: " f"{len({v.get('export') for _s, v in corpus})} " f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}") # A floor on the assertion count, for the same reason the corpus may not be empty: a pass # must not be able to degenerate into "nothing was actually asked". MIN_CHECKS = 40 check(CHECKS >= MIN_CHECKS, f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not " f"evidence; the corpus or the extraction has degraded") print() if FAILURES: print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)", file=sys.stderr) for f in FAILURES: print(f" - {f}", file=sys.stderr) raise SystemExit(1) print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)") PY