home-network-k3s-impl/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated
# Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the
#                      replicated tier." — spec docs/04-storage-and-state.md)
#
# WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY
# ----------------------------------------------------------------------------------------
# `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2).
# Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is
# resolved", and each of them said, correctly, that adding `class: bulk` before then would
# leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing
# behind it, and the only thing preventing a multi-terabyte media library from being
# authored onto the 3-replica `longhorn` class was four hand-written comments.
#
# That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk
# storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the
# replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on
# `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the
# tier fills up. Distinctness is therefore a property worth asserting statically, on every
# gate run, rather than trusting to review.
#
# It also pins the four decisions that make the STATIC form of this tier safe, each of
# which has a plausible-looking "simplification" that silently breaks something:
#
#   * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has
#     consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an
#     inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl
#     delete pvc` a data-loss operation.
#   * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The
#     pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of
#     existing data was never attached. This is the single most deceptive failure available
#     in this tier, so `storageClassName` must be explicitly empty and binding must be
#     pinned from BOTH ends (claimRef + volumeName).
#   * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or
#     EIO handed to the application. For a media library that is silent corruption. `hard`
#     hangs instead — recoverable, where truncated is not.
#   * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port
#     (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is
#     a FIREWALL fact as much as a storage one: prod reaches this server through pfSense,
#     and the number of ports the operator must open is a consequence of this field.
#
# AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6).
# Sonarr and Radarr import by hardlinking or atomically renaming a finished download into
# the library. Both are same-filesystem operations, and to the kernel "same filesystem"
# means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`)
# looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no
# matter that the server knows they are one dataset. The arrs do not fail loudly on that:
# they degrade to copy-then-delete, rewriting every imported file over NFS and destroying
# the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is
# gone". A reviewer cannot see it, so this check does: one export, at most one mount, per
# workload.
#
# GIVEN the repository as authored,
# WHEN  every `class: bulk` storage entry in catalog/ is compared against the manifests
#       render.py produces for it (read from the committed apps/ goldens, i.e. what Flux
#       actually reconciles — not from the renderer, which would only agree with itself),
# THEN  (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC
#           storageClassName is explicitly empty — never `longhorn`, never any class the
#           repo declares with the Longhorn provisioner — and its PV's source is an
#           EXTERNAL one (nfs), not a Longhorn volume;
#       (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the
#           bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own
#           replicated volume. This is the requirement stated as arithmetic: the two tiers
#           exist because the data has two magnitudes, so a "bulk" volume sized like a
#           config PVC means the strategy was named and not actually applied;
#       (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its
#           PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and
#           the two agree — so no other claim can win a race, and nothing can provision an
#           empty replacement;
#       (4) RECLAIM: every bulk PV is `Retain`;
#       (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an
#           `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`;
#       (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above);
#       (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH
#           `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount,
#           and its access mode is ReadOnlyMany; a writable one renders NEITHER and is
#           ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed
#           by several namespaces and several nodes, which ReadWriteOnce cannot express;
#       (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk
#           PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that
#           knob has a base default; and EVERY environment overlay supplies it — so the two
#           clusters, which reach this storage by different addresses, cannot be served by
#           one hardcoded value;
#   AND (9) LOAD-BEARING + NON-VACUOUS: this check DECLARES `conformance.home.k3s/scope:
#           whole-repo`, so ci/l0.sh runs it unconditionally via --include-scope, and
#           ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT
#           empty — a repo with zero bulk volumes fails here rather than passing trivially,
#           because a check that is satisfiable by deleting the feature proves nothing.
#
# WHAT THIS DOES *NOT* PROVE (honest gaps)
#   - Nothing about RUNTIME. Not that the server is reachable, not that the export permits
#     these clients, not that the PVC binds, not that a write succeeds. The uid analysis in
#     docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the
#     in-cluster proof — they need a cluster and are not run here.
#   - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While
#     prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md
#     §7); catching that needs the placeholder guard, which is absent from main.
#   - Read-only for the media tree is a CLIENT-side guarantee, and this check can only
#     assert the client side. The export serves a reader and two writers from the same node
#     addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5).
#   - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It
#     catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a
#     static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit.
#   - Assertion (6) reasons over ONE workload's mounts. It cannot see two different
#     services hardlinking between each other's mounts, which is not a thing that happens.
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
PYTHON="${PYTHON:-python3}"

cd "${ROOT}"

if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then
  echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2
  exit 3
fi

"${PYTHON}" - "${ROOT}" <<'PY'
import re
import sys
from pathlib import Path

try:
    import yaml
except ImportError:                                  # pragma: no cover
    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
          file=sys.stderr)
    raise SystemExit(1)

ROOT = Path(sys.argv[1])
FAILURES: list[str] = []
CHECKS = 0


def check(cond, msg):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(msg)


def bail(msg):
    """A failure that makes the rest of the check meaningless: report and stop."""
    print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr)
    raise SystemExit(1)


def qty_to_bytes(q):
    """Kubernetes quantity -> bytes. Only the binary suffixes the schema permits."""
    m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip())
    if not m:
        return None
    return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)]


# =====================================================================================
# The replicated tier, named by the repo rather than by this file. Any StorageClass the
# repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1);
# deriving it means adding a new Longhorn class cannot create a blind spot here.
# =====================================================================================
REPLICATED_CLASSES: set[str] = set()
LONGHORN_DIR = ROOT / "platform" / "longhorn"
if LONGHORN_DIR.is_dir():
    for f in sorted(LONGHORN_DIR.rglob("*.yaml")):
        for doc in yaml.safe_load_all(f.read_text()):
            if isinstance(doc, dict) and doc.get("kind") == "StorageClass":
                if "longhorn" in str(doc.get("provisioner", "")):
                    REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", ""))
# `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in
# the set even if platform/longhorn/ is not present in this checkout.
REPLICATED_CLASSES.add("longhorn")
REPLICATED_CLASSES.discard("")

# =====================================================================================
# The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired
# with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling
# the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is
# a separate gate stage. Asking the renderer would only prove it agrees with itself.
# =====================================================================================
sds: dict[str, dict] = {}
for f in sorted((ROOT / "catalog").glob("*.yaml")):
    doc = yaml.safe_load(f.read_text())
    if isinstance(doc, dict) and doc.get("name"):
        sds[doc["name"]] = doc

if not sds:
    bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier")

# Every rendered app-tier document, indexed by (kind, name).
rendered: dict[tuple, dict] = {}
for f in sorted((ROOT / "apps").rglob("*.yaml")):
    if f.name == "kustomization.yaml" or ".sops." in f.name:
        continue
    try:
        docs = list(yaml.safe_load_all(f.read_text()))
    except yaml.YAMLError:
        continue
    for doc in docs:
        if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"):
            rendered[(doc["kind"], doc["metadata"]["name"])] = doc

# (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2).
corpus: list[tuple[str, dict]] = []
replicated_bytes: dict[str, list[int]] = {}
for svc, sd in sds.items():
    for vol in sd.get("storage") or []:
        cls = vol.get("class", "longhorn")
        if cls == "bulk":
            corpus.append((svc, vol))
        else:
            b = qty_to_bytes(vol.get("size"))
            if b:
                replicated_bytes.setdefault(svc, []).append(b)

# ------------------------------------------------------------------- (9) NON-VACUOUS
# A corpus of zero would make every loop below trivially satisfied. The bulk tier is
# DELIVERED; a change that removes the last bulk volume must fail here and be argued for,
# not sail through on an empty iteration.
check(len(corpus) > 0,
      "NO `class: bulk` storage entry exists in catalog/ — this check would then pass "
      "vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME "
      "stub for six roadmap phases. If the tier is genuinely being retired, retire this "
      "check in the same change and say so; do not let it pass on an empty corpus")
if not corpus:
    bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)")

inventory: list[str] = []

for svc, vol in corpus:
    vname = vol.get("name", "<unnamed>")
    label = f"{svc}/{vname}"
    obj = f"{svc}-{vname}"
    ro = bool(vol.get("readOnly"))

    # `export` is what the schema requires of a bulk entry; without it there is nothing to
    # attach and the rest of the assertions have no subject.
    export = vol.get("export")
    check(bool(export) and str(export).startswith("/"),
          f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an "
          f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)")

    pv = rendered.get(("PersistentVolume", obj))
    pvc = rendered.get(("PersistentVolumeClaim", obj))
    check(pv is not None,
          f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk "
          f"entry that renders no PV has NO external storage behind it — the pod would get "
          f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory")
    check(pvc is not None,
          f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens")
    if pv is None or pvc is None:
        continue

    pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {}

    # ------------------------------------------------------ (1) DISTINCT FROM REPLICATED
    for what, spec in (("PV", pvs), ("PVC", pvcs)):
        sc = spec.get("storageClassName")
        check(sc not in REPLICATED_CLASSES,
              f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares "
              f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier "
              f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not "
              f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)")
        check(sc == "",
              f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY "
              f"string. Empty is what disables dynamic provisioning AND the default-class "
              f"annotation. Any named class risks a provisioner handing out a fresh EMPTY "
              f"directory that mounts and binds cleanly while the real data is never attached")
    check("nfs" in pvs,
          f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole "
          f"premise is that durability lives OUTSIDE the cluster (docs/04)")

    # ------------------------------------------------------------ (4) RECLAIM POLICY
    check(pvs.get("persistentVolumeReclaimPolicy") == "Retain",
          f"{label}: bulk PV reclaim policy is "
          f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists "
          f"the cluster and has consumers outside it; anything but Retain makes `kubectl "
          f"delete pvc` a data-loss operation")

    # ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS
    check(pvcs.get("volumeName") == obj,
          f"{label}: bulk PVC does not name its PV via volumeName (got "
          f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be "
          f"satisfied by some OTHER volume of matching size and mode")
    cref = pvs.get("claimRef") or {}
    check(cref.get("name") == obj and cref.get("namespace") == svc,
          f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} "
          f"— pre-binding must be pinned from BOTH ends or another PVC can win the race")

    # ------------------------------------------------------------- (5) MOUNT SAFETY
    mopts = [str(o) for o in (pvs.get("mountOptions") or [])]
    check(bool(mopts),
          f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS "
          f"version are whatever the client happens to default to, and both are decisions "
          f"with consequences (silent short reads; how many firewall ports must open)")
    check("hard" in mopts,
          f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount "
          f"hands a server hiccup to the application as a short read or EIO — silent "
          f"corruption for a media library. `hard` hangs instead, which is recoverable")
    check("soft" not in mopts,
          f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the "
          f"one option that turns an outage into damaged data")
    vers = [o for o in mopts if o.startswith("nfsvers=")]
    check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]),
          f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). "
          f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port "
          f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the "
          f"operator has to open in pfSense for prod to reach this server at all")

    # ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT
    want_mode = "ReadOnlyMany" if ro else "ReadWriteMany"
    for what, spec in (("PV", pvs), ("PVC", pvcs)):
        modes = spec.get("accessModes") or []
        check(modes == [want_mode],
              f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk "
              f"volume is a `...Many` (one export, several namespaces, several nodes — "
              f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write "
              f"posture stays legible in `kubectl get pv`")
    nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly"))
    check(nfs_ro == ro,
          f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. "
          f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared "
          f"posture is not the one the kernel enforces")

    # The volumeMount half, read out of the rendered Deployment.
    dep = rendered.get(("Deployment", svc))
    check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against")
    if dep is not None:
        containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or []
        # The app container is containers[0] by the renderer's contract (a forward-auth
        # sidecar is appended AFTER it).
        mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else []
        check(len(mounts) == 1,
              f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, "
              f"expected exactly 1 — a bulk volume that is declared and never mounted is a "
              f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)")
        if len(mounts) == 1:
            m = mounts[0]
            check(bool(m.get("readOnly")) == ro,
                  f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the "
                  f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, "
                  f"so that dropping one still leaves the other")
            check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"),
                  f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the "
                  f"declared subPath {vol.get('subPath')!r}")

    # ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV
    server = (pvs.get("nfs") or {}).get("server")
    m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or ""))
    check(m is not None,
          f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars "
          f"token, not a literal address. dev and prod reach this storage host by DIFFERENT "
          f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct "
          f"in at most one environment and silently wrong in the other (R-ENV-02)")

    cap = (pvs.get("capacity") or {}).get("storage")
    req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage")
    check(cap is not None and qty_to_bytes(cap) is not None,
          f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, "
          f"and it is what makes `kubectl get pv` report something true about this tier")
    check(cap == req,
          f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the "
          f"capacity never binds")

    inventory.append(
        f"  {label:<28} export={export}  {'ro' if ro else 'rw'}  {want_mode:<14} "
        f"cap={cap}  server={server}  opts={','.join(mopts)}"
    )

    # ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes
    cap_b = qty_to_bytes(cap)
    reps = replicated_bytes.get(svc) or []
    if cap_b and reps:
        biggest = max(reps)
        # ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed:
        # the media library is ~460x its service's config volume, and the Nextcloud archive
        # ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate
        # bulk volume, and this check would be teaching people to weaken it. 10x cannot be
        # met by accident and still catches the regression that matters: a "bulk" volume
        # sized like a config PVC, i.e. the strategy named but never applied.
        check(cap_b >= 10 * biggest,
              f"{label}: bulk capacity {cap} is less than 10x this service's largest "
              f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the "
              f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps "
              f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A "
              f"bulk volume sized like a config volume means the tier was named, not applied")

# ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT
# The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod.
by_service: dict[str, list[dict]] = {}
for svc, vol in corpus:
    by_service.setdefault(svc, []).append(vol)
for svc, vols in sorted(by_service.items()):
    seen: dict[str, str] = {}
    for vol in vols:
        exp = vol.get("export")
        prev = seen.get(exp)
        check(prev is None,
              f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and "
              f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the "
              f"kernel, so link() between them fails EXDEV even though the server knows "
              f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they "
              f"degrade to copy-then-delete, rewriting every imported file over NFS and "
              f"destroying the seeding copy. Mount the common ancestor ONCE instead "
              f"(docs/BULK-NFS-STORAGE.md §4)")
        seen[exp] = vol.get("name")

# ------------------------------------- (8b) the knob has a base default AND every overlay
tokens = set()
for _svc, vol in corpus:
    pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}"))
    if pv:
        mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or ""))
        if mm:
            tokens.add(mm.group(1))

cv = ROOT / "clusters" / "base" / "cluster-vars.yaml"
base_keys: set[str] = set()
if cv.is_file():
    doc = yaml.safe_load(cv.read_text()) or {}
    base_keys = set((doc.get("data") or {}).keys())

overlays = sorted(
    d for d in (ROOT / "clusters").iterdir()
    if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file()
) if (ROOT / "clusters").is_dir() else []

for tok in sorted(tokens):
    check(tok in base_keys,
          f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no "
          f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without "
          f"it the token ships into the cluster as a literal server name")
    for ov in overlays:
        text = (ov / "kustomization.yaml").read_text()
        check(f"/data/{tok}" in text,
              f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV "
              f"value by this repo's convention, so an overlay that omits it silently "
              f"inherits an address on the WRONG SIDE of pfSense — a mount that either "
              f"never connects or connects to something unintended (R-ENV-02)")

# --------------------------------------------------------------- (9) LOAD-BEARING
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism.
# It used to assert `"bulk-tier-distinct-from-replicated/check.sh" in L0`: ci/l0.sh carried a hand-written
# "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone
# (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED
# to write it). The same property is now asserted two ways: this test DECLARES
# `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into
# the run via `--include-scope`. Neither half alone is enough, so both are checked here; the
# mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than
# passing) is proven end-to-end and falsifiably by
# conformance/tests/l0-static/whole-repo-invariants-wired.
SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "bulk-tier-distinct-from-replicated" / "chainsaw-test.yaml"
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None,
      "bulk-tier-distinct-from-replicated/chainsaw-test.yaml does not declare "
      'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only '
      "when a change happens to touch one of the requirements it is labelled with. "
      "The path map keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement, so moving a multi-terabyte bulk volume onto the 3-replica tier could attribute no requirement at all and be gated by nothing")
check("--include-scope" in L0,
      "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is "
      "a purely requirement-scoped run again and the scope annotation is inert decoration — "
      "this check would then run only by coincidence")
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
      "ci/gate.sh does not set the merge block when the l0 stage fails")

# ------------------------------------------------------------------------- report
print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):")
for line in inventory:
    print(line)
print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): "
      f"{', '.join(sorted(REPLICATED_CLASSES))}")
print(f"bulk volumes: {len(corpus)}   distinct exports: "
      f"{len({v.get('export') for _s, v in corpus})}   "
      f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}")

# A floor on the assertion count, for the same reason the corpus may not be empty: a pass
# must not be able to degenerate into "nothing was actually asked".
MIN_CHECKS = 40
check(CHECKS >= MIN_CHECKS,
      f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not "
      f"evidence; the corpus or the extraction has degraded")

print()
if FAILURES:
    print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)",
          file=sys.stderr)
    for f in FAILURES:
        print(f"  - {f}", file=sys.stderr)
    raise SystemExit(1)

print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)")
PY