From 730d12ddc4dbdf9ea618df3be1b608a15f601bd9 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 7 Sep 2026 22:55:58 -0500 Subject: [PATCH 1/3] fix(conformance): R-ENV-03 shared-path rule tests the harm, not the filename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blanket regex around RENDERED_DIRS failed any tracked file whose name carries a delimited `prod` token. That caught the four per-environment backup credential halves (BACKUP_CREDENTIALS_SUFFIX) even though BOTH halves are applied to BOTH clusters as base resources and the overlay selects only which one the shared consumer REFERENCES — i.e. exactly the R-ENV-02 contract the regex exists to protect. Renaming the files to dodge the regex was rejected as gaming the guard, so the rule now encodes the property instead. The fix is not an allowlist of those four paths. A prod-token file in a shared path is excused only when the tree proves all of: (A) it declares nothing but Secrets whose names end in the environment suffix (1) its own directory's kustomization applies it, and EVERY environment value of the selector resolves to a Secret that some resource of that same kustomization declares, in the same namespace (2) nothing names the suffixed Secret — or the suffixed file — literally; every reference is `${VAR}` (3) that VAR is defined for the base and for every overlay, and prod's value is the one that selects this file References are read from parsed YAML values, never raw text: the mechanism is spelled out by name in comments beside the resources, and a comment references nothing. No other assertion is weakened; the check still runs 96 assertions, and the pass output now notes each excused file and why. --- .../prod-baremetal-not-dev-surface/check.sh | 172 +++++++++++++++++- 1 file changed, 166 insertions(+), 6 deletions(-) diff --git a/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh b/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh index 1fd041a..b980cf8 100755 --- a/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh +++ b/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh @@ -437,12 +437,172 @@ tracked = [f for f in subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"], capture_output=True, text=True, check=True ).stdout.split("\0") if f] RENDERED_DIRS = ("platform/", "platform-config", "apps/", "catalog/", "clusters/base/") -for f in tracked: - if f.startswith(RENDERED_DIRS) and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)", - f, re.I): - check(False, - f"{f} is a prod-specific file inside the shared configuration — every environment " - "renders these paths, so prod-only content here is development against prod") + +# THE ONE STRUCTURAL EXCEPTION: AN OVERLAY-SELECTED ENVIRONMENT HALF. +# ------------------------------------------------------------------ +# A filename carrying an environment token is a HEURISTIC for the harm R-ENV-03 names +# (content only prod runs, in a path every environment renders). It is not the harm +# itself. There is exactly one shape where a `-prod` name is the OPPOSITE of a prod-only +# fork: a credential that is TWO Secret objects, BOTH applied to BOTH clusters as base +# resources, where a cluster-vars knob decides only which one the shared consumer +# REFERENCES (clusters/base/cluster-vars.yaml, BACKUP_CREDENTIALS_SUFFIX: dev backs up to +# MinIO, prod to Cloudflare R2, and one shared Secret meant either environment's rotation +# broke the other). Nothing is forked: every environment renders the identical manifests +# and the identical object set, which is precisely the R-ENV-02 contract. +# +# So the exception is not a path allowlist — it is the PROPERTY that makes that shape +# safe, asserted from the tree, and a file is excused ONLY when all of it holds: +# +# (A) it declares nothing but Secrets whose names end in the environment suffix — the +# only kind of object this repo has a selection mechanism for; +# (1) it is listed in its own directory's kustomization.yaml, so EVERY environment +# applies it, and every environment's value of the selector resolves to an object +# that some resource of that same kustomization declares (same namespace) — no +# environment is left selecting a Secret that does not exist; +# (2) nothing references the suffixed name literally: every reference is a substituted +# expression `${VAR}`, and the suffixed FILE is named only by the +# kustomization that applies it — so no shared manifest hardcodes prod's half; +# (3) that VAR is defined for the base AND for every overlay (StrictPostBuildSubstitutions +# fails on an UNDEFINED variable, so an overlay that forgot it would break that +# cluster's reconciliation), and prod's value is the one that selects THIS file. +# +# References are read from PARSED YAML VALUES, never from raw text: the mechanism is +# documented by name in comments beside the resources (cluster-vars says so explicitly), +# and a name in a comment references nothing. Anything that fails any condition still +# fails, naming the condition it broke. +ENV_SUFFIX = f"-{prod_token}" +# Every environment that renders the shared configuration. +ENVS = ["base"] + sorted(p.parent.name + for p in (ROOT / "clusters").glob("*/kustomization.yaml") + if p.parent.name != "base") + + +def _docs(rel: str): + p = ROOT / rel + if not p.is_file(): + return None + try: + return [d for d in yaml.safe_load_all(p.read_text(errors="replace")) + if isinstance(d, dict)] + except Exception: + return None + + +def _strings(node): + if isinstance(node, str): + yield node + elif isinstance(node, dict): + for k, v in node.items(): + yield from _strings(k) + yield from _strings(v) + elif isinstance(node, (list, tuple)): + for v in node: + yield from _strings(v) + + +RENDERED_YAML = [f for f in tracked + if f.startswith(RENDERED_DIRS) and f.endswith((".yaml", ".yml"))] +VALUE_STRINGS = {f: list(_strings(_docs(f) or [])) for f in RENDERED_YAML} + + +def overlay_selected(f: str) -> list[str]: + """[] if f is an overlay-SELECTED environment half; else the conditions it breaks.""" + docs = _docs(f) + objs = [((d.get("kind")), (d.get("metadata") or {}).get("name"), + (d.get("metadata") or {}).get("namespace")) for d in (docs or [])] + if not objs or any(k != "Secret" or not isinstance(n, str) or not n.endswith(ENV_SUFFIX) + for k, n, _ in objs): + return [f"(A) it does not consist solely of Secret objects whose names end in " + f"{ENV_SUFFIX!r} — an overlay can select which SECRET a shared manifest " + f"references, and nothing else, so any other prod-suffixed object in a " + f"shared path is prod-only content"] + + reasons: list[str] = [] + d = str(Path(f).parent) + kfile = f"{d}/kustomization.yaml" + kdocs = _docs(kfile) or [] + res = [str(r) for r in ((kdocs[0].get("resources") if kdocs else None) or [])] + if Path(f).name not in res: + reasons.append(f"(1) {kfile} does not list it in `resources`, so it is not applied " + f"in every environment — one environment would carry an object the " + f"other does not") + + # Every Secret the applying kustomization declares: name -> namespace. + declared: dict[str, object] = {} + for r in res: + for dd in (_docs(f"{d}/{r}") or []): + if dd.get("kind") == "Secret": + md = dd.get("metadata") or {} + declared[str(md.get("name"))] = md.get("namespace") + + for _, name, ns in objs: + base_name = name[:-len(ENV_SUFFIX)] + pat = re.escape(base_name) + r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}" + sel = sorted({m for ss in VALUE_STRINGS.values() for s in ss + for m in re.findall(pat, s)}) + # (2) literal references to the suffixed OBJECT — anywhere but this file, except + # the applying kustomization naming this FILE (which is how it gets applied). + lit = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss + if name in s and f2 != f + and not (f2 == kfile and s == Path(f).name)}) + if lit: + reasons.append(f"(2) {', '.join(lit)} names the Secret {name!r} literally — a " + f"shared manifest that hardcodes prod's half makes every " + f"environment use prod's credential; reference " + f"{base_name}${{...}} instead") + # (2) literal references to the suffixed FILE from anywhere but its kustomization. + stray = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss + if Path(f).name in s and f2 not in (f, kfile)}) + if stray: + reasons.append(f"(2) {', '.join(stray)} names the file {Path(f).name!r} " + f"literally — only {kfile} may apply it") + if not sel: + reasons.append(f"(2) no manifest references {base_name}${{...}}: nothing selects " + f"between the halves, so this file is prod-only content that " + f"only prod could ever consume") + continue + if len(sel) > 1: + reasons.append(f"(2) {base_name} is selected by more than one variable {sel} — " + f"they can drift apart and leave one consumer on the other " + f"environment's credential") + var = sel[0] + vals = env_values(var) + missing = [e for e in ENVS if e not in vals] + if missing: + reasons.append(f"(3) ${{{var}}} is not defined for {missing} (defined for " + f"{sorted(vals)}) — StrictPostBuildSubstitutions fails on an " + f"undefined variable, so that environment would stop " + f"reconciling, or silently inherit the other's credential") + for e, v in sorted(vals.items()): + want = base_name + v + if want not in declared: + reasons.append(f"(1) environment {e!r} selects Secret {want!r} via " + f"${{{var}}}={v!r}, which no resource of {kfile} declares — " + f"that environment renders a dangling reference") + elif declared[want] != ns: + reasons.append(f"(1) environment {e!r} selects Secret {want!r}, declared in " + f"namespace {declared[want]!r}, but this half is in {ns!r} — " + f"the halves are not interchangeable") + if prod_overlay and prod_overlay in vals and base_name + vals[prod_overlay] != name: + reasons.append(f"(3) the prod overlay's ${{{var}}}={vals[prod_overlay]!r} selects " + f"{base_name + vals[prod_overlay]!r}, not this file's {name!r} — " + f"prod's half is applied but no environment selects it") + return reasons + + +flagged = [f for f in tracked + if f.startswith(RENDERED_DIRS) + and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)", f, re.I)] +for f in flagged: + why = overlay_selected(f) + check(not why, + f"{f} is a prod-specific file inside the shared configuration — every environment " + f"renders these paths, so prod-only content here is development against prod. It is " + f"not an overlay-selected environment half either: " + " AND ".join(why)) + if not why: + note(f"{f} carries the {prod_token!r} token but is an OVERLAY-SELECTED half: both " + f"halves are applied in every environment and only the cluster-vars selector " + f"differs (R-ENV-02), so it is not prod-only content") check(len(tracked) > 50, f"git ls-files returned only {len(tracked)} paths — not a real checkout") # --------------------------- (6) prod work exists only with a declaration of readiness From e319777849ede0153c1d21270f6d349953b1a7e3 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 7 Sep 2026 23:02:56 -0500 Subject: [PATCH 2/3] docs(pm): draft the R-ENV-03 amendment, and name the gate hole that hid it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The requirement's literal reading ("no prod-specific artifact in a shared path") forbids the environment-selected PAIR that R-ENV-02 prescribes for a value that must differ per environment while the manifests stay identical — and a Secret's difference cannot live in cluster-vars, because it is ciphertext in a SOPS file. Drafted wording that keeps the guard's teeth (an artifact only prod APPLIES still fails) and removes the false positive, for the System Architect: the spec repo is read-only here and CI-gated by R-TEST-06. Also recorded, checked rather than assumed: the merge that turned R-ENV-03 red was gated with a touched-requirement subset that never selected R-ENV-03, so ci/l0.sh Stage 3b filtered the already-red check out of the run that printed PASS. Replaying that delivery's twelve paths through fix/touched-requirements-fail-closed's requirement-paths.map still yields no R-ENV-03, so landing that branch would NOT have caught this. The two changes that would (unconditional whole-repo L0 stages; a changed conformance test attributing its own requirement labels) are ci/ work and are left to that branch, not smuggled in here. --- DECISIONS-LOG.md | 135 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/DECISIONS-LOG.md b/DECISIONS-LOG.md index 0bd477c..30f11df 100644 --- a/DECISIONS-LOG.md +++ b/DECISIONS-LOG.md @@ -1954,3 +1954,138 @@ not one.** Confirmed by `--list-hosts` (dev default → 3 dev hosts, where the n Nothing was run against either cluster for this change. 3. **Decide on `kubeEtcd` metrics**, so §2's exit condition can be measured with etcd's own fsync histograms rather than inferred from node PSI. + +## 2026-09-07 — R-ENV-03's L0 check was RED on `main`, and the requirement's literal wording is why (branch `fix/r-env-03-overlay-selected-credentials`) + +`main`'s `conformance/tests/l0-static/prod-baremetal-not-dev-surface` failed 4 of 96 assertions. +All four are the per-environment backup credentials merged in `e7c4a95`: + +``` +apps/wave-0/nextcloud-db-backup-credentials-prod.sops.yaml +apps/wave-0/semaphore-db-backup-credentials-prod.sops.yaml +platform/keycloak/keycloak-db-backup-credentials-prod.sops.yaml +platform/velero/velero-cloud-prod.sops.yaml + ... is a prod-specific file inside the shared configuration +``` + +**The guard was a filename regex.** Any tracked file under `platform/ platform-config apps/ +catalog/ clusters/base/` whose name contained a delimited `prod` token failed, with no allowlist +and no exception mechanism. Nothing about the four files' CONTENT was inspected. + +**Renaming them to dodge the regex was rejected** (operator, explicitly): the guard exists for a +reason and a rename would leave the reason unserved while making the check lie. So the check was +refined to test the harm instead of the spelling, and the requirement wording is escalated below. + +### Why these four files are compliant with R-ENV-02, and why the regex could not see it + +Backup credentials became per-environment because dev backs up to MinIO and prod to Cloudflare +R2, and ONE shared Secret meant either environment's key rotation broke the other (worse for +CNPG: `archive_mode=on` turns a 403 on WAL archive into a monotonically filling pgdata PVC). + +The mechanism is a single cluster-vars knob, `BACKUP_CREDENTIALS_SUFFIX` (base `""`, dev `""`, +prod `"-prod"`). **Both Secret objects are applied to BOTH clusters** as base resources; the +overlay changes only which one the shared consumer REFERENCES +(`${BACKUP_CREDENTIALS_SUFFIX}`). Every environment renders the identical manifests and the +identical object set. That is exactly the "same manifests, overlay-scoped values" contract of +R-ENV-02 — nothing is forked, and there is no prod-only content for anyone to develop against. +The `-prod` in a filename is not a prod-only manifest; it is one half of an environment-selected +pair, present everywhere. + +### What the check now asserts (committed on the branch; not a path allowlist) + +A file whose name carries the environment token is excused ONLY when the tree proves all of: + +- **(A)** it declares nothing but Secrets whose names end in the environment suffix — an overlay + can select which SECRET a shared manifest references and nothing else, so any other + prod-suffixed object in a shared path is prod-only content and still fails; +- **(1)** its own directory's `kustomization.yaml` lists it in `resources` (so EVERY environment + applies it), and EVERY environment's value of the selector resolves to a Secret that some + resource of that same kustomization declares, in the same namespace — no environment is left + selecting an object that does not exist; +- **(2)** nothing names the suffixed Secret — or the suffixed FILE — literally: every reference is + `${VAR}`, and only the applying kustomization may name the file; +- **(3)** that `VAR` is defined for `clusters/base` AND for every `clusters/` overlay + (StrictPostBuildSubstitutions fails on an UNDEFINED variable, so an overlay that forgot it + would stop that cluster reconciling), and prod's value is the one that selects THIS file. + +References are read from PARSED YAML VALUES, never raw text — the mechanism is documented by name +in comments beside the resources, and a name in a comment references nothing. + +Mutation-tested: eleven mutations, each restored afterwards, each of which must FAIL and does — a +prod-only ConfigMap in a shared path, a prod-only Secret with no counterpart, a consumer +hardcoding prod's Secret name, a kustomization naming prod's file from another directory, the +knob missing from dev, the knob missing from base, prod's value drifted to a suffix nothing +declares, the prod half dropped from `resources`, the base half deleted (with and without its +`resources` entry), the halves split across namespaces, and one of two consumer references moved +to a second knob. The legitimate four pass, with a `note:` per file saying why they were excused. + +### PROPOSED R-ENV-03 AMENDMENT — for the System Architect (spec repo is READ-ONLY here) + +**What the requirement says today** (`docs/00-vision-and-scope.md`, detail in `docs/02`): prod is +a 3x mini-PC bare-metal k3s cluster with local storage and **"MUST NOT be an active development +surface until the roadmap declares it ready"**; `docs/02` adds **"Never develop against prod. +Prod hardware is a target, not a sandbox (R-ENV-03)."** + +**The problem with the literal reading.** Taken literally against the repository, "no prod-only +content in the shared configuration" reads as "no artifact in a shared path may be specific to +prod". That forbids the environment-selected PAIR, which is the very pattern R-ENV-02 prescribes +for a value that must differ per environment while the manifests stay identical. The two +requirements contradict each other on this case: R-ENV-02 says put the difference in an +overlay-scoped value, and a Secret's difference cannot live in cluster-vars (it is ciphertext, per +SOPS file, R-SEC-01), so it must live in a per-environment FILE that every environment applies. +The requirement should forbid what is actually harmful — configuration only prod runs — not the +appearance of an environment name in a path. + +**Proposed wording** (add as a clarifying clause to R-ENV-03 in `docs/02`, and mirror the phrase +in `docs/00`): + +> Prod MUST NOT be an active development surface. Concretely: no artifact in the shared +> configuration may be reachable by prod alone. An artifact whose NAME carries an environment +> token is not by itself a violation — provided every environment APPLIES it, no shared manifest +> references it by its environment-qualified name, and the selection between environment +> counterparts is made solely by an overlay-scoped cluster-vars value that is DEFINED for every +> environment. An artifact that only prod applies, or that a shared manifest names literally, IS +> a violation, whatever it is called. + +That wording keeps the guard's teeth (an artifact only prod applies still fails), removes the +false positive, and states the property the check now enforces so the test and the spec agree. + +**Related spec ask, same clause.** R-ENV-02 should say out loud that a per-environment SECRET is +expressed as counterpart SOPS files applied everywhere plus a cluster-vars selector, because that +is the one value class that cannot be expressed as a cluster-vars value itself. Without it, the +next person hits the same contradiction. + +### How this reached `main` green — the gate hole, and whether the fix in flight closes it + +`ci/l0.sh` Stage 3b scopes the L0 conformance run to the TOUCHED requirement subset +(`load_touched_requirements` -> `run.sh --requirements ...`). The delivery `e7c4a95` touched +`apps/wave-0/`, `catalog/render.py`, `platform/keycloak/`, `platform/velero/` and declared no +R-ID on a non-comment line, so the path map attributed R-GITOPS-02 / R-IDP-01..06 / R-STATE-03 — +**and never R-ENV-03**. `prod-baremetal-not-dev-surface` was therefore FILTERED OUT of the run +that gated the merge. The gate printed PASS having never executed the check that was already red. + +**Landing `fix/touched-requirements-fail-closed` would NOT have caught this.** Checked, not +assumed: that branch's `ci/requirement-paths.map` classifies every path (so nothing is silently +unmapped any more), but it attributes R-ENV-03 only to `infra/ansible/*` and `infra/tofu/*`. +Replaying `e7c4a95`'s twelve paths through that map yields +`R-GITOPS-01 R-GITOPS-02 R-IDP-01..06 R-PLAT-02 R-STATE-03` — still no R-ENV-03. Its `ci/l0.sh` +change also leaves Stage 3b scoped (only the substitution stage stays unconditional). The +fail-closed branch fixes the "unmapped path contributes nothing" hole; it does not fix this one. + +**Two things that would have caught it, for the operator/architect to choose between:** + +1. **Whole-repo L0 invariants should be unconditional.** `flux-postbuild-substitution`, + `keycloak-client-secret-reconciler` and `spec-immutability` already run as their own + unconditional stages for exactly this reason. `prod-baremetal-not-dev-surface`, + `no-plaintext-secrets` and `least-privilege-rbac` are the same shape — they assert over the + WHOLE tracked tree, so scoping them to a changed path is meaningless. Cheapest correct fix. +2. **A changed test should attribute the requirements it declares.** `conformance/*` maps to + R-TEST-01..04 only, so editing `prod-baremetal-not-dev-surface/check.sh` does not select + R-ENV-03 — this very branch's gate run reports `touched requirements: R-ENV-02 R-TEST-01 + R-TEST-02 R-TEST-03 R-TEST-04` and ran 2 of 12 L0 checks, NOT including the one it changed. + A changed `conformance/tests//` should contribute that test's own + `conformance.home.k3s/requirements` label. + +Both belong to `ci/`, so they are left to that branch rather than smuggled in here. The full L0 +layer was run unscoped on this branch to compensate: **12 offline checks, 11 PASS, 1 SKIP** +(`placeholder-guard-coupling`, pre-existing), 0 FAIL. From 89cf59ffb7fdfb6ab0ddb49247bb1473f167a62c Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 7 Sep 2026 23:04:39 -0500 Subject: [PATCH 3/3] docs(conformance): the GIVEN/WHEN/THEN header states the exception it now allows --- .../tests/l0-static/prod-baremetal-not-dev-surface/check.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh b/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh index b980cf8..dfd5b3f 100755 --- a/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh +++ b/conformance/tests/l0-static/prod-baremetal-not-dev-surface/check.sh @@ -48,7 +48,10 @@ # sync.yaml, every patch it applies targets an ENVIRONMENT-SCOPED object # (cluster-vars, a Flux Kustomization's sops decryption, or the Git source), # prod's Git ref is the SAME ref dev tracks (no prod-only branch to develop on), -# and no prod-only manifest exists anywhere in the rendered configuration; +# and no prod-only manifest exists anywhere in the rendered configuration — a file +# whose NAME carries the environment token is excused ONLY when the tree proves it +# is an OVERLAY-SELECTED half that every environment applies (see the four +# conditions at "THE ONE STRUCTURAL EXCEPTION" below); anything else still fails; # (6) prod artifacts exist only WITH a declaration of readiness: the spec roadmap # carries the phase that opens prod build (checked when the spec repo is mounted), # and the repository records that the operator opened it.