From 79c703bc799b14eb6167c086ba4d30e2c8152c9b Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Fri, 18 Sep 2026 22:14:51 -0500 Subject: [PATCH] fix(conformance): whole-repo checks audited nested agent worktrees as source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six discovery sites walked ROOT without pruning dot-directories, so they descended into .claude/worktrees/agent-*/ — up to 18 full copies of the repo, created for isolated agent runs and gitignored (.gitignore:43). The visible half was false FAILs: four ServiceMonitors reported as selecting no Service, and nine PrometheusRule manifests reported as "under test but not in the repo", all of them worktree paths. The whole-repo L0 gate was red for reasons that had nothing to do with authored surface. The dangerous half is quieter. A check that asks "does anything in the repo declare X" can be satisfied by a worktree copy instead of real source, so the same bug that invents failures can also manufacture a pass. That is the reason to fix all six sites rather than only the ones that happened to be failing. egress-paths-selected-by-rules already had this fix and documents it (it once saw 5 CiliumEgressGatewayPolicy objects instead of the 1 authored); this applies the same dot-directory predicate everywhere else, which skips dot-DIRECTORIES while still matching dot-FILES such as .sops.yaml. platform/monitoring/tests/run.sh is the other side of the same failure: its `--list` reported each manifest once per worktree, and the L0 check compares that listing against the tree. Both find calls now prune .claude alongside .git. `--list` returns 21 real pairs where it previously returned them ~19x over. Verified in the pinned toolbox: L0 --include-scope whole-repo is 21 cases, 0 failures, 0 skips, 0 unwired (was 4 failures, then 1). L1 3/3. The 18 stale worktrees are all on already-merged branches and remain on disk; cleaning them up is a separate change and is no longer load-bearing for the gate. Co-Authored-By: Claude Opus 5 --- .../alert-metrics-have-scrape-targets/check.sh | 9 +++++++-- .../tests/l0-static/alert-rules-unit-tested/check.sh | 9 +++++++-- .../tests/l0-static/alert-runbooks-resolvable/check.sh | 7 ++++++- .../l0-static/placeholder-guard-coupling/check.sh | 2 +- .../l0-static/servicemonitor-ports-resolve/check.sh | 7 ++++++- platform/monitoring/tests/run.sh | 10 +++++++--- 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh index 6740752..23653f9 100755 --- a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh +++ b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh @@ -448,9 +448,14 @@ def is_golden_fixture(p: Path) -> bool: return "conformance" in parts and "golden" in parts +# NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such +# as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs, so +# a bare rglob from ROOT audited up to 18 duplicates of every file — which both invents FAILs +# and, worse, can satisfy an "something declares X" check from a copy rather than real source. +# Same fix as egress-paths-selected-by-rules. .claude/ is gitignored (.gitignore:43). rule_files = sorted( p for p in ROOT.rglob("*.yaml") - if ".git" not in p.parts and not is_golden_fixture(p) and "kind: PrometheusRule" in read(p) + if not any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts) and not is_golden_fixture(p) and "kind: PrometheusRule" in read(p) ) check(len(rule_files) > 0, "no file declaring `kind: PrometheusRule` found — the parser or the repo layout changed; " @@ -619,7 +624,7 @@ def yaml_docs(path: Path) -> list[dict]: # job name -> (relpath, monitor endpoint ports, ports the co-authored Services define) same_file_monitors: dict[str, tuple[str, set[str], set[str]]] = {} for p in sorted(ROOT.rglob("*.yaml")): - if ".git" in p.parts: + if any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts): continue txt = read(p) if "kind: ServiceMonitor" not in txt or "kind: Service" not in txt: diff --git a/conformance/tests/l0-static/alert-rules-unit-tested/check.sh b/conformance/tests/l0-static/alert-rules-unit-tested/check.sh index 5716316..f782119 100755 --- a/conformance/tests/l0-static/alert-rules-unit-tested/check.sh +++ b/conformance/tests/l0-static/alert-rules-unit-tested/check.sh @@ -280,11 +280,16 @@ FETCH_RE = re.compile( # versions.env. What is forbidden is a RULE-UNIT RUN acquiring its own engine: then the # version, the network and the machine are decided silently at test time, and a missing # toolchain becomes a green run instead of an exit 3. +# NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such +# as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs, so +# a bare rglob from ROOT audited up to 18 duplicates of every file — which both invents FAILs +# and, worse, can satisfy an "something declares X" check from a copy rather than real source. +# Same fix as egress-paths-selected-by-rules. .claude/ is gitignored (.gitignore:43). SELF_DIR = "conformance/tests/l0-static/alert-rules-unit-tested/" INSTALLERS = (".gitea/workflows/scripts/install-tools.sh", "infra/devcontainer/") for p in sorted(ROOT.rglob("*.sh")): rel = str(p.relative_to(ROOT)) - if rel.startswith((".git/", "conformance/runs/", SELF_DIR)): + if rel.startswith((".git/", ".claude/", "conformance/runs/", SELF_DIR)): continue try: text = p.read_text() @@ -319,7 +324,7 @@ def tracked_yaml() -> list[Path]: return [ROOT / p for p in out.stdout.split("\0") if p] # No usable index (the gate container mounts a worktree whose .git points outside the # mount). Fall back to a filesystem walk rather than auditing nothing. - return sorted(p for p in ROOT.rglob("*.y*ml") if ".git/" not in str(p)) + return sorted(p for p in ROOT.rglob("*.y*ml") if not any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts)) KIND_RE = re.compile(r"^\s*kind:\s*PrometheusRule\s*$", re.M) diff --git a/conformance/tests/l0-static/alert-runbooks-resolvable/check.sh b/conformance/tests/l0-static/alert-runbooks-resolvable/check.sh index 8777529..1d2df3a 100755 --- a/conformance/tests/l0-static/alert-runbooks-resolvable/check.sh +++ b/conformance/tests/l0-static/alert-runbooks-resolvable/check.sh @@ -126,9 +126,14 @@ def read(p: Path) -> str: SCOPE_DECL_RE = r"^\s*conformance\.home\.k3s/scope:\s*[\"\']?whole-repo" RUNBOOK_URL_RE = re.compile(r"""^\s*runbook_url:\s*["']?([^"'\s#]+)["']?\s*$""") +# NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such +# as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs, so +# a bare rglob from ROOT audited up to 18 duplicates of every file — which both invents FAILs +# and, worse, can satisfy an "something declares X" check from a copy rather than real source. +# Same fix as egress-paths-selected-by-rules. .claude/ is gitignored (.gitignore:43). rule_files = sorted( p for p in ROOT.rglob("*.yaml") - if ".git" not in p.parts and "kind: PrometheusRule" in read(p) + if not any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts) and "kind: PrometheusRule" in read(p) ) check(len(rule_files) > 0, "no file declaring `kind: PrometheusRule` found — the parser or the repo layout changed; " diff --git a/conformance/tests/l0-static/placeholder-guard-coupling/check.sh b/conformance/tests/l0-static/placeholder-guard-coupling/check.sh index 8189263..f3843f3 100755 --- a/conformance/tests/l0-static/placeholder-guard-coupling/check.sh +++ b/conformance/tests/l0-static/placeholder-guard-coupling/check.sh @@ -238,7 +238,7 @@ def bail(msg): DEF_RE = re.compile(r"^([a-z0-9_]+_is_placeholder)\(\)\s*\{") SELF_DIR = ROOT / "conformance" / "tests" / "l0-static" / "placeholder-guard-coupling" -SKIP_DIRS = {".git", "_artifacts", "node_modules", ".venv"} +SKIP_DIRS = {".git", ".claude", "_artifacts", "node_modules", ".venv"} # ------------------------------------------------------------------ discover the guard FILE # Hardcoding a path is what let a file that never landed disable this whole check. diff --git a/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh b/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh index a03038b..f720678 100755 --- a/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh +++ b/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh @@ -154,8 +154,13 @@ services: list[tuple] = [] monitors: list[tuple] = [] unparseable: list[str] = [] +# NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such +# as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs, so +# a bare rglob from ROOT audited up to 18 duplicates of every file — which both invents FAILs +# and, worse, can satisfy an "something declares X" check from a copy rather than real source. +# Same fix as egress-paths-selected-by-rules. .claude/ is gitignored (.gitignore:43). for p in sorted(ROOT.rglob("*.yaml")): - if ".git" in p.parts or is_test_fixture(p) or ".sops." in p.name: + if any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts) or is_test_fixture(p) or ".sops." in p.name: continue txt = read(p) if not txt: diff --git a/platform/monitoring/tests/run.sh b/platform/monitoring/tests/run.sh index fab6956..1458601 100755 --- a/platform/monitoring/tests/run.sh +++ b/platform/monitoring/tests/run.sh @@ -116,10 +116,14 @@ fi # `.git` file points outside the mount, so git cannot read the index there — and with # `pipefail` that failure silently emptied the discovery, i.e. a green run over ZERO tests # (observed 2026-09-12). `find` needs no repo metadata, so discovery cannot be disabled by -# how the tree happens to be mounted. .git/ is pruned so a stale object copy is never picked up. +# how the tree happens to be mounted. .git/ is pruned so a stale object copy is never picked up, +# and .claude/ with it: .claude/worktrees/agent-*/ holds full repo copies for isolated agent +# runs, so an unpruned find reported the SAME manifest under test once per worktree (up to 18 +# times), and the L0 check that compares this listing against the tree failed on paths that +# are not authored surface at all (.claude/ is gitignored, .gitignore:43). discover() { # Companion convention, anywhere in the tree. - find "${ROOT}" -path "${ROOT}/.git" -prune -o -name '*.promtool-test.yaml' -print 2>/dev/null + find "${ROOT}" \( -path "${ROOT}/.git" -o -path "${ROOT}/.claude" \) -prune -o -name '*.promtool-test.yaml' -print 2>/dev/null # Suite convention, ANYWHERE in the tree — not just this directory. # # This used to be `find "${HERE}" -maxdepth 1`, which made the suite convention a @@ -129,7 +133,7 @@ discover() { # as the rules this harness exists to test. Found 2026-09-13 adding apps/wave-*/tests/. Scoped by # NAME only; manifests_for() then refuses (loudly) any *.test.yaml that is not inside a tests/ # directory, so a misplaced file fails rather than disappearing. - find "${ROOT}" -path "${ROOT}/.git" -prune -o -name '*.test.yaml' -print 2>/dev/null + find "${ROOT}" \( -path "${ROOT}/.git" -o -path "${ROOT}/.claude" \) -prune -o -name '*.test.yaml' -print 2>/dev/null } TESTS=()