diff --git a/.gitea/workflows/gate.yml b/.gitea/workflows/gate.yml index 9b7e5c1..2265f51 100644 --- a/.gitea/workflows/gate.yml +++ b/.gitea/workflows/gate.yml @@ -110,9 +110,22 @@ jobs: # UNVERIFIED and NAMED in the PASS headline, never faked green. GATE_STRICT_SKIPS + the # tolerated exit 3 are explained in the file header; the tolerance covers exactly one # thing, the unverifiable spec repo, and nothing else can produce a 3 here. + # GITEA_SETTINGS_TOKEN feeds ci/gitea-settings.sh, the only check that can see Gitea's + # OWN database (branch protection + runner registration) — the blind spot that let `main` + # sit unprotected with zero runners for a month while this repo recorded both as working. + # Referenced BY NAME only; create it with infra/gitea/set-ci-token.sh. + # + # WIDENED TOLERANCE, DECLARED: `|| [ "$?" -eq 3 ]` used to cover exactly one skip + # (spec-immutability). It now also covers gitea-settings skipping when that secret is + # absent. That is a deliberate, temporary weakening: making an unwired credential BLOCK + # every PR is the lock-yourself-out failure this repo keeps walking into. Once the secret + # exists and the stage has been SEEN green, set REQUIRE_GITEA_SETTINGS=true below and the + # tolerance narrows back to spec-immutability alone. - name: Merge gate — ci/gate.sh (what `make -C ci gate` runs) env: GATE_STRICT_SKIPS: "true" + GITEA_SETTINGS_TOKEN: ${{ secrets.GITEA_SETTINGS_TOKEN }} + # REQUIRE_GITEA_SETTINGS: "true" # <- uncomment AFTER seeing gitea-settings green run: ./ci/gate.sh || [ "$?" -eq 3 ] # NOTE: no actions/upload-artifact step. Gitea Actions rejects upload-artifact@v4 diff --git a/ci/Makefile b/ci/Makefile index 3d426da..5706807 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -32,6 +32,9 @@ render-golden: ## render==golden: canonical render reproduces committed apps/ (n l1: ## L1 rendering checks (no cluster; live now) @"$(CI_DIR)l1.sh" +gitea-settings: ## Gitea branch protection + runners match infra/gitea/expected-branch-protection.json (SKIPS w/o GITEA_SETTINGS_TOKEN) + @./gitea-settings.sh + conformance: ## L2/L3 in-cluster Chainsaw Job (SKIPS w/o CLUSTER_AVAILABLE=true) @"$(CI_DIR)conformance.sh" diff --git a/ci/gate.sh b/ci/gate.sh index 1ad105e..b1cf049 100755 --- a/ci/gate.sh +++ b/ci/gate.sh @@ -112,6 +112,7 @@ run_stage lint "${CI_DIR}/lint.sh" run_stage l0 "${CI_DIR}/l0.sh" run_stage render-golden "${CI_DIR}/render-golden.sh" run_stage l1 "${CI_DIR}/l1.sh" +run_stage gitea-settings "${CI_DIR}/gitea-settings.sh" run_stage conformance "${CI_DIR}/conformance.sh" echo >&2 @@ -188,6 +189,31 @@ case "${OUTCOME[spec-immutability]}" in pass) ok "spec-immutability: green" ;; esac +# gitea-settings: the ONLY check that can see Gitea's own database (branch protection + +# runner registration). A FAIL is real drift and always blocks -- that is the whole point. +# A SKIP means no token or unreachable Gitea, which is an UNWIRED CREDENTIAL, not a defect +# in the change under test. It is deliberately NOT in the strict-skips loop below: making an +# absent secret block every PR is precisely the lock-yourself-out failure this repo keeps +# walking into (a required check with nothing able to satisfy it). So a skip denies a PASS +# (exit 3) and says loudly what is unverified. Set REQUIRE_GITEA_SETTINGS=true once the +# token is wired, and the skip becomes a block as it should. +case "${OUTCOME[gitea-settings]}" in + fail) err "gitea-settings: Gitea DRIFTED from infra/gitea/expected-branch-protection.json -> blocks merge"; BLOCK=1 ;; + skip) + if truthy "${REQUIRE_GITEA_SETTINGS:-false}"; then + err "gitea-settings: skipped and REQUIRE_GITEA_SETTINGS=true -> blocks merge" + summary "- gitea-settings: **SKIPPED** (no token) — blocks merge (REQUIRE_GITEA_SETTINGS=true)" + BLOCK=1 + else + warn "gitea-settings: skipped (no GITEA_SETTINGS_TOKEN) — branch protection and runner" + warn " registration are UNVERIFIED. Both were silently wrong for a month. Wire the token" + warn " (infra/gitea/set-ci-token.sh), then set REQUIRE_GITEA_SETTINGS=true." + summary "- gitea-settings: **SKIPPED** (no token — protection/runners unverified; not a pass)" + NOT_ESTABLISHED+=("gitea-settings") + fi ;; + pass) ok "gitea-settings: green" ;; +esac + # L0/L1 stages: any failure blocks. A SKIP here means a pinned tool is missing or a seam is # unwired — NEVER green. In CI (strict) that is a defect and blocks; on a workstation it # degrades to UNDETERMINED, which is still non-zero and still not a pass. render-golden is a diff --git a/ci/gitea-settings.sh b/ci/gitea-settings.sh new file mode 100755 index 0000000..5293329 --- /dev/null +++ b/ci/gitea-settings.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# ci/gitea-settings.sh — Stage: the Gitea platform settings that gate this repo still +# match what git declares. +# +# WHY THIS STAGE EXISTS AT ALL +# ---------------------------- +# Every other check in this repo reads the repository, the rendered manifests, or the +# cluster. NONE of them can see Gitea's own database — and that is where branch +# protection and runner registration live. On 2026-09-29 that blind spot was measured: +# +# * `main` had NO branch protection, and +# * ZERO Actions runners were registered, +# +# for roughly a month, while DECISIONS-LOG.md recorded both as working and every agent +# and human in the loop repeated that claim because the repository said so. PR #1 — the +# first pull request this repo ever had — sat `pending` for 30.6 minutes with nothing to +# execute it. Later the same day `enable_status_check` and `required_approvals` were turned +# on and STILL did not bite, because `block_admin_merge_override` was false; PRs #5-#9 +# merged with zero approvals recorded against `required_approvals: 1`. +# +# Three separate failures, one shape: a setting that READ as enforced while not being, with +# no mechanism anywhere that could notice. Scripts that ASSERT the desired state +# (infra/gitea/*.sh) do not detect drift from it. This stage is the detector. +# +# WHAT IT ASSERTS +# 1. every key in infra/gitea/expected-branch-protection.json `.protection` matches the +# LIVE rule at /repos/{repo}/branch_protections/{branch}, exactly; +# 2. at least `.runners_online_min` Actions runners report status "online". +# +# (2) is not decoration. A required status check with no runner is the deadlock this repo +# nearly walked into: every PR pends forever, including the one that would fix it. +# +# READ THE RULE OBJECT, NOT THE BRANCH VIEW. /repos/{repo}/branches/{branch} reports +# `protected`, `enable_status_check` and `required_approvals` but NOT +# `block_admin_merge_override` — it showed all-green while the rule was overridable by the +# only account that merges. This stage queries /branch_protections/ deliberately. +# +# EXIT CODES (the gate contract) +# 0 live Gitea matches the declaration +# 1 DIVERGENCE — something drifted, or was never applied. Blocks the merge. +# 3 SKIP — no token, or Gitea unreachable. Never a pass; see gate.sh, which treats this +# skip like spec-immutability's (UNDETERMINED, exit 3) rather than a hard block, so +# that an unwired credential cannot lock the repository. +# +# TO WIRE IT: create a Gitea repo secret GITEA_SETTINGS_TOKEN (a token that can read repo +# settings) — infra/gitea/set-ci-token.sh does it. Until then this stage skips LOUDLY and +# the drift it exists to catch is still undetected. +set -euo pipefail +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${CI_DIR}/.." && pwd)" +# shellcheck disable=SC1091 +source "${CI_DIR}/lib.sh" + +log "Stage: Gitea platform settings match the declaration" + +DECL="${REPO_ROOT}/infra/gitea/expected-branch-protection.json" +GITEA_URL="${GITEA_URL:-https://gitea.home.keithjackson.dev}" +GITEA_REPO="${GITEA_REPO:-Home-Network/home-network-k3s-impl}" +TOKEN="${GITEA_SETTINGS_TOKEN:-}" + +[ -f "$DECL" ] || die "declaration missing: ${DECL}" +require_tool jq +require_tool curl + +if [ -z "$TOKEN" ]; then + skip "GITEA_SETTINGS_TOKEN is not set — cannot read the live Gitea rule." + skip " Branch protection and runner registration are therefore UNVERIFIED. They were" + skip " wrong, unnoticed, for a month (2026-09-29). Wire it: infra/gitea/set-ci-token.sh" + summary "- gitea-settings: **SKIPPED — no token** (protection + runners UNVERIFIED; not a pass)" + exit 3 +fi + +api() { curl -sS --max-time 20 -H "Authorization: token ${TOKEN}" "$@"; } + +BRANCH="$(jq -r '.branch' "$DECL")" +LIVE="$(api "${GITEA_URL}/api/v1/repos/${GITEA_REPO}/branch_protections/${BRANCH}" || true)" +if [ -z "$LIVE" ] || ! jq -e 'type == "object" and has("branch_name")' >/dev/null 2>&1 <<<"$LIVE"; then + skip "could not read the branch-protection rule from ${GITEA_URL} (unreachable, or the token" + skip " lacks permission). Not treating this as a pass." + summary "- gitea-settings: **SKIPPED — Gitea unreadable** (not a pass)" + exit 3 +fi + +RC=0 + +# --- 1. every declared protection key must match the live rule, exactly --------------- +while IFS= read -r key; do + want="$(jq -Sc --arg k "$key" '.protection[$k]' "$DECL")" + got="$(jq -Sc --arg k "$key" 'if has($k) then .[$k] else "<>" end' <<<"$LIVE")" + if [ "$want" = "$got" ]; then + ok " ${key} = ${got}" + else + err " ${key}: DECLARED ${want} but LIVE is ${got}" + RC=1 + fi +done < <(jq -r '.protection | keys[]' "$DECL") + +# --- 2. a runner must actually be online ---------------------------------------------- +MIN="$(jq -r '.runners_online_min // 1' "$DECL")" +RUNNERS="$(api "${GITEA_URL}/api/v1/admin/actions/runners" || true)" +N="$(jq -r '[(.runners//[])[]|select(.status=="online")]|length' <<<"${RUNNERS:-{\}}" 2>/dev/null || echo 0)" +if [ "${N:-0}" -ge "$MIN" ]; then + ok " runners online: ${N} (>= ${MIN})" +else + err " runners online: ${N}, DECLARED minimum ${MIN}." + err " A required status check with no runner leaves every PR pending forever —" + err " including the PR that would fix it. This is the 2026-09-29 outage condition." + RC=1 +fi + +if [ "$RC" -eq 0 ]; then + ok "Gitea settings match the declaration." + summary "- gitea-settings: **green** (branch protection + runners match \`infra/gitea/expected-branch-protection.json\`)" +else + err "Gitea has DRIFTED from infra/gitea/expected-branch-protection.json." + err " Re-assert it (infra/gitea/require-gate.sh, block-admin-override.sh), or if the live" + err " state is what you now want, change the DECLARATION in the same commit — deliberately." + summary "- gitea-settings: **DRIFT** — live Gitea does not match the declaration; blocks merge" +fi +exit "$RC" diff --git a/infra/gitea/README.md b/infra/gitea/README.md index 719e0f6..0b8c2da 100644 --- a/infra/gitea/README.md +++ b/infra/gitea/README.md @@ -86,6 +86,32 @@ enabling enforcement and probing the push. `require-gate.sh` is the enforcement Re-run `verify-then-enforce.sh` after any future change to the act-runner Deployment. +### `expected-branch-protection.json` + `ci/gitea-settings.sh` — the detector + +`expected-branch-protection.json` **declares** the desired rule. `ci/gitea-settings.sh` runs as +a merge-gate stage, reads the LIVE rule, and **fails the gate on any divergence** — plus asserts +at least one runner is online. It is the only thing in this repository that can see Gitea's own +database. + +It reads `/branch_protections/main`, not `/branches/main`: the branch view does **not** expose +`block_admin_merge_override`, and it showed all-green for hours while the rule was overridable. + +Behaviour, proven by test rather than assumed: + +| Situation | Exit | Effect | +|---|---|---| +| live matches the declaration | 0 | green | +| any key drifted, or no runner online | 1 | **blocks the merge** | +| no `GITEA_SETTINGS_TOKEN`, or Gitea unreachable | 3 | skips **loudly**; never a pass, never a block | + +The skip is deliberately *not* a block: an unwired credential blocking every PR is the +lock-yourself-out failure this repo keeps hitting. Wire the token with `set-ci-token.sh`, confirm +the stage goes **green on a real PR**, and only then set `REQUIRE_GITEA_SETTINGS=true` in +`gate.yml` so a future missing token blocks instead of skipping. + +**Changing the declaration is how you change policy.** If the live state is what you now want, +edit this JSON in the same commit — deliberately, in review — rather than letting the two drift. + ### Why these scripts exist at all, and what is still missing Gitea's repo settings live in **its own database**, not in anything Flux reconciles or the @@ -94,10 +120,12 @@ the root cause of the entire 2026-09-29 session: branch protection was off and t gone for a month while the log recorded both as working, and **no check existed that could have caught it.** -These scripts encode the desired state and can re-assert it, but they are not a detector. The -durable fix is a conformance check that reads the live Gitea API and fails when it diverges -from what is declared here — turning a class of silent, invisible failure into a loud one. -**That check does not exist yet and is the highest-value remaining work in this directory.** +These scripts encode the desired state and can re-assert it; `ci/gitea-settings.sh` above is the +**detector** that turns divergence from a silent, invisible failure into a loud one. It closes the +root cause of the 2026-09-29 session. + +It is only as good as its wiring: **until `GITEA_SETTINGS_TOKEN` exists it skips**, and a skipped +detector detects nothing. That is the remaining step. ## Historical sequence (kept for context) diff --git a/infra/gitea/expected-branch-protection.json b/infra/gitea/expected-branch-protection.json new file mode 100644 index 0000000..da88569 --- /dev/null +++ b/infra/gitea/expected-branch-protection.json @@ -0,0 +1,20 @@ +{ + "_comment": "DECLARED desired state of Gitea branch protection on main. ci/gitea-settings.sh fails the merge gate when the LIVE Gitea diverges from this. Gitea settings live in Gitea database - invisible to Flux, unreadable by every other check - so this file plus that check are the only drift detection that exists for them.", + "_generated": "2026-09-29 from the live rule, after the session that found protection OFF and zero runners for a month", + "branch": "main", + "runners_online_min": 1, + "protection": { + "enable_push": false, + "enable_push_whitelist": false, + "enable_merge_whitelist": false, + "enable_status_check": true, + "status_check_contexts": [ + "gate / gate (pull_request)" + ], + "required_approvals": 1, + "enable_approvals_whitelist": false, + "block_admin_merge_override": true, + "block_on_rejected_reviews": true, + "dismiss_stale_approvals": true + } +} diff --git a/infra/gitea/set-ci-token.sh b/infra/gitea/set-ci-token.sh new file mode 100755 index 0000000..ca00b30 --- /dev/null +++ b/infra/gitea/set-ci-token.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Mint a Gitea API token and store it as the repo Actions secret GITEA_SETTINGS_TOKEN, +# so ci/gitea-settings.sh can read the live branch-protection rule during the gate. +# +# The token value is NEVER printed. It goes from the token API into the secrets API and +# nowhere else -- not to stdout, not to a file, not into an agent transcript. +# +# Scope: `read:repository` + `read:admin` are what the check needs (the rule object, and the +# runner list). Gitea token scopes are coarse; grant the least that works on your version and +# confirm the check goes green rather than assuming the scope name. +set -euo pipefail +HOST=gitea.home.keithjackson.dev; OWNER=Home-Network; REPO=home-network-k3s-impl +NAME="ci-gitea-settings-$(date +%Y%m%d%H%M%S)" +creds="$(printf 'protocol=https\nhost=%s\n\n' "$HOST" | git credential fill)" +U="$(sed -n 's/^username=//p' <<<"$creds")"; P="$(sed -n 's/^password=//p' <<<"$creds")" +basic() { curl -sS --config <(printf 'user = "%s:%s"\n' "$U" "$P") "$@"; } + +echo "==> minting token '${NAME}' as ${U}" +TOKEN="$(basic -X POST -H 'Content-Type: application/json' \ + -d "{\"name\":\"${NAME}\",\"scopes\":[\"read:repository\",\"read:admin\"]}" \ + "https://${HOST}/api/v1/users/${U}/tokens" | jq -r '.sha1 // empty')" +[ -n "$TOKEN" ] || { echo "token creation failed (check scopes for your Gitea version)" >&2; exit 1; } +echo "==> got a ${#TOKEN}-character token (value not shown)" + +echo "==> storing it as repo Actions secret GITEA_SETTINGS_TOKEN" +basic -X PUT -H 'Content-Type: application/json' \ + -d "$(jq -n --arg d "$TOKEN" '{data:$d}')" \ + "https://${HOST}/api/v1/repos/${OWNER}/${REPO}/actions/secrets/GITEA_SETTINGS_TOKEN" >/dev/null +unset TOKEN +echo "==> stored." + +cat <<'NOTE' + +Next: + 1. .gitea/workflows/gate.yml already passes this secret to the gate step. + 2. Open any PR and confirm the gate prints "gitea-settings: green" -- it SKIPS silently + without the token, so a green run is the only proof the wiring works. + 3. THEN set REQUIRE_GITEA_SETTINGS=true in gate.yml, so a future missing token BLOCKS + instead of skipping. Do that only after you have seen it green, or you lock the repo. +NOTE