From e6f6e4febbb932af30a0f1ceb86711d32ac9303f Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Tue, 29 Sep 2026 02:52:28 -0500 Subject: [PATCH] feat(ci): detect Gitea settings drift -- the blind spot that hid a month of outage Every check in this repo reads the repository, the rendered manifests, or the cluster. NONE could see Gitea's own database, and that is where branch protection and runner registration live. Three failures came out of that blind spot on 2026-09-29, all the same shape -- a setting that READ as enforced while not being, with nothing able to notice: * main had NO branch protection for ~a month while DECISIONS-LOG recorded it as a required status check; * ZERO Actions runners were registered, so PR #1 -- the first PR this repo ever had -- sat pending for 30.6 minutes with nothing to execute it; * after enable_status_check and required_approvals were turned on they STILL did not bite, because block_admin_merge_override was false: PRs #5-#9 merged with zero approvals recorded against required_approvals: 1. infra/gitea/*.sh ASSERT the desired state. Nothing DETECTED drift from it. This is the detector. ci/gitea-settings.sh runs as a merge-gate stage: it diffs the live rule at /branch_protections/main against infra/gitea/expected-branch-protection.json and asserts at least one runner is online. It reads the RULE object, not /branches/main -- the branch view does not expose block_admin_merge_override and showed all-green while the rule was overridable by the only account that merges. Verified by test, not assumed: no token -> exit 3, skips loudly, does NOT block live == declaration -> exit 0 injected drift -> exit 1, blocks, naming both the drifted key and the runner count The skip is DELIBERATELY not in gate.sh's strict-skips loop. Making an absent credential block every PR is precisely the lock-yourself-out failure this repo keeps walking into, so a skip denies a PASS (exit 3) and says loudly what is unverified. REQUIRE_GITEA_SETTINGS=true upgrades it to a block once the token is wired. gate.yml's `|| [ "$?" -eq 3 ]` tolerance now covers a second stage. That widening is declared in the file rather than left to be discovered, and it narrows back the moment REQUIRE_GITEA_SETTINGS is set. infra/gitea/set-ci-token.sh mints the token and stores it as the repo Actions secret without ever printing the value. UNTIL THAT TOKEN EXISTS THIS STAGE SKIPS, AND A SKIPPED DETECTOR DETECTS NOTHING. Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/gate.yml | 13 +++ ci/Makefile | 3 + ci/gate.sh | 26 +++++ ci/gitea-settings.sh | 120 ++++++++++++++++++++ infra/gitea/README.md | 36 +++++- infra/gitea/expected-branch-protection.json | 20 ++++ infra/gitea/set-ci-token.sh | 40 +++++++ 7 files changed, 254 insertions(+), 4 deletions(-) create mode 100755 ci/gitea-settings.sh create mode 100644 infra/gitea/expected-branch-protection.json create mode 100755 infra/gitea/set-ci-token.sh diff --git a/.gitea/workflows/gate.yml b/.gitea/workflows/gate.yml index 9b7e5c1..2265f51 100644 --- a/.gitea/workflows/gate.yml +++ b/.gitea/workflows/gate.yml @@ -110,9 +110,22 @@ jobs: # UNVERIFIED and NAMED in the PASS headline, never faked green. GATE_STRICT_SKIPS + the # tolerated exit 3 are explained in the file header; the tolerance covers exactly one # thing, the unverifiable spec repo, and nothing else can produce a 3 here. + # GITEA_SETTINGS_TOKEN feeds ci/gitea-settings.sh, the only check that can see Gitea's + # OWN database (branch protection + runner registration) — the blind spot that let `main` + # sit unprotected with zero runners for a month while this repo recorded both as working. + # Referenced BY NAME only; create it with infra/gitea/set-ci-token.sh. + # + # WIDENED TOLERANCE, DECLARED: `|| [ "$?" -eq 3 ]` used to cover exactly one skip + # (spec-immutability). It now also covers gitea-settings skipping when that secret is + # absent. That is a deliberate, temporary weakening: making an unwired credential BLOCK + # every PR is the lock-yourself-out failure this repo keeps walking into. Once the secret + # exists and the stage has been SEEN green, set REQUIRE_GITEA_SETTINGS=true below and the + # tolerance narrows back to spec-immutability alone. - name: Merge gate — ci/gate.sh (what `make -C ci gate` runs) env: GATE_STRICT_SKIPS: "true" + GITEA_SETTINGS_TOKEN: ${{ secrets.GITEA_SETTINGS_TOKEN }} + # REQUIRE_GITEA_SETTINGS: "true" # <- uncomment AFTER seeing gitea-settings green run: ./ci/gate.sh || [ "$?" -eq 3 ] # NOTE: no actions/upload-artifact step. Gitea Actions rejects upload-artifact@v4 diff --git a/ci/Makefile b/ci/Makefile index 3d426da..5706807 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -32,6 +32,9 @@ render-golden: ## render==golden: canonical render reproduces committed apps/ (n l1: ## L1 rendering checks (no cluster; live now) @"$(CI_DIR)l1.sh" +gitea-settings: ## Gitea branch protection + runners match infra/gitea/expected-branch-protection.json (SKIPS w/o GITEA_SETTINGS_TOKEN) + @./gitea-settings.sh + conformance: ## L2/L3 in-cluster Chainsaw Job (SKIPS w/o CLUSTER_AVAILABLE=true) @"$(CI_DIR)conformance.sh" diff --git a/ci/gate.sh b/ci/gate.sh index 1ad105e..b1cf049 100755 --- a/ci/gate.sh +++ b/ci/gate.sh @@ -112,6 +112,7 @@ run_stage lint "${CI_DIR}/lint.sh" run_stage l0 "${CI_DIR}/l0.sh" run_stage render-golden "${CI_DIR}/render-golden.sh" run_stage l1 "${CI_DIR}/l1.sh" +run_stage gitea-settings "${CI_DIR}/gitea-settings.sh" run_stage conformance "${CI_DIR}/conformance.sh" echo >&2 @@ -188,6 +189,31 @@ case "${OUTCOME[spec-immutability]}" in pass) ok "spec-immutability: green" ;; esac +# gitea-settings: the ONLY check that can see Gitea's own database (branch protection + +# runner registration). A FAIL is real drift and always blocks -- that is the whole point. +# A SKIP means no token or unreachable Gitea, which is an UNWIRED CREDENTIAL, not a defect +# in the change under test. It is deliberately NOT in the strict-skips loop below: making an +# absent secret block every PR is precisely the lock-yourself-out failure this repo keeps +# walking into (a required check with nothing able to satisfy it). So a skip denies a PASS +# (exit 3) and says loudly what is unverified. Set REQUIRE_GITEA_SETTINGS=true once the +# token is wired, and the skip becomes a block as it should. +case "${OUTCOME[gitea-settings]}" in + fail) err "gitea-settings: Gitea DRIFTED from infra/gitea/expected-branch-protection.json -> blocks merge"; BLOCK=1 ;; + skip) + if truthy "${REQUIRE_GITEA_SETTINGS:-false}"; then + err "gitea-settings: skipped and REQUIRE_GITEA_SETTINGS=true -> blocks merge" + summary "- gitea-settings: **SKIPPED** (no token) — blocks merge (REQUIRE_GITEA_SETTINGS=true)" + BLOCK=1 + else + warn "gitea-settings: skipped (no GITEA_SETTINGS_TOKEN) — branch protection and runner" + warn " registration are UNVERIFIED. Both were silently wrong for a month. Wire the token" + warn " (infra/gitea/set-ci-token.sh), then set REQUIRE_GITEA_SETTINGS=true." + summary "- gitea-settings: **SKIPPED** (no token — protection/runners unverified; not a pass)" + NOT_ESTABLISHED+=("gitea-settings") + fi ;; + pass) ok "gitea-settings: green" ;; +esac + # L0/L1 stages: any failure blocks. A SKIP here means a pinned tool is missing or a seam is # unwired — NEVER green. In CI (strict) that is a defect and blocks; on a workstation it # degrades to UNDETERMINED, which is still non-zero and still not a pass. render-golden is a diff --git a/ci/gitea-settings.sh b/ci/gitea-settings.sh new file mode 100755 index 0000000..5293329 --- /dev/null +++ b/ci/gitea-settings.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# ci/gitea-settings.sh — Stage: the Gitea platform settings that gate this repo still +# match what git declares. +# +# WHY THIS STAGE EXISTS AT ALL +# ---------------------------- +# Every other check in this repo reads the repository, the rendered manifests, or the +# cluster. NONE of them can see Gitea's own database — and that is where branch +# protection and runner registration live. On 2026-09-29 that blind spot was measured: +# +# * `main` had NO branch protection, and +# * ZERO Actions runners were registered, +# +# for roughly a month, while DECISIONS-LOG.md recorded both as working and every agent +# and human in the loop repeated that claim because the repository said so. PR #1 — the +# first pull request this repo ever had — sat `pending` for 30.6 minutes with nothing to +# execute it. Later the same day `enable_status_check` and `required_approvals` were turned +# on and STILL did not bite, because `block_admin_merge_override` was false; PRs #5-#9 +# merged with zero approvals recorded against `required_approvals: 1`. +# +# Three separate failures, one shape: a setting that READ as enforced while not being, with +# no mechanism anywhere that could notice. Scripts that ASSERT the desired state +# (infra/gitea/*.sh) do not detect drift from it. This stage is the detector. +# +# WHAT IT ASSERTS +# 1. every key in infra/gitea/expected-branch-protection.json `.protection` matches the +# LIVE rule at /repos/{repo}/branch_protections/{branch}, exactly; +# 2. at least `.runners_online_min` Actions runners report status "online". +# +# (2) is not decoration. A required status check with no runner is the deadlock this repo +# nearly walked into: every PR pends forever, including the one that would fix it. +# +# READ THE RULE OBJECT, NOT THE BRANCH VIEW. /repos/{repo}/branches/{branch} reports +# `protected`, `enable_status_check` and `required_approvals` but NOT +# `block_admin_merge_override` — it showed all-green while the rule was overridable by the +# only account that merges. This stage queries /branch_protections/ deliberately. +# +# EXIT CODES (the gate contract) +# 0 live Gitea matches the declaration +# 1 DIVERGENCE — something drifted, or was never applied. Blocks the merge. +# 3 SKIP — no token, or Gitea unreachable. Never a pass; see gate.sh, which treats this +# skip like spec-immutability's (UNDETERMINED, exit 3) rather than a hard block, so +# that an unwired credential cannot lock the repository. +# +# TO WIRE IT: create a Gitea repo secret GITEA_SETTINGS_TOKEN (a token that can read repo +# settings) — infra/gitea/set-ci-token.sh does it. Until then this stage skips LOUDLY and +# the drift it exists to catch is still undetected. +set -euo pipefail +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${CI_DIR}/.." && pwd)" +# shellcheck disable=SC1091 +source "${CI_DIR}/lib.sh" + +log "Stage: Gitea platform settings match the declaration" + +DECL="${REPO_ROOT}/infra/gitea/expected-branch-protection.json" +GITEA_URL="${GITEA_URL:-https://gitea.home.keithjackson.dev}" +GITEA_REPO="${GITEA_REPO:-Home-Network/home-network-k3s-impl}" +TOKEN="${GITEA_SETTINGS_TOKEN:-}" + +[ -f "$DECL" ] || die "declaration missing: ${DECL}" +require_tool jq +require_tool curl + +if [ -z "$TOKEN" ]; then + skip "GITEA_SETTINGS_TOKEN is not set — cannot read the live Gitea rule." + skip " Branch protection and runner registration are therefore UNVERIFIED. They were" + skip " wrong, unnoticed, for a month (2026-09-29). Wire it: infra/gitea/set-ci-token.sh" + summary "- gitea-settings: **SKIPPED — no token** (protection + runners UNVERIFIED; not a pass)" + exit 3 +fi + +api() { curl -sS --max-time 20 -H "Authorization: token ${TOKEN}" "$@"; } + +BRANCH="$(jq -r '.branch' "$DECL")" +LIVE="$(api "${GITEA_URL}/api/v1/repos/${GITEA_REPO}/branch_protections/${BRANCH}" || true)" +if [ -z "$LIVE" ] || ! jq -e 'type == "object" and has("branch_name")' >/dev/null 2>&1 <<<"$LIVE"; then + skip "could not read the branch-protection rule from ${GITEA_URL} (unreachable, or the token" + skip " lacks permission). Not treating this as a pass." + summary "- gitea-settings: **SKIPPED — Gitea unreadable** (not a pass)" + exit 3 +fi + +RC=0 + +# --- 1. every declared protection key must match the live rule, exactly --------------- +while IFS= read -r key; do + want="$(jq -Sc --arg k "$key" '.protection[$k]' "$DECL")" + got="$(jq -Sc --arg k "$key" 'if has($k) then .[$k] else "<>" end' <<<"$LIVE")" + if [ "$want" = "$got" ]; then + ok " ${key} = ${got}" + else + err " ${key}: DECLARED ${want} but LIVE is ${got}" + RC=1 + fi +done < <(jq -r '.protection | keys[]' "$DECL") + +# --- 2. a runner must actually be online ---------------------------------------------- +MIN="$(jq -r '.runners_online_min // 1' "$DECL")" +RUNNERS="$(api "${GITEA_URL}/api/v1/admin/actions/runners" || true)" +N="$(jq -r '[(.runners//[])[]|select(.status=="online")]|length' <<<"${RUNNERS:-{\}}" 2>/dev/null || echo 0)" +if [ "${N:-0}" -ge "$MIN" ]; then + ok " runners online: ${N} (>= ${MIN})" +else + err " runners online: ${N}, DECLARED minimum ${MIN}." + err " A required status check with no runner leaves every PR pending forever —" + err " including the PR that would fix it. This is the 2026-09-29 outage condition." + RC=1 +fi + +if [ "$RC" -eq 0 ]; then + ok "Gitea settings match the declaration." + summary "- gitea-settings: **green** (branch protection + runners match \`infra/gitea/expected-branch-protection.json\`)" +else + err "Gitea has DRIFTED from infra/gitea/expected-branch-protection.json." + err " Re-assert it (infra/gitea/require-gate.sh, block-admin-override.sh), or if the live" + err " state is what you now want, change the DECLARATION in the same commit — deliberately." + summary "- gitea-settings: **DRIFT** — live Gitea does not match the declaration; blocks merge" +fi +exit "$RC" diff --git a/infra/gitea/README.md b/infra/gitea/README.md index 719e0f6..0b8c2da 100644 --- a/infra/gitea/README.md +++ b/infra/gitea/README.md @@ -86,6 +86,32 @@ enabling enforcement and probing the push. `require-gate.sh` is the enforcement Re-run `verify-then-enforce.sh` after any future change to the act-runner Deployment. +### `expected-branch-protection.json` + `ci/gitea-settings.sh` — the detector + +`expected-branch-protection.json` **declares** the desired rule. `ci/gitea-settings.sh` runs as +a merge-gate stage, reads the LIVE rule, and **fails the gate on any divergence** — plus asserts +at least one runner is online. It is the only thing in this repository that can see Gitea's own +database. + +It reads `/branch_protections/main`, not `/branches/main`: the branch view does **not** expose +`block_admin_merge_override`, and it showed all-green for hours while the rule was overridable. + +Behaviour, proven by test rather than assumed: + +| Situation | Exit | Effect | +|---|---|---| +| live matches the declaration | 0 | green | +| any key drifted, or no runner online | 1 | **blocks the merge** | +| no `GITEA_SETTINGS_TOKEN`, or Gitea unreachable | 3 | skips **loudly**; never a pass, never a block | + +The skip is deliberately *not* a block: an unwired credential blocking every PR is the +lock-yourself-out failure this repo keeps hitting. Wire the token with `set-ci-token.sh`, confirm +the stage goes **green on a real PR**, and only then set `REQUIRE_GITEA_SETTINGS=true` in +`gate.yml` so a future missing token blocks instead of skipping. + +**Changing the declaration is how you change policy.** If the live state is what you now want, +edit this JSON in the same commit — deliberately, in review — rather than letting the two drift. + ### Why these scripts exist at all, and what is still missing Gitea's repo settings live in **its own database**, not in anything Flux reconciles or the @@ -94,10 +120,12 @@ the root cause of the entire 2026-09-29 session: branch protection was off and t gone for a month while the log recorded both as working, and **no check existed that could have caught it.** -These scripts encode the desired state and can re-assert it, but they are not a detector. The -durable fix is a conformance check that reads the live Gitea API and fails when it diverges -from what is declared here — turning a class of silent, invisible failure into a loud one. -**That check does not exist yet and is the highest-value remaining work in this directory.** +These scripts encode the desired state and can re-assert it; `ci/gitea-settings.sh` above is the +**detector** that turns divergence from a silent, invisible failure into a loud one. It closes the +root cause of the 2026-09-29 session. + +It is only as good as its wiring: **until `GITEA_SETTINGS_TOKEN` exists it skips**, and a skipped +detector detects nothing. That is the remaining step. ## Historical sequence (kept for context) diff --git a/infra/gitea/expected-branch-protection.json b/infra/gitea/expected-branch-protection.json new file mode 100644 index 0000000..da88569 --- /dev/null +++ b/infra/gitea/expected-branch-protection.json @@ -0,0 +1,20 @@ +{ + "_comment": "DECLARED desired state of Gitea branch protection on main. ci/gitea-settings.sh fails the merge gate when the LIVE Gitea diverges from this. Gitea settings live in Gitea database - invisible to Flux, unreadable by every other check - so this file plus that check are the only drift detection that exists for them.", + "_generated": "2026-09-29 from the live rule, after the session that found protection OFF and zero runners for a month", + "branch": "main", + "runners_online_min": 1, + "protection": { + "enable_push": false, + "enable_push_whitelist": false, + "enable_merge_whitelist": false, + "enable_status_check": true, + "status_check_contexts": [ + "gate / gate (pull_request)" + ], + "required_approvals": 1, + "enable_approvals_whitelist": false, + "block_admin_merge_override": true, + "block_on_rejected_reviews": true, + "dismiss_stale_approvals": true + } +} diff --git a/infra/gitea/set-ci-token.sh b/infra/gitea/set-ci-token.sh new file mode 100755 index 0000000..ca00b30 --- /dev/null +++ b/infra/gitea/set-ci-token.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Mint a Gitea API token and store it as the repo Actions secret GITEA_SETTINGS_TOKEN, +# so ci/gitea-settings.sh can read the live branch-protection rule during the gate. +# +# The token value is NEVER printed. It goes from the token API into the secrets API and +# nowhere else -- not to stdout, not to a file, not into an agent transcript. +# +# Scope: `read:repository` + `read:admin` are what the check needs (the rule object, and the +# runner list). Gitea token scopes are coarse; grant the least that works on your version and +# confirm the check goes green rather than assuming the scope name. +set -euo pipefail +HOST=gitea.home.keithjackson.dev; OWNER=Home-Network; REPO=home-network-k3s-impl +NAME="ci-gitea-settings-$(date +%Y%m%d%H%M%S)" +creds="$(printf 'protocol=https\nhost=%s\n\n' "$HOST" | git credential fill)" +U="$(sed -n 's/^username=//p' <<<"$creds")"; P="$(sed -n 's/^password=//p' <<<"$creds")" +basic() { curl -sS --config <(printf 'user = "%s:%s"\n' "$U" "$P") "$@"; } + +echo "==> minting token '${NAME}' as ${U}" +TOKEN="$(basic -X POST -H 'Content-Type: application/json' \ + -d "{\"name\":\"${NAME}\",\"scopes\":[\"read:repository\",\"read:admin\"]}" \ + "https://${HOST}/api/v1/users/${U}/tokens" | jq -r '.sha1 // empty')" +[ -n "$TOKEN" ] || { echo "token creation failed (check scopes for your Gitea version)" >&2; exit 1; } +echo "==> got a ${#TOKEN}-character token (value not shown)" + +echo "==> storing it as repo Actions secret GITEA_SETTINGS_TOKEN" +basic -X PUT -H 'Content-Type: application/json' \ + -d "$(jq -n --arg d "$TOKEN" '{data:$d}')" \ + "https://${HOST}/api/v1/repos/${OWNER}/${REPO}/actions/secrets/GITEA_SETTINGS_TOKEN" >/dev/null +unset TOKEN +echo "==> stored." + +cat <<'NOTE' + +Next: + 1. .gitea/workflows/gate.yml already passes this secret to the gate step. + 2. Open any PR and confirm the gate prints "gitea-settings: green" -- it SKIPS silently + without the token, so a green run is the only proof the wiring works. + 3. THEN set REQUIRE_GITEA_SETTINGS=true in gate.yml, so a future missing token BLOCKS + instead of skipping. Do that only after you have seen it green, or you lock the repo. +NOTE