From fe0ee698c4f98c73fc5771d6520517b66f274ad5 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Fri, 28 Aug 2026 11:11:52 -0500 Subject: [PATCH] fix(apps): reconcile goldens to one canonical render + enforce render==golden gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The committed apps/ goldens were a MIX of two render flag sets: 8 services were rendered with `--no-image-automation` (no image CRs) while gitea and nextcloud carried stale ImageRepository/ImagePolicy CRs from a render WITHOUT the flag. Neither single command reproduced all goldens, and the gate did not catch it. Part A — canonical render: python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation `--no-image-automation` is canonical: image automation is authored-but-DISABLED (DECISIONS-LOG; image-reflector/-automation controllers not installed), and every app was onboarded via this flag. Re-render drops the ImageRepository + ImagePolicy objects from gitea + nextcloud only (52 line deletions, purely the trailing image-automation blocks). No functional content changed — the nextcloud postStart hardening and gitea OIDC hook are untouched. Part B — enforcement: ci/render-golden.sh renders every onboarded catalog service with the canonical command into a temp dir and FAILS on ANY diff vs committed apps/ (missing golden or content drift), across all 10 services (not just the 3 L1 spot-checks). Not-onboarded catalog entries (keycloak, vm-demo) are excluded per DESIGN-waves. Wired as a required no-cluster stage in ci/gate.sh, ci/Makefile, and both the live (.gitea/workflows/gate.yml) and dormant (merge-gate.yaml) pipelines. Proof: tampering a golden makes the stage exit 1 (blocks merge); reverting restores exit 0. `make -C ci gate` green in the devcontainer. Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/README.md | 4 +- .gitea/workflows/gate.yml | 9 +-- .gitea/workflows/merge-gate.yaml | 17 ++++- apps/wave-0/gitea.yaml | 26 ------- apps/wave-1/nextcloud.yaml | 26 ------- ci/Makefile | 5 +- ci/gate.sh | 15 ++-- ci/render-golden.sh | 119 +++++++++++++++++++++++++++++++ 8 files changed, 153 insertions(+), 68 deletions(-) create mode 100755 ci/render-golden.sh diff --git a/.gitea/workflows/README.md b/.gitea/workflows/README.md index 64e49ee..932ae96 100644 --- a/.gitea/workflows/README.md +++ b/.gitea/workflows/README.md @@ -15,8 +15,8 @@ it touches is green** (R-TEST-03). ## The pipeline ``` - lint ─▶ l0 (schema + Kyverno) ─▶ l1 (rendering) ─▶ conformance (L2/L3) ─▶ merge-gate decision - └────────── run in CI, NO cluster ──────────┘ └── in-cluster Chainsaw Job ──┘ + lint ─▶ l0 (schema + Kyverno) ─▶ render-golden ─▶ l1 (rendering) ─▶ conformance (L2/L3) ─▶ merge-gate decision + └──────────────── run in CI, NO cluster ────────────────┘ └── in-cluster Chainsaw Job ──┘ live now cluster-gated (SKIPS in Phase 0) ``` diff --git a/.gitea/workflows/gate.yml b/.gitea/workflows/gate.yml index 6e84cc6..b9f05b6 100644 --- a/.gitea/workflows/gate.yml +++ b/.gitea/workflows/gate.yml @@ -3,8 +3,9 @@ # This is the LIVE gate wired to the in-cluster Gitea Actions runner (ns gitea-actions, # Docker executor + DinD). It runs the STATIC layers of the conformance gate — the ones # docs/14 says run "without a cluster": spec-immutability (best-effort), lint, Kyverno L0, -# and L1 offline render checks. It is the single required status check on `main` -# (see branch protection). `make -C ci lint l0 l1` here is byte-identical to a laptop run +# the render==golden reproducibility check, and L1 offline render checks. It is the single +# required status check on `main` +# (see branch protection). `make -C ci lint l0 render-golden l1` here is byte-identical to a laptop run # (R-TEST-02) — the runner just installs the pinned tools from ci/versions.env first. # # The L2/L3 CONFORMANCE layer (behavioural + fault-injection, needs a live cluster) is @@ -57,8 +58,8 @@ jobs: - name: Spec immutability (R-TEST-06, skips if spec repo absent) run: ./ci/spec-immutability.sh || [ "$?" -eq 3 ] - - name: Static gate — lint + Kyverno L0 + L1 render - run: make -C ci lint l0 l1 + - name: Static gate — lint + Kyverno L0 + render==golden + L1 render + run: make -C ci lint l0 render-golden l1 # NOTE: no actions/upload-artifact step. Gitea Actions rejects upload-artifact@v4 # ("not supported on GHES" — Gitea does not implement the artifact-v2 backend), which diff --git a/.gitea/workflows/merge-gate.yaml b/.gitea/workflows/merge-gate.yaml index 48f4f52..7046509 100644 --- a/.gitea/workflows/merge-gate.yaml +++ b/.gitea/workflows/merge-gate.yaml @@ -3,7 +3,7 @@ # platform. Enforces docs/14 (Testing & Conformance) gating rules as Gitea Actions. # # Pipeline (docs/14 "Where each layer runs"): -# lint -> schema -> kyverno-L0 -> L1-render [no cluster; live now] +# lint -> schema -> kyverno-L0 -> render==golden -> L1-render [no cluster; live now] # | # v # conformance-L2-L3 (in-cluster Chainsaw Job) @@ -89,9 +89,20 @@ jobs: - name: Schema validation + Kyverno L0 run: make -C ci l0 - l1: + render-golden: runs-on: ubuntu-latest needs: [l0] + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 0 } + - name: Install pinned tools + run: ./.gitea/workflows/scripts/install-tools.sh l1 + - name: render==golden reproducibility (canonical render vs committed apps/) + run: make -C ci render-golden + + l1: + runs-on: ubuntu-latest + needs: [render-golden] steps: - uses: actions/checkout@v4 with: { fetch-depth: 0 } @@ -143,7 +154,7 @@ jobs: # conformance skipped + no cluster -> report UNVERIFIED (Phase 0), do not fake green merge-gate: runs-on: ubuntu-latest - needs: [lint, l0, l1, conformance] + needs: [lint, l0, render-golden, l1, conformance] if: always() steps: - uses: actions/checkout@v4 diff --git a/apps/wave-0/gitea.yaml b/apps/wave-0/gitea.yaml index 771a12c..a89908e 100644 --- a/apps/wave-0/gitea.yaml +++ b/apps/wave-0/gitea.yaml @@ -255,29 +255,3 @@ spec: annotations: summary: gitea is down description: 'gitea has been unreachable for 5m (SLI: up).' ---- -# Flux ImageRepository (R-UPD) -apiVersion: image.toolkit.fluxcd.io/v1beta2 -kind: ImageRepository -metadata: - name: gitea - namespace: gitea -spec: - image: gitea/gitea - interval: 1h0m0s ---- -# Flux ImagePolicy (R-UPD) -apiVersion: image.toolkit.fluxcd.io/v1beta2 -kind: ImagePolicy -metadata: - name: gitea - namespace: gitea - annotations: - homelab.keithjackson.dev/update-channel: stable - homelab.keithjackson.dev/auto-rollback: 'true' -spec: - imageRepositoryRef: - name: gitea - policy: - semver: - range: '>=1.22.0 <2.0.0' diff --git a/apps/wave-1/nextcloud.yaml b/apps/wave-1/nextcloud.yaml index 47c4b08..f4edfc7 100644 --- a/apps/wave-1/nextcloud.yaml +++ b/apps/wave-1/nextcloud.yaml @@ -275,29 +275,3 @@ spec: annotations: summary: nextcloud is down description: 'nextcloud has been unreachable for 5m (SLI: up).' ---- -# Flux ImageRepository (R-UPD) -apiVersion: image.toolkit.fluxcd.io/v1beta2 -kind: ImageRepository -metadata: - name: nextcloud - namespace: nextcloud -spec: - image: nextcloud - interval: 1h0m0s ---- -# Flux ImagePolicy (R-UPD) -apiVersion: image.toolkit.fluxcd.io/v1beta2 -kind: ImagePolicy -metadata: - name: nextcloud - namespace: nextcloud - annotations: - homelab.keithjackson.dev/update-channel: stable - homelab.keithjackson.dev/auto-rollback: 'true' -spec: - imageRepositoryRef: - name: nextcloud - policy: - semver: - range: '>=31.0.0 <32.0.0' diff --git a/ci/Makefile b/ci/Makefile index ca0030c..986af72 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -11,7 +11,7 @@ CI_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) .DEFAULT_GOAL := help -.PHONY: help spec-immutability lint schema l0 l1 conformance gate all touched versions tools +.PHONY: help spec-immutability lint schema l0 render-golden l1 conformance gate all touched versions tools help: ## Show this help @awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " \033[36m%-14s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST) @@ -25,6 +25,9 @@ lint: ## yaml/lint (no cluster; live now) schema l0: ## ServiceDefinition schema validation + Kyverno L0 policy gate (no cluster; live now) @"$(CI_DIR)l0.sh" +render-golden: ## render==golden: canonical render reproduces committed apps/ (no cluster; live now) + @"$(CI_DIR)render-golden.sh" + l1: ## L1 rendering checks (no cluster; live now) @"$(CI_DIR)l1.sh" diff --git a/ci/gate.sh b/ci/gate.sh index 4a44ab4..3a54ba4 100755 --- a/ci/gate.sh +++ b/ci/gate.sh @@ -38,10 +38,11 @@ summary "## Merge gate (R-TEST-03)" summary "" run_stage spec-immutability "${CI_DIR}/spec-immutability.sh" -run_stage lint "${CI_DIR}/lint.sh" -run_stage l0 "${CI_DIR}/l0.sh" -run_stage l1 "${CI_DIR}/l1.sh" -run_stage conformance "${CI_DIR}/conformance.sh" +run_stage lint "${CI_DIR}/lint.sh" +run_stage l0 "${CI_DIR}/l0.sh" +run_stage render-golden "${CI_DIR}/render-golden.sh" +run_stage l1 "${CI_DIR}/l1.sh" +run_stage conformance "${CI_DIR}/conformance.sh" echo >&2 log "==== gate decision ====" @@ -56,8 +57,10 @@ case "${OUTCOME[spec-immutability]}" in esac # L0/L1 stages: any failure blocks. A SKIP here means a seam is unwired — in -# Phase 0 that is tolerated (placeholders), but surfaced. -for s in lint l0 l1; do +# Phase 0 that is tolerated (placeholders), but surfaced. render-golden is a +# no-cluster reproducibility gate (canonical render == committed apps/ goldens): +# a FAIL means a golden drifted from its ServiceDefinition source — always blocks. +for s in lint l0 render-golden l1; do case "${OUTCOME[$s]}" in fail) err "${s}: FAILED -> blocks merge"; BLOCK=1 ;; skip) warn "${s}: skipped (unwired seam / missing tool) — not green" ;; diff --git a/ci/render-golden.sh b/ci/render-golden.sh new file mode 100755 index 0000000..416317c --- /dev/null +++ b/ci/render-golden.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# ci/render-golden.sh — Stage: render==golden reproducibility gate. +# +# WHY: the committed apps/ tree is a set of GOLDENS produced by ONE canonical +# invocation of catalog/render.py. If a golden is hand-edited, or a catalog +# ServiceDefinition is changed without re-rendering, the goldens drift from +# their source of truth and Flux deploys something no one reviewed. The three +# L1 render spot-checks (conformance/tests/l1-rendering/*) only cover a couple +# of assertions on a couple of services — they let a real source/golden +# mismatch through (image-automation objects present in gitea/nextcloud but +# absent everywhere else). This stage closes that hole: it renders EVERY +# onboarded catalog service with the canonical command and FAILS on ANY diff +# vs the committed golden. +# +# CANONICAL RENDER (source of truth for apps/): +# python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation +# +# The `--no-image-automation` flag is canonical because image automation is +# authored-but-DISABLED (DECISIONS-LOG: "Phase 4 image-automation merged but +# DISABLED pending a Gitea write-cred"; pihole/all apps onboarded via +# `--no-image-automation`). The Flux image-reflector/-automation controllers +# are NOT installed in-cluster, so emitting ImageRepository/ImagePolicy CRs +# would leave apps referencing controllers that don't exist. Every onboarded +# app must render identically under this ONE command. +# +# Runs in CI WITHOUT a cluster (pure render + diff). Exit codes per ci/lib.sh: +# 0 = pass, 1 = fail (blocks merge), 3 = skip (renderer/tooling missing). +set -euo pipefail +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${CI_DIR}/lib.sh" + +APPS_DIR="${APPS_DIR:-${REPO_ROOT}/apps}" +RENDER_PY="${RENDER_PY:-${REPO_ROOT}/catalog/render.py}" +PYTHON="${PYTHON:-python3}" + +# Catalog services that are intentionally NOT onboarded to the apps tree yet +# (computed into a wave by the renderer but with NO committed manifest — see +# DESIGN-waves.md §"Catalog services not yet onboarded"). They render as a +# by-product of `--all catalog` but must NOT have a golden; the gate ignores +# their rendered output. Onboarding one = add its golden and drop it here. +: "${RENDER_GOLDEN_NOT_ONBOARDED:=keycloak vm-demo}" + +log "Stage: render==golden reproducibility (canonical render vs committed apps/)" + +if ! entrypoint_ready "$PYTHON"; then + skip "python3 not available — cannot run the renderer. UNWIRED, not green." + summary "- render-golden: **SKIPPED** (python3 missing)" + exit 3 +fi +if [ ! -f "$RENDER_PY" ]; then + skip "renderer not present at '${RENDER_PY}' (catalog/render.py) — UNWIRED, not green." + summary "- render-golden: **SKIPPED** (renderer missing)" + exit 3 +fi +if [ ! -d "$CATALOG_DIR" ] || ! compgen -G "${CATALOG_DIR}/*.yaml" >/dev/null 2>&1; then + skip "no catalog ServiceDefinitions under '${CATALOG_DIR}' — nothing to render." + summary "- render-golden: **SKIPPED** (empty catalog)" + exit 3 +fi + +# --- Render every service with the ONE canonical command into a temp dir ----- +TMP_RENDER="$(mktemp -d)" +trap 'rm -rf "$TMP_RENDER"' EXIT + +if ! "$PYTHON" "$RENDER_PY" --all "$CATALOG_DIR" -o "$TMP_RENDER" --wave-layout --no-image-automation; then + err "canonical render failed to run" + summary "- render-golden: **FAILED** (renderer error)" + exit 1 +fi + +# Drop the not-onboarded services' rendered output — they have no golden by design. +for svc in $RENDER_GOLDEN_NOT_ONBOARDED; do + find "$TMP_RENDER" -type f -name "${svc}.yaml" -delete +done + +# --- Compare EVERY rendered service against its committed golden -------------- +# For each rendered file (relative path wave-N/.yaml): +# - the committed golden apps/wave-N/.yaml MUST exist, and +# - it MUST be byte-identical to the freshly rendered output. +# Any missing golden or any diff is a hard FAIL (blocks merge). +fail=0 +checked=0 +while IFS= read -r -d '' rendered; do + rel="${rendered#"$TMP_RENDER"/}" + golden="${APPS_DIR}/${rel}" + checked=$((checked + 1)) + if [ ! -f "$golden" ]; then + err "MISSING GOLDEN: catalog renders '${rel}' but apps/${rel} does not exist" + err " -> re-render (see canonical command in this script's header) and commit, or add it to RENDER_GOLDEN_NOT_ONBOARDED" + fail=1 + continue + fi + if ! diff -u "$golden" "$rendered" >/dev/null 2>&1; then + err "GOLDEN DRIFT: apps/${rel} does not match the canonical render:" + diff -u "$golden" "$rendered" >&2 || true + fail=1 + fi +done < <(find "$TMP_RENDER" -type f -print0) + +if [ "$checked" -eq 0 ]; then + err "renderer produced no output — cannot verify goldens" + summary "- render-golden: **FAILED** (renderer produced nothing)" + exit 1 +fi + +if [ "$fail" -ne 0 ]; then + err "render==golden check FAILED — the committed apps/ goldens are NOT reproducible" + err " from the canonical render. A golden was hand-edited, or a catalog source" + err " changed without re-rendering. Fix:" + err " ${PYTHON} catalog/render.py --all catalog -o apps --wave-layout --no-image-automation" + err " then remove the not-onboarded outputs (${RENDER_GOLDEN_NOT_ONBOARDED// /, }) and commit." + summary "- render-golden: **FAILED** (goldens drifted from canonical render)" + exit 1 +fi + +ok "render==golden: all ${checked} onboarded service golden(s) reproduce from the canonical render" +summary "- render-golden: **passed** (${checked} services; canonical \`--no-image-automation\` render)" +exit 0