jakman85andClaude Opus 5
63bc890261
INCOMPLETE — preserved by the PM after the authoring agent hit the session
rate limit mid-task. Do not merge without finishing and gating it.
Problem: in ci/gate.sh a FAILED stage sets BLOCK=1 but a SKIPPED stage only
warns, so the gate prints the words 'not green' and then concludes
'MERGE GATE: PASS' and exits 0. Observed live tonight with prom-rules, and
independently with l0 when the pinned kyverno CLI was absent. That is
fail-open in the one component whose whole job is to fail closed.
The agent was mid-way through adding truthy/strict_skips helpers to ci/lib.sh.
WHAT REMAINS:
- Per-stage justification for which skips must BLOCK and which may degrade
to a non-zero UNDECIDED (spec-immutability and L2/L3 skip for legitimate
environmental reasons and must stay usable locally).
- Whether CI and local can be distinguished reliably here.
- New ci/tests/ suite for the decision logic, and a before/after proof.
- Reconciling with the unmerged branch fix/touched-requirements-fail-closed,
whose 47068ee already makes an unusable touched subset block, and whose
56c242d claims 255 of 476 paths were unmapped in the path->requirement
map — a far larger fail-open that still needs measuring on current main.
Co-Authored-By: Claude Opus 5 <[email protected]>
177 lines
9.0 KiB
Bash
Executable File
177 lines
9.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ci/lib.sh — shared helpers for the merge-gate scripts.
|
|
# Sourced by every ci/*.sh; never executed directly.
|
|
#
|
|
# Conventions:
|
|
# - All scripts: `set -euo pipefail`.
|
|
# - Log to stderr; machine-readable output (e.g. requirement lists) to stdout.
|
|
# - Exit codes: 0 = pass, 1 = fail (gate blocks), 3 = SKIPPED/not-run
|
|
# (cluster-gated stage with no cluster — NOT a pass, NOT a fail).
|
|
set -euo pipefail
|
|
|
|
# --- Paths -------------------------------------------------------------------
|
|
# CI_DIR = this ci/ directory (always the real script location). REPO_ROOT = its
|
|
# parent (impl repo root) by default, but OVERRIDABLE via the environment so the
|
|
# scripts can be pointed at a throwaway git repo under test (see ci/tests/) without
|
|
# copying them out of the tree. CI and normal runs leave it unset -> unchanged.
|
|
CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
: "${REPO_ROOT:=$(cd "${CI_DIR}/.." && pwd)}"
|
|
# shellcheck disable=SC1091
|
|
source "${CI_DIR}/versions.env"
|
|
|
|
# --- Integration points (other agents' entrypoints) --------------------------
|
|
# These are the seams this gate depends on. Each has an env override so the PM /
|
|
# owning agent can reconcile the real entrypoint at integration WITHOUT editing
|
|
# these scripts. Defaults are this workstream's *assumptions* — see README.
|
|
: "${CATALOG_DIR:=${REPO_ROOT}/catalog}" # ServiceDefinition sources
|
|
: "${SCHEMAS_DIR:=${REPO_ROOT}/schemas}" # owned by T-0.3 (schema + renderer contract)
|
|
: "${KYVERNO_POLICY_DIR:=${REPO_ROOT}/platform/kyverno}" # owned by T-0.5 (L0 Kyverno policies + tests)
|
|
: "${CONFORMANCE_DIR:=${REPO_ROOT}/conformance}" # owned by T-0.4 (Chainsaw harness + coverage)
|
|
# Commands (overridable). Defaults documented in ci/README.md.
|
|
: "${SCHEMA_VALIDATE_CMD:=${SCHEMAS_DIR}/validate.sh}" # ASSUMED: validates catalog/*.yaml against the ServiceDefinition schema
|
|
: "${CONFORMANCE_CMD:=${CONFORMANCE_DIR}/run.sh}" # ASSUMED: single-command conformance runner (R-TEST-02)
|
|
|
|
# --- Artifact locations (git-ignored: see .gitignore) ------------------------
|
|
: "${RESULTS_DIR:=${REPO_ROOT}/results}" # JUnit XML lands here
|
|
: "${REPORTS_DIR:=${REPO_ROOT}/reports}" # requirement-coverage report lands here
|
|
: "${JUNIT_DIR:=${RESULTS_DIR}/junit}"
|
|
: "${COVERAGE_DIR:=${REPORTS_DIR}/coverage}"
|
|
|
|
# --- Logging -----------------------------------------------------------------
|
|
_is_tty() { [ -t 2 ]; }
|
|
if _is_tty; then C_RED=$'\033[31m'; C_GRN=$'\033[32m'; C_YEL=$'\033[33m'; C_BLU=$'\033[34m'; C_DIM=$'\033[2m'; C_RST=$'\033[0m'
|
|
else C_RED=""; C_GRN=""; C_YEL=""; C_BLU=""; C_DIM=""; C_RST=""; fi
|
|
|
|
log() { printf '%s[gate]%s %s\n' "$C_BLU" "$C_RST" "$*" >&2; }
|
|
ok() { printf '%s[ ok ]%s %s\n' "$C_GRN" "$C_RST" "$*" >&2; }
|
|
warn() { printf '%s[warn]%s %s\n' "$C_YEL" "$C_RST" "$*" >&2; }
|
|
err() { printf '%s[fail]%s %s\n' "$C_RED" "$C_RST" "$*" >&2; }
|
|
skip() { printf '%s[skip]%s %s\n' "$C_YEL" "$C_RST" "$*" >&2; }
|
|
die() { err "$*"; exit 1; }
|
|
|
|
# Emit a GitHub/Gitea-Actions step-summary line if running in Actions.
|
|
summary() { [ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf '%s\n' "$*" >> "$GITHUB_STEP_SUMMARY" || true; }
|
|
|
|
# --- Tooling -----------------------------------------------------------------
|
|
have() { command -v "$1" >/dev/null 2>&1; }
|
|
|
|
# require_tool <bin> <hint> — fail with an actionable message if missing.
|
|
require_tool() {
|
|
have "$1" && return 0
|
|
die "required tool '$1' not found. $2"
|
|
}
|
|
|
|
# entrypoint_ready <path-or-cmd> — returns 0 only if it exists AND is runnable.
|
|
# Used to distinguish 'other agent hasn't delivered yet / non-executable stub'
|
|
# (-> SKIP) from a wired entrypoint that genuinely fails (-> FAIL). A file that
|
|
# exists but isn't executable counts as NOT ready (unwired), not a failure.
|
|
entrypoint_ready() {
|
|
local ep="$1"
|
|
have "$ep" && return 0 # a bare command on PATH
|
|
[ -f "$ep" ] && [ -x "$ep" ] && return 0 # an executable file
|
|
return 1
|
|
}
|
|
|
|
# --- Cluster detection (drives SKIP vs run for L2/L3) -------------------------
|
|
# A cluster is 'available' only if explicitly declared. We NEVER auto-assume a
|
|
# cluster in Phase 0 — silence means SKIP, not pass (HARD RULE 4).
|
|
# CLUSTER_AVAILABLE=true -> L2/L3 run and are REQUIRED green
|
|
# CLUSTER_AVAILABLE=false/unset -> L2/L3 SKIP (reported, never faked green)
|
|
cluster_available() {
|
|
case "${CLUSTER_AVAILABLE:-false}" in
|
|
true|1|yes) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
# truthy <value> — the ONE place the repo's "is this switch on" spelling lives, so
|
|
# CLUSTER_AVAILABLE / REQUIRE_CLUSTER_TESTS / REQUIRE_SPEC_VERIFICATION / GATE_STRICT_SKIPS
|
|
# all agree. Anything not listed is OFF: a misspelled value never turns a guard ON.
|
|
truthy() {
|
|
case "${1:-}" in
|
|
true|TRUE|True|1|yes|YES|y|on|ON) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
# --- Skip strictness: is this a CI runner, or a workstation? -------------------
|
|
# A SKIPPED stage is never green anywhere (HARD RULE 4), but its SEVERITY differs:
|
|
#
|
|
# CI the pinned toolchain is installed by .gitea/workflows/scripts/install-tools.sh
|
|
# before the gate runs, and every seam is on `main`. A stage that skips here is a
|
|
# DEFECT (a tool dropped from the installer, a seam that went unwired) -> BLOCK.
|
|
# local a missing pinned tool is a real, ordinary local condition -> the gate reports
|
|
# UNDETERMINED (exit 3): not green, not a pass, but distinguishable from a red.
|
|
#
|
|
# HOW CI IS DETECTED, AND WHY A WRONG ANSWER CANNOT FAIL OPEN. GITHUB_ACTIONS / CI are set
|
|
# by the Actions runner ITSELF (Gitea's act_runner sets both, as does GitHub) — we do not
|
|
# sniff hostnames, paths or tool availability, because a heuristic that guesses "this looks
|
|
# like CI" is exactly the kind of unreliable inference this gate must not contain. If
|
|
# detection is WRONG in either direction the gate still refuses to pass: a CI run misread as
|
|
# local yields UNDETERMINED, which is non-zero and fails the job all the same; the only thing
|
|
# lost is the harsher BLOCKED label. Override explicitly with GATE_STRICT_SKIPS=true|false
|
|
# (e.g. to rehearse CI strictness locally).
|
|
: "${GATE_STRICT_SKIPS:=auto}"
|
|
strict_skips() {
|
|
case "$GATE_STRICT_SKIPS" in
|
|
auto) truthy "${GITHUB_ACTIONS:-}" || truthy "${CI:-}" ;;
|
|
*) truthy "$GATE_STRICT_SKIPS" ;;
|
|
esac
|
|
}
|
|
|
|
mkartifacts() { mkdir -p "$JUNIT_DIR" "$COVERAGE_DIR"; }
|
|
|
|
# --- Offline (L0/L1) per-check step summary -----------------------------------
|
|
# conformance/lib/run_l1_offline.py writes one Markdown line per check it executed to
|
|
# conformance/_artifacts/offline-summary.md. Splice those into $GITHUB_STEP_SUMMARY so the
|
|
# gate's output still NAMES every check that ran and its verdict.
|
|
#
|
|
# WHY THIS EXISTS: ci/l0.sh used to carry a hand-written stage per whole-repo check, each with
|
|
# its own `summary "- subst-safety: **passed**"` line. Those stages are gone (the tests declare
|
|
# `conformance.home.k3s/scope: whole-repo` instead and the runner unions them in), so the
|
|
# naming is generated for EVERY offline check rather than for the three that had a stage.
|
|
: "${OFFLINE_SUMMARY_FILE:=${CONFORMANCE_DIR}/_artifacts/offline-summary.md}"
|
|
emit_offline_summary() {
|
|
local f="${1:-$OFFLINE_SUMMARY_FILE}" line
|
|
[ -f "$f" ] || return 0
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] && summary " $line"
|
|
done < "$f"
|
|
return 0
|
|
}
|
|
|
|
# --- Touched-requirement subset (R-TEST-03 input) ----------------------------
|
|
# Sets: TOUCHED_RC (0 ok / 3 not determinable), TOUCHED_REQS[] and REQ_ARGS[]
|
|
# (ready to splice into a conformance/run.sh invocation).
|
|
#
|
|
# ci/touched-requirements.sh exits 3 — LOUDLY — when it cannot establish a reliable
|
|
# diff base, instead of the old silent degradation to an empty set. An empty set
|
|
# means run.sh gets no `--requirements`, which means its coverage `--require` gate
|
|
# never runs; treating "could not determine" as "nothing touched" is precisely the
|
|
# false green this helper exists to prevent. A stage still runs its FULL layer in
|
|
# that case (a superset, so nothing goes untested), but says plainly that the
|
|
# SCOPED check did not happen; ci/gate.sh turns that into an UNDECIDED decision.
|
|
load_touched_requirements() {
|
|
local out rc=0
|
|
out="$("${CI_DIR}/touched-requirements.sh")" || rc=$?
|
|
TOUCHED_RC=$rc
|
|
TOUCHED_REQS=()
|
|
# `if`, not `[ ... ] && ...`: under `set -e` a false AND-list is the caller's last
|
|
# command status and would abort the stage script on an EMPTY touched set.
|
|
if [ -n "$out" ]; then mapfile -t TOUCHED_REQS <<< "$out"; fi
|
|
REQ_ARGS=()
|
|
if [ "$rc" -eq 3 ]; then
|
|
warn "touched-requirement subset UNDETERMINED (reason logged above)."
|
|
warn " Running the FULL layer as a superset sanity net; the SCOPED R-TEST-03"
|
|
warn " coverage check is NOT performed here — ci/gate.sh reports that as UNDECIDED."
|
|
elif [ "$rc" -ne 0 ]; then
|
|
warn "touched-requirements.sh exited ${rc}; proceeding with the full layer."
|
|
elif [ "${#TOUCHED_REQS[@]}" -gt 0 ]; then
|
|
log "touched requirements: ${TOUCHED_REQS[*]}"
|
|
REQ_ARGS=(--requirements "$(IFS=,; echo "${TOUCHED_REQS[*]}")")
|
|
else
|
|
log "empty touched-requirement set (base WAS established) — running the full layer."
|
|
fi
|
|
return 0
|
|
}
|