home-network-k3s-impl/ci/l0.sh

#!/usr/bin/env bash
# ci/l0.sh — Stage 2+3: ServiceDefinition schema validation + Kyverno L0 policy gate.
# Both run in CI WITHOUT a cluster (docs/14 §"Where each layer runs") — live now.
#
#   Stage 2  schema : catalog/*.yaml validated against the ServiceDefinition
#                     schema (owner: T-0.3, schemas/).
#   Stage 3  L0     : `kyverno test` over platform/kyverno/ — default-deny baseline
#                     present, no plaintext secrets, required labels/egress-class,
#                     ServiceDefinition schema conformance (owner: T-0.5, ADR-0011).
#   Stage 3b L0     : conformance/run.sh --layer l0 — the WHOLE-REPO L0 invariants that
#                     a per-resource policy cannot express (secret hygiene across every
#                     tracked file, least-privilege RBAC, spec-immutability enforcement).
#                     Selection = the TOUCHED requirement subset (R-TEST-03) UNION every L0
#                     test DECLARING `conformance.home.k3s/scope: whole-repo`. That union
#                     replaces what used to be a hand-written "UNCONDITIONAL" stage per
#                     whole-repo check (3c subst-safety / 3d keycloak-client-secrets / 3e
#                     bulk-tier): three near-identical 12-line rationales, three hardcoded
#                     check.sh paths and three copies of the same rc 0/3/* case block, which
#                     did not scale and — worse — failed SILENTLY, because whoever added the
#                     next whole-repo invariant had to REMEMBER to hand-write a stage. Now the
#                     test declares its own scope and the runner enforces it. The reasoning and
#                     the incidents those stages recorded were moved INTO the tests they
#                     describe (see each chainsaw-test.yaml's `scope` annotation) and the
#                     general mechanism is documented in conformance/lib/common.sh.
#
# Integration seams are env-overridable (see ci/lib.sh + README). When an owning
# subtree is still a placeholder, this reports SKIPPED (exit 3) for that sub-stage
# rather than a false pass — the gate then knows the seam is unwired, not green.
set -euo pipefail
CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${CI_DIR}/lib.sh"

rc=0

# ---- Stage 2: ServiceDefinition schema validation ---------------------------
log "Stage: ServiceDefinition schema validation (catalog/ against schemas/)"
if ! entrypoint_ready "$SCHEMA_VALIDATE_CMD"; then
  skip "schema validator not present at '${SCHEMA_VALIDATE_CMD}' (T-0.3 / schemas/ not delivered yet) — UNWIRED, not green"
  summary "- schema: **SKIPPED** (validator entrypoint \`${SCHEMA_VALIDATE_CMD}\` missing — T-0.3)"
else
  if [ -d "$CATALOG_DIR" ] && compgen -G "${CATALOG_DIR}/*.y*ml" >/dev/null 2>&1; then
    if "$SCHEMA_VALIDATE_CMD" "${CATALOG_DIR}"/*.y*ml; then
      ok "schema validation passed"
      summary "- schema: **passed**"
    else
      err "schema validation failed"; summary "- schema: **FAILED**"; rc=1
    fi
  else
    log "no ServiceDefinitions in ${CATALOG_DIR} yet — validating validator is callable only"
    "$SCHEMA_VALIDATE_CMD" --version >/dev/null 2>&1 || true
    ok "schema validator present; no catalog entries to validate"
    summary "- schema: **passed** (no catalog entries yet)"
  fi
fi

# ---- Stage 3: Kyverno L0 policy gate ----------------------------------------
log "Stage: Kyverno L0 policy gate (kyverno test ${KYVERNO_POLICY_DIR})"
# `kyverno test <dir>` discovers kyverno-test.yaml files recursively.
if ! have kyverno; then
  skip "kyverno CLI not installed (pin v${KYVERNO_VERSION}) — cannot run L0 gate"
  summary "- kyverno-l0: **SKIPPED** (kyverno CLI not installed)"
  # Missing tool is an environment gap, not a code pass -> exit 3 unless already failing.
  [ "$rc" -eq 0 ] && rc=3
elif [ ! -d "$KYVERNO_POLICY_DIR" ] || ! compgen -G "${KYVERNO_POLICY_DIR}/**/kyverno-test.yaml" >/dev/null 2>&1 \
     && ! find "$KYVERNO_POLICY_DIR" -name 'kyverno-test.yaml' 2>/dev/null | grep -q .; then
  skip "no kyverno-test.yaml under '${KYVERNO_POLICY_DIR}' (T-0.5 / platform/kyverno not delivered yet) — UNWIRED, not green"
  summary "- kyverno-l0: **SKIPPED** (no policy tests yet — T-0.5)"
else
  # The ClusterPolicies carry a Flux postBuild token in validationFailureAction
  # (${KYVERNO_VALIDATION_ACTION}, substituted in-cluster from
  # clusters/base/cluster-vars.yaml). The offline `kyverno test` CLI does NOT run
  # Flux substitution, so the literal token is an invalid enum value and those
  # policies fail to load (no result rows -> "Not found"). Render a throwaway copy
  # with the base/prod value (Enforce — the strict posture the gate must prove)
  # before testing. This does not change what the policies enforce in-cluster.
  : "${KYVERNO_VALIDATION_ACTION:=Enforce}"
  l0_render="$(mktemp -d)"
  cp -a "${KYVERNO_POLICY_DIR}/." "$l0_render/"
  find "$l0_render" -type f -name '*.yaml' -print0 \
    | xargs -0 sed -i "s/\${KYVERNO_VALIDATION_ACTION}/${KYVERNO_VALIDATION_ACTION}/g"
  if kyverno test "$l0_render"; then
    ok "Kyverno L0 policy tests passed"
    summary "- kyverno-l0: **passed**"
  else
    err "Kyverno L0 policy tests failed"; summary "- kyverno-l0: **FAILED**"; rc=1
  fi
  rm -rf "$l0_render"
fi

# ---- Stage 3b: L0 STATIC conformance checks ---------------------------------
# docs/14's L0 row is "manifests/ServiceDefinitions obey invariants", which is wider
# than what Kyverno alone can express over single resources: whole-repo invariants
# (no plaintext secrets anywhere in git, least-privilege RBAC across every tracked
# binding, the spec-immutability gate's own behaviour) live in the conformance
# harness's tests/l0-static subtree and run with no cluster. This stage executes them
# through the same single-command runner CI and a laptop both use (R-TEST-02).
#
# THE UNION, AND WHY IT IS NOT OPTIONAL (R-TEST-03's blind spot)
# --------------------------------------------------------------
# 3b scopes the conformance run to the TOUCHED requirements. That is right for an invariant
# whose inputs live under the subtree its requirements map to. It is WRONG for one that
# quantifies over the WHOLE tracked tree, or over subtrees (apps/, infra/, most of platform/)
# that ci/touched-requirements.sh's coarse path map keys to NOTHING: for those, the change
# most likely to break the invariant attributes no requirement at all, so requirement-scoped
# selection filters the check out of EXACTLY that change. `--include-scope whole-repo` unions
# in every L0 test that DECLARES that scope, so those run on every gate invocation.
#
# rc 3 IS NOT A PASS. The runner returns 3 when a scope-declared check SKIPPED (its subject
# subtree is absent, or it has no check script): UNWIRED, not green, exactly as the deleted
# per-check stages had it ("check SKIPPED (clusters/ absent) — not a pass"). It is mapped to
# the gate's own rc 3 below, never to rc 0.
: "${WHOLE_REPO_SCOPE:=whole-repo}"
log "Stage: L0 static conformance checks (${CONFORMANCE_CMD} --layer l0 --include-scope ${WHOLE_REPO_SCOPE})"
if ! entrypoint_ready "$CONFORMANCE_CMD"; then
  skip "conformance runner not present at '${CONFORMANCE_CMD}' — UNWIRED, not green"
  summary "- l0-static: **SKIPPED** (conformance runner missing)"
  if [ "$rc" -eq 0 ]; then rc=3; fi
else
  load_touched_requirements
  l0_static_rc=0
  "$CONFORMANCE_CMD" --layer l0 --include-scope "$WHOLE_REPO_SCOPE" \
    ${REQ_ARGS[@]+"${REQ_ARGS[@]}"} || l0_static_rc=$?
  case "$l0_static_rc" in
    0) ok "L0 static conformance checks passed"
       summary "- l0-static: **passed**" ;;
    3) skip "an L0 check declaring scope='${WHOLE_REPO_SCOPE}' SKIPPED — UNWIRED, not a pass"
       summary "- l0-static: **SKIPPED** (a whole-repo check is UNWIRED — not a pass)"
       if [ "$rc" -eq 0 ]; then rc=3; fi ;;
    *) err "L0 static conformance checks failed"
       summary "- l0-static: **FAILED**"
       rc=1 ;;
  esac
  # Per-check step-summary lines, so the gate output still NAMES what ran — the generated
  # replacement for the one hardcoded `summary` line each deleted stage carried.
  emit_offline_summary
fi

# ---- Stage 3f: every alert runbook_url resolves to a served runbook (UNCONDITIONAL) ----
# Every platform-health alert rule carries `runbook_url: "https://${BASE_DOMAIN}/runbooks/<name>"` —
# 42 annotations over 34 names. For three roadmap phases there was no runbooks/ directory and no
# HTTPRoute serving that path, so a live in-cluster probe of .../runbooks/node-down returned HTTP 404:
# every alert that reached a human handed them a dead link at the one moment the documentation is read,
# while the annotation made it look solved. The check asserts the whole chain — the URL's shape, the
# runbook's existence, a quality floor that a stub cannot clear, the configMapGenerator entry that
# actually projects it into the server, and the route/listener/nginx prefix agreeing with the
# annotations — because "the file exists in git" and "the URL resolves in the cluster" are different
# claims and only the second one helps an operator.
#
# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c/3d/3e: Stage 3b scopes the conformance run to
# the TOUCHED requirements (R-TEST-03), and ci/touched-requirements.sh maps platform/monitoring/* to
# "R-OBS-01 R-OBS-02" — NOT R-OBS-04 — and maps platform/kubevirt/* and runbooks/* to nothing at all.
# So the single most likely breaking change (adding an alert with a runbook_url, or adding a runbook
# without its generator entry) attributes a requirement set that would FILTER THIS CHECK OUT of exactly
# the change that introduces the bug. The invariant spans platform/, runbooks/ and clusters/, so it runs
# on EVERY gate invocation.
RUNBOOK_CHECK="${RUNBOOK_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/alert-runbooks-resolvable/check.sh}"
log "Stage: alert runbook links resolve (${RUNBOOK_CHECK})"
if [ ! -f "$RUNBOOK_CHECK" ]; then
  skip "alert runbook check not present at '${RUNBOOK_CHECK}' — UNWIRED, not green"
  summary "- alert-runbooks: **SKIPPED** (check missing)"
  if [ "$rc" -eq 0 ]; then rc=3; fi
else
  rb_rc=0
  bash "$RUNBOOK_CHECK" || rb_rc=$?
  case "$rb_rc" in
    0) ok "Alert runbook links resolve"
       summary "- alert-runbooks: **passed**" ;;
    3) skip "alert runbook check SKIPPED (no PrometheusRule present) — not a pass"
       summary "- alert-runbooks: **SKIPPED** (nothing to audit)"
       if [ "$rc" -eq 0 ]; then rc=3; fi ;;
    *) err "Alert runbook links FAILED — an alert links to a runbook that does not exist or is not served"
       summary "- alert-runbooks: **FAILED**"; rc=1 ;;
  esac
fi

case "$rc" in
  0) ok "L0 stage complete" ;;
  3) skip "L0 stage incomplete (unwired seam / missing tool) — not a pass" ;;
  *) err "L0 stage failed" ;;
esac
exit "$rc"