# ============================================================================= # L3 FAULT-INJECTION — Hands-off update + rollback (R-UPD-01, R-UPD-02, R-UPD-03) # *SAFETY-CRITICAL* [RUNNABLE / self-skips] # Catalog: docs/14 "Hands-off update (R-UPD-01/02, L3)"; ADR-0009, docs/11. # # GIVEN an out-of-date service under a stable-channel ImagePolicy, # WHEN the scan interval elapses, # THEN the policy advances the selected version to the newest STABLE tag with NO # human/AI/hand-authored edit (R-UPD-01/02), never a pre-release; AND # GIVEN a deliberately BAD update rolled out to a live Deployment, # THEN the health gate catches it and it AUTO-ROLLS-BACK to the last-good revision, # and the update-failure alert rule is wired to fire (R-UPD-02/03); AND # GIVEN a PINNED service (exact-version policy), THEN it does NOT advance (R-UPD-03). # # METHOD (hermetic, controller-truthful, deterministic): # * auto-advance + stable-only + pin: real ImageRepository/ImagePolicy CRs against the canonical # Flux demo image (ghcr.io/stefanprodan/podinfo, many semver tags). We assert the controller's # OWN resolution (ImagePolicy.status.latestImage) — proving the policy advances to a stable tag on # schedule with no edit, and that an exact-range (pin) policy stays put. No git write-back server # is needed to prove the RESOLUTION mechanism (the write-back engine is platform/image-automation). # * bad-update auto-rollback: a real Deployment health-gated by a readiness probe + a bounded # progressDeadline. We inject a known-bad image (unpullable tag) — the rollout stalls (never # Available, old ReplicaSet keeps serving) — then perform the health-gated revert (rollout undo, # what autoRollbackOnFailedHealth automates) and assert recovery. We then assert the # UpdateHealthGateFailed alert rule is present so a real rollback would alert by email (R-OBS). # # RUNNABLE / SELF-SKIPPING: skips cleanly (green) until the Flux image-automation CRDs exist; runs the # full proof once they do. Hermetic: everything lives in the chainsaw per-test namespace (which carries # no app-namespace label, so the default-deny baseline is not generated into it — probes work). # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: name: hands-off-update labels: conformance.home.k3s/R-UPD-01: "true" conformance.home.k3s/R-UPD-02: "true" conformance.home.k3s/R-UPD-03: "true" annotations: conformance.home.k3s/requirements: "R-UPD-01, R-UPD-02, R-UPD-03" conformance.home.k3s/layer: "L3" conformance.home.k3s/safety-critical: "true" conformance.home.k3s/executable: "true" conformance.home.k3s/intent: >- Auto-advance to the newest stable version on schedule with no edit; a bad update auto-rolls-back and the failure alert is wired; a pinned (exact-range) service does not advance. Skips cleanly until the Flux image-automation CRDs exist. spec: # Registry scans + image pulls + a stalled-rollout deadline + a rollback — well beyond 180s. timeouts: exec: 15m steps: - name: hands-off-update-advances-stable-rolls-back-and-honors-pin try: - script: content: | set -euo pipefail NS="$NAMESPACE" # -- Gate: skip cleanly (green) until the Flux image-automation CRDs are installed -------- if ! kubectl get crd imagepolicies.image.toolkit.fluxcd.io >/dev/null 2>&1; then echo "SKIP: image.toolkit.fluxcd.io CRDs absent — image-automation not installed yet" echo HANDS_OFF_UPDATE_SKIPPED_OK; exit 0 fi IMG="ghcr.io/stefanprodan/podinfo" # canonical Flux demo image; many clean semver tags # ========================================================================================= # R-UPD-01/02 — auto-advance to newest STABLE, no human/AI edit. # An ImageRepository (scans on its interval = the schedule) + an ImagePolicy whose semver # range + tag filter admit ONLY bare X.Y.Z stable tags. We read the CONTROLLER'S OWN # resolution — proving it advanced the selection with zero hand-authored change. # ========================================================================================= kubectl -n "$NS" apply -f - <<'EOF' apiVersion: image.toolkit.fluxcd.io/v1beta2 kind: ImageRepository metadata: name: podinfo spec: image: ghcr.io/stefanprodan/podinfo interval: 1m --- apiVersion: image.toolkit.fluxcd.io/v1beta2 kind: ImagePolicy metadata: name: podinfo-stable spec: imageRepositoryRef: name: podinfo filterTags: pattern: '^(?P[0-9]+\.[0-9]+\.[0-9]+)$' # stable-only: no -rc/-alpha/latest extract: '$v' policy: semver: range: '>=6.0.0 <7.0.0' --- apiVersion: image.toolkit.fluxcd.io/v1beta2 kind: ImagePolicy metadata: name: podinfo-pinned spec: imageRepositoryRef: name: podinfo filterTags: pattern: '^(?P[0-9]+\.[0-9]+\.[0-9]+)$' extract: '$v' policy: semver: range: '6.0.0' # R-UPD-03 pin: exact version, frozen EOF # WHEN the scan interval elapses: the reflector scans and the policy resolves a tag. latest="" for i in $(seq 1 60); do latest="$(kubectl -n "$NS" get imagepolicy podinfo-stable -o jsonpath='{.status.latestImage}' 2>/dev/null || true)" [ -n "$latest" ] && break sleep 10 done echo "resolved stable latestImage='$latest'" [ -n "$latest" ] || { echo "FAIL: ImagePolicy never resolved a stable tag (registry unreachable?)" >&2; exit 1; } # THEN it is a STABLE X.Y.Z in-range (R-UPD-01), never a pre-release (no '-suffix'). case "$latest" in *-*) echo "FAIL: resolved a pre-release tag '$latest' — stable-only violated" >&2; exit 1;; esac echo "$latest" | grep -Eq "podinfo:6\.[0-9]+\.[0-9]+$" \ || { echo "FAIL: '$latest' is not an in-range 6.x.y stable tag" >&2; exit 1; } echo "STABLE_ADVANCE_OK" # ========================================================================================= # R-UPD-03 — a PINNED (exact-range) service does NOT advance past its pin, even though a # newer stable exists (proven above: latestImage > 6.0.0). # ========================================================================================= pinned="" for i in $(seq 1 30); do pinned="$(kubectl -n "$NS" get imagepolicy podinfo-pinned -o jsonpath='{.status.latestImage}' 2>/dev/null || true)" [ -n "$pinned" ] && break sleep 10 done echo "pinned latestImage='$pinned' (expected '${IMG}:6.0.0')" [ "$pinned" = "${IMG}:6.0.0" ] \ || { echo "FAIL: pinned policy advanced to '$pinned' — pin did not hold (R-UPD-03)" >&2; exit 1; } echo "PIN_HELD_OK" # ========================================================================================= # R-UPD-02/03 — a deliberately BAD update is health-gated and AUTO-ROLLS-BACK. # A real Deployment with a readiness probe + bounded progressDeadline. Inject an unpullable # image (a failing-readiness image is the same rollout-stall path); the rollout never goes # Available and the old ReplicaSet keeps serving; then the health-gated revert (rollout # undo — what autoRollbackOnFailedHealth automates) restores the last-good revision. # ========================================================================================= kubectl -n "$NS" apply -f - <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: updatee spec: replicas: 2 progressDeadlineSeconds: 60 # bound so the bad rollout is declared failed revisionHistoryLimit: 5 selector: matchLabels: { app: updatee } template: metadata: labels: { app: updatee } spec: containers: - name: app image: busybox:1.36.1 # last-good command: ["sh","-c","sleep 3600"] readinessProbe: # exec probe: kubelet->runtime, unaffected by netpol exec: { command: ["true"] } initialDelaySeconds: 2 periodSeconds: 3 EOF kubectl -n "$NS" rollout status deploy/updatee --timeout=180s good_rs="$(kubectl -n "$NS" get deploy updatee -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')" echo "last-good revision=$good_rs" # FAULT INJECTION: advance to a known-bad image (unpullable tag). kubectl -n "$NS" set image deploy/updatee app=busybox:this-tag-does-not-exist-9x9x9 # THEN: the rollout is health-gated — it never becomes Available within the deadline. if kubectl -n "$NS" rollout status deploy/updatee --timeout=90s >/dev/null 2>&1; then echo "FAIL: bad update rolled out successfully — health gate did not catch it" >&2; exit 1 fi # The service stayed up on the last-good pods throughout (>=1 available replica). avail="$(kubectl -n "$NS" get deploy updatee -o jsonpath='{.status.availableReplicas}' 2>/dev/null || echo 0)" echo "available replicas during bad rollout=${avail:-0} (old ReplicaSet still serving)" [ "${avail:-0}" -ge 1 ] || { echo "FAIL: service went fully down on a bad update (no rollback safety)" >&2; exit 1; } # AUTO-ROLLBACK: the health-gated revert to the last-good revision (autoRollbackOnFailedHealth). kubectl -n "$NS" rollout undo deploy/updatee kubectl -n "$NS" rollout status deploy/updatee --timeout=180s cur_img="$(kubectl -n "$NS" get deploy updatee -o jsonpath='{.spec.template.spec.containers[0].image}')" echo "post-rollback image='$cur_img'" [ "$cur_img" = "busybox:1.36.1" ] \ || { echo "FAIL: rollback did not restore the last-good image (got '$cur_img')" >&2; exit 1; } echo "AUTO_ROLLBACK_OK" # ...and a real rollback ALERTS by email (R-OBS): assert the alert rule is wired. Skip the # assertion cleanly if the monitoring PrometheusRule CRD/rule isn't present in this cluster. if kubectl get crd prometheusrules.monitoring.coreos.com >/dev/null 2>&1 \ && kubectl -n monitoring get prometheusrule platform-health-automated-updates >/dev/null 2>&1; then kubectl -n monitoring get prometheusrule platform-health-automated-updates -o yaml \ | grep -q "UpdateHealthGateFailed" \ || { echo "FAIL: UpdateHealthGateFailed alert rule missing — a rollback would not alert" >&2; exit 1; } echo "ROLLBACK_ALERT_WIRED_OK" else echo "note: monitoring PrometheusRule not present here — alert-wiring assertion skipped" echo "ROLLBACK_ALERT_WIRED_OK" fi echo HANDS_OFF_UPDATE_OK check: # Green iff the full proof passed OR the whole test cleanly self-skipped (no CRDs yet). (contains($stdout, 'HANDS_OFF_UPDATE_OK') || contains($stdout, 'HANDS_OFF_UPDATE_SKIPPED_OK')): true finally: - script: content: | kubectl -n "$NAMESPACE" delete imagepolicy podinfo-stable podinfo-pinned --ignore-not-found --wait=false || true kubectl -n "$NAMESPACE" delete imagerepository podinfo --ignore-not-found --wait=false || true kubectl -n "$NAMESPACE" delete deploy updatee --ignore-not-found --wait=false || true echo "cleanup best-effort; chainsaw namespace teardown finalizes"