T-0.4 Chainsaw conformance harness + coverage + behavior scaffolds (R-TEST-01..06)
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
1 parent
a778d923b6
commit
9d8e03892f
31 files changed
+2445
-2
No files matched your search
@@ -0,0 +1,54 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — Alerting (R-OBS-02, R-OBS-04)
|
||||
# Catalog: docs/14 "Alerting (R-OBS-02/04, L3)"
|
||||
#
|
||||
# GIVEN a monitored condition with a defined alert rule,
|
||||
# WHEN a fault is INJECTED to trip the condition,
|
||||
# THEN an email alert is DELIVERED (R-OBS-02) and the dashboard reflects it;
|
||||
# platform self-health conditions (node/storage/Flux/cert/VPN/backup) also
|
||||
# alert (R-OBS-04).
|
||||
#
|
||||
# R-TEST-05 style: the alert must fire from a real injected fault, not a synthetic
|
||||
# test-send. Email delivery is asserted via a captured mailbox / SMTP sink fixture
|
||||
# (no real inbox dependency, hermetic).
|
||||
#
|
||||
# STATUS: scaffold — needs T-4.1 (Alertmanager email) + T-4.2 (platform alerts).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: alerting
|
||||
labels:
|
||||
conformance.home.k3s/R-OBS-02: "true"
|
||||
conformance.home.k3s/R-OBS-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-OBS-02, R-OBS-04"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/intent: >-
|
||||
Injected fault trips a rule; an email alert is delivered; platform self-health
|
||||
also alerts.
|
||||
spec:
|
||||
steps:
|
||||
- name: given-monitored-condition-and-mail-sink
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/smtp-sink.yaml # TODO(T-4.1): hermetic SMTP capture sink
|
||||
- script:
|
||||
content: 'echo "TODO: point Alertmanager email route at the in-cluster sink"'
|
||||
- name: when-inject-fault-to-trip-alert
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(T-4.2): inject a fault (e.g. crashloop / fill volume) to fire a rule"'
|
||||
- name: then-email-alert-delivered
|
||||
try:
|
||||
- script:
|
||||
# Poll the SMTP sink (bounded) for a message matching the fired alert.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO: query smtp-sink for an email whose subject matches the alert"
|
||||
echo "EMAIL_DELIVERED_OK"
|
||||
check: { ($stdout =~ 'EMAIL_DELIVERED_OK'): true }
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "TODO: clear the injected fault + drain the sink (self-cleaning)"'
|
||||
@@ -0,0 +1,55 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — Data survival on rebuild (R-STATE-01)
|
||||
# *SAFETY-CRITICAL*
|
||||
# Catalog: docs/14 "Data survival (R-STATE-01, L3)"
|
||||
#
|
||||
# GIVEN a stateful service with written data (PVC on longhorn),
|
||||
# WHEN the workload is DESTROYED entirely and re-applied from git,
|
||||
# THEN the data is intact.
|
||||
#
|
||||
# R-TEST-05: proves no-data-loss by actually destroying the workload, not by a
|
||||
# happy-path read. Applies to PaaS pods AND `kind: vm` disks (shared with
|
||||
# R-PLAT-03 rebuild).
|
||||
#
|
||||
# STATUS: scaffold — needs T-2.1 (Longhorn StorageClass) + T-1.5 rebuild path.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: data-survival
|
||||
labels:
|
||||
conformance.home.k3s/R-STATE-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-STATE-01"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Destroy the workload entirely and re-apply from git; written data survives.
|
||||
spec:
|
||||
steps:
|
||||
- name: given-stateful-workload-with-written-data
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/stateful-app.yaml # TODO(T-2.1): PVC on longhorn + writer
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
kubectl -n "$NAMESPACE" exec app -- \
|
||||
sh -c 'echo conformance-canary-$(date +%s) > /data/canary && sync'
|
||||
kubectl -n "$NAMESPACE" exec app -- cat /data/canary
|
||||
- name: when-destroy-workload-entirely
|
||||
try:
|
||||
# FAULT INJECTION: delete the workload (NOT the PVC) — simulate rebuild.
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO(T-1.5): delete Deployment/pod entirely; keep PVC; re-apply from git"
|
||||
- name: then-data-is-intact-after-reapply
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
val=$(kubectl -n "$NAMESPACE" exec app -- cat /data/canary)
|
||||
echo "recovered=$val"
|
||||
case "$val" in conformance-canary-*) echo DATA_SURVIVED_OK ;; *) exit 1 ;; esac
|
||||
check: { ($stdout =~ 'DATA_SURVIVED_OK'): true }
|
||||
@@ -0,0 +1,68 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — Default-deny fail-closed (R-NET-01, R-NET-02)
|
||||
# *SAFETY-CRITICAL*
|
||||
# Catalog/spec: docs/14 R-TEST-05 lists default-deny among the behaviors that MUST
|
||||
# be proven by fault-injection (not happy-path). This is the L3 partner of the L2
|
||||
# behavioral default-deny test (tests/l2-behavioral/default-deny).
|
||||
#
|
||||
# GIVEN a source that CAN reach a dest only because of an explicit allow rule,
|
||||
# WHEN the allow rule is REVOKED (fault injection: delete the CiliumNetworkPolicy),
|
||||
# THEN connectivity drops to fail-closed within a bounded window (revocation takes
|
||||
# effect immediately — the baseline denies once the allow is gone), while DNS
|
||||
# still resolves.
|
||||
#
|
||||
# This proves the deny is enforced by the live datapath, not merely declared, and
|
||||
# that removing access fails closed rather than lingering open.
|
||||
#
|
||||
# STATUS: scaffold — needs T-2.2 (baseline) + T-2.3 (rendered allow) + T-3.0 harness.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: default-deny-failclosed
|
||||
labels:
|
||||
conformance.home.k3s/R-NET-01: "true"
|
||||
conformance.home.k3s/R-NET-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-01, R-NET-02"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Revoking an allow rule drops connectivity to fail-closed immediately; DNS
|
||||
still works — the datapath enforces default-deny, not just declares it.
|
||||
spec:
|
||||
steps:
|
||||
- name: given-source-allowed-to-dest
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/allowed-pair.yaml # TODO(T-2.3): source+dest+allow policy
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
kubectl -n "$NAMESPACE" exec source -- nc -z -w 3 dest 80 && echo ALLOWED_OK
|
||||
check: { ($stdout =~ 'ALLOWED_OK'): true }
|
||||
- name: when-revoke-allow-then-fail-closed
|
||||
try:
|
||||
# FAULT INJECTION: delete the allow policy; baseline default-deny remains.
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO(T-3.0): delete the source->dest CiliumNetworkPolicy allow"
|
||||
- script:
|
||||
# Bounded poll: connection must start failing (fail-closed on revocation).
|
||||
content: |
|
||||
set -euo pipefail
|
||||
if kubectl -n "$NAMESPACE" exec source -- nc -z -w 3 dest 80; then
|
||||
echo "LEAK: still reachable after allow revoked" >&2; exit 1
|
||||
fi
|
||||
echo FAIL_CLOSED_OK
|
||||
check: { ($stdout =~ 'FAIL_CLOSED_OK'): true }
|
||||
- script:
|
||||
# DNS must still work (default-deny permits DNS egress).
|
||||
content: |
|
||||
set -euo pipefail
|
||||
kubectl -n "$NAMESPACE" exec source -- nslookup kubernetes.default && echo DNS_OK
|
||||
check: { ($stdout =~ 'DNS_OK'): true }
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "cleanup via chainsaw namespace teardown"'
|
||||
@@ -0,0 +1,52 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — DR restore (R-STATE-03) *SAFETY-CRITICAL*
|
||||
# Catalog: docs/14 "DR restore (R-STATE-03, L3)"
|
||||
#
|
||||
# GIVEN an OFF-CLUSTER backup of a namespace + volumes,
|
||||
# WHEN the namespace + volumes are restored into a CLEAN state,
|
||||
# THEN the service returns with its data.
|
||||
#
|
||||
# R-TEST-05: proves recoverability by restoring into a genuinely clean namespace
|
||||
# (not by reading the still-live original). Backup TARGET is OQ-BKP (OPEN) — the
|
||||
# restore mechanism is Velero (T-4.3/T-4.4); this scaffold is target-agnostic.
|
||||
#
|
||||
# STATUS: scaffold — needs T-4.3 (Velero + off-cluster schedule) + T-4.4 (drill).
|
||||
# BLOCKED-BY spec open question OQ-BKP (off-cluster backup target) — see report.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: dr-restore
|
||||
labels:
|
||||
conformance.home.k3s/R-STATE-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-STATE-03"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Restore a namespace + volumes from an off-cluster backup into a clean state;
|
||||
the service returns with its data.
|
||||
conformance.home.k3s/blocked-by: "OQ-BKP (off-cluster backup target, spec-open)"
|
||||
spec:
|
||||
steps:
|
||||
- name: given-off-cluster-backup-exists
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(T-4.3): ensure a Velero backup of ns+volumes exists off-cluster"'
|
||||
- name: when-restore-into-clean-namespace
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO(T-4.4): create a clean target ns; trigger Velero restore of ns+PVs"
|
||||
- name: then-service-returns-with-data
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO: assert app Ready in restored ns AND canary data present"
|
||||
echo "DR_RESTORE_OK"
|
||||
check: { ($stdout =~ 'DR_RESTORE_OK'): true }
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "TODO: delete the restore target namespace (self-cleaning)"'
|
||||
@@ -0,0 +1,51 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — GitOps drift correction (R-GITOPS-03)
|
||||
# Catalog: docs/14 "GitOps drift (R-GITOPS-03, L3)"
|
||||
#
|
||||
# GIVEN a reconciled resource,
|
||||
# WHEN it is MUTATED live (out-of-band, simulating drift/click-ops),
|
||||
# THEN Flux reverts it to the git-declared state within a bounded window.
|
||||
#
|
||||
# Fault-injection style: we deliberately corrupt live state and assert the control
|
||||
# loop heals it (converges back). `finally` leaves the resource in its declared
|
||||
# state regardless of outcome.
|
||||
#
|
||||
# STATUS: scaffold — needs T-1.4 (Flux bootstrapped) + a reconciled sample resource.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: gitops-drift
|
||||
labels:
|
||||
conformance.home.k3s/R-GITOPS-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-GITOPS-03"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/intent: >-
|
||||
A live mutation to a reconciled resource is reverted by Flux to git state.
|
||||
spec:
|
||||
steps:
|
||||
- name: given-a-reconciled-resource
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(T-1.4): pick a Flux-reconciled resource with a known git value"'
|
||||
- name: when-mutate-live-out-of-band
|
||||
try:
|
||||
# FAULT INJECTION: patch the live resource to a value that differs from git.
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO: kubectl patch the resource to a drifted value (e.g. replicas/annotation)"
|
||||
- name: then-flux-reverts-to-git-state
|
||||
try:
|
||||
- script:
|
||||
# Bounded poll: assert the live value returns to the git-declared value.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO: within Flux interval, assert live value == git-declared value"
|
||||
echo "DRIFT_REVERTED_OK"
|
||||
check: { ($stdout =~ 'DRIFT_REVERTED_OK'): true }
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "TODO: ensure resource left in git-declared state"'
|
||||
@@ -0,0 +1,66 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — Hands-off update + rollback
|
||||
# (R-UPD-01, R-UPD-02, R-UPD-03)
|
||||
# Catalog: docs/14 "Hands-off update (R-UPD-01/02, L3)"; ADR-0009.
|
||||
# R-UPD-02 (auto-rollback) is *SAFETY-CRITICAL*.
|
||||
#
|
||||
# GIVEN an out-of-date service under a stable-channel policy,
|
||||
# WHEN the scan interval elapses,
|
||||
# THEN it advances with NO human/AI/hand-authored git edit (R-UPD-01/02); AND
|
||||
# GIVEN a deliberately BAD update,
|
||||
# WHEN it is rolled out,
|
||||
# THEN health-gating AUTO-ROLLS-BACK and an email alert fires (R-UPD-02); AND
|
||||
# GIVEN a PINNED service, THEN it does not advance (R-UPD-03).
|
||||
#
|
||||
# STATUS: scaffold — needs T-4.5 (image-automation + per-service policy) + T-4.1
|
||||
# (Alertmanager email). The bad-update injection is the R-TEST-05 fault case.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: hands-off-update
|
||||
labels:
|
||||
conformance.home.k3s/R-UPD-01: "true"
|
||||
conformance.home.k3s/R-UPD-02: "true"
|
||||
conformance.home.k3s/R-UPD-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-UPD-01, R-UPD-02, R-UPD-03"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Auto-advance on schedule with no edit; bad update auto-rolls-back + alerts;
|
||||
a pinned service does not advance.
|
||||
spec:
|
||||
steps:
|
||||
- name: given-out-of-date-service-on-stable-channel
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(T-4.5): deploy service pinned one minor behind; stable policy"'
|
||||
- name: then-advances-with-no-human-or-ai-edit
|
||||
try:
|
||||
- script:
|
||||
# R-UPD-01/02: assert image tag advanced via automation, git history shows
|
||||
# only a machine commit (no hand-authored change).
|
||||
content: 'echo "TODO: wait scan interval (bounded); assert tag advanced automatically"'
|
||||
- name: when-inject-bad-update-then-auto-rollback-and-alert
|
||||
try:
|
||||
# FAULT INJECTION: point automation at a known-bad image tag.
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO(T-4.6): inject known-bad image; expect health gate to fail"
|
||||
- script:
|
||||
# THEN: health-gating rolls back to last-good AND an email alert fires.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO: assert rolled back to last-good revision"
|
||||
echo "TODO: assert Alertmanager delivered an email alert (see alerting test)"
|
||||
echo "ROLLBACK_OK"
|
||||
check: { ($stdout =~ 'ROLLBACK_OK'): true }
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "TODO: reset automation to a good tag (self-cleaning)"'
|
||||
- name: then-pinned-service-does-not-advance
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(R-UPD-03): a pinned service stays put across a scan cycle"'
|
||||
@@ -0,0 +1,70 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION SCAFFOLD — VPN kill-switch (R-NET-06) *SAFETY-CRITICAL*
|
||||
# Catalog: docs/14 "VPN kill-switch (R-NET-06, L3)"; ADR-0013 (PIA/Gluetun).
|
||||
#
|
||||
# GIVEN a `vpn`-class workload with a HEALTHY tunnel,
|
||||
# WHEN the tunnel is FORCED DOWN (fault injection),
|
||||
# THEN the workload has ZERO egress to the plain internet (fail-closed), and
|
||||
# `internet`-class workloads are UNAFFECTED;
|
||||
# WHEN the tunnel recovers, THEN connectivity returns.
|
||||
#
|
||||
# R-TEST-05: this MUST break the tunnel and assert no-leak — a happy-path check
|
||||
# does not satisfy R-NET-06. `finally` ALWAYS restores the tunnel so a failure
|
||||
# mid-test never leaves the gateway down (self-healing fixture).
|
||||
#
|
||||
# STATUS: scaffold — needs T-3.0 (fault harness) + T-3.2 (Gluetun gateway).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vpn-killswitch
|
||||
labels:
|
||||
conformance.home.k3s/R-NET-06: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-06"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Tunnel down => vpn workload leaks nothing; internet workloads unaffected;
|
||||
recovery restores connectivity.
|
||||
spec:
|
||||
steps:
|
||||
- name: given-healthy-tunnel-and-two-workloads
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/vpn-and-internet-pods.yaml # TODO(T-3.2): author
|
||||
- script:
|
||||
content: 'echo "TODO: assert tunnel healthy AND vpn workload egress works"'
|
||||
- name: when-force-tunnel-down-then-no-leak
|
||||
try:
|
||||
# FAULT INJECTION: force the shared Gluetun tunnel down (scale/patch/kill).
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
echo "TODO(T-3.0): scale gluetun to 0 / drop wg iface (force tunnel DOWN)"
|
||||
- script:
|
||||
# THEN: vpn workload must reach NOTHING on the plain internet.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
if kubectl -n "$NAMESPACE" exec egress-vpn -- \
|
||||
curl -fsS --max-time 5 https://api.ipify.org; then
|
||||
echo "LEAK: vpn workload reached internet with tunnel down" >&2; exit 1
|
||||
fi
|
||||
echo NO_LEAK_OK
|
||||
check: { ($stdout =~ 'NO_LEAK_OK'): true }
|
||||
- script:
|
||||
# AND: internet-class workload is unaffected.
|
||||
content: |
|
||||
set -euo pipefail
|
||||
kubectl -n "$NAMESPACE" exec egress-internet -- \
|
||||
curl -fsS --max-time 5 https://api.ipify.org && echo INTERNET_UNAFFECTED_OK
|
||||
check: { ($stdout =~ 'INTERNET_UNAFFECTED_OK'): true }
|
||||
- name: when-tunnel-recovers-connectivity-returns
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(T-3.0): restore gluetun; assert vpn egress returns"'
|
||||
# ALWAYS restore the tunnel, even if an assertion above failed.
|
||||
finally:
|
||||
- script:
|
||||
content: |
|
||||
echo "TODO(T-3.0): guarantee gluetun scaled back / wg iface restored"
|
||||
Reference in new issue
Block a user