Compare commits
27
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f341a9b744 | ||
|
|
6f61c9622d | ||
|
|
3ade1d440f | ||
|
|
cb1916f092 | ||
|
|
5ec1460532 | ||
|
|
a984442cae | ||
|
|
da3e9cc3f3 | ||
|
|
04569abd78 | ||
|
|
21e111b08c | ||
|
|
577c62136b | ||
|
|
fe45d0f5c5 | ||
|
|
8f9b2e4270 | ||
|
|
59ffbaf11a | ||
|
|
765ad032db | ||
|
|
32d67660a6 | ||
|
|
9063e53ce2 | ||
|
|
6d62aca0d2 | ||
|
|
b80e8eef82 | ||
|
|
e6bcecc78c | ||
|
|
acb5cabdad | ||
|
|
4f37fc375e | ||
|
|
1b056dc9d9 | ||
|
|
4160fee6ce | ||
|
|
7432012713 | ||
|
|
6b88670129 | ||
|
|
13081a863f | ||
|
|
9137bbd0c3 |
No files matched your search
@@ -39,21 +39,6 @@ install_chainsaw() {
|
||||
curl -fsSL "$url" | tar -xz -C "$bindir" chainsaw
|
||||
}
|
||||
|
||||
# promtool — the alert-rule UNIT-TEST engine the L0 check
|
||||
# conformance/tests/l0-static/alert-rules-unit-tested runs. Installed for the l0 stage because
|
||||
# without it that check exits 3 (SKIPPED, never green) on the runner and the alert unit tests
|
||||
# would be enforced nowhere — the exact "coverage that looks present" failure it exists to end.
|
||||
# Pinned from ci/versions.env to the Prometheus version prod runs (promtool IS the rule engine);
|
||||
# the check asserts `promtool --version` equals that pin, so a drifted install FAILS, not skips.
|
||||
install_promtool() {
|
||||
have promtool && return 0
|
||||
log "promtool ${PROMTOOL_VERSION} (from the prometheus release tarball)"
|
||||
local tarball="prometheus-${PROMTOOL_VERSION}.${os}-${arch}"
|
||||
curl -fsSL "https://github.com/prometheus/prometheus/releases/download/v${PROMTOOL_VERSION}/${tarball}.tar.gz" \
|
||||
| tar -xz -C "$bindir" --strip-components=1 "${tarball}/promtool"
|
||||
chmod +x "${bindir}/promtool"
|
||||
}
|
||||
|
||||
install_kubectl() {
|
||||
have kubectl && return 0
|
||||
log "kubectl ${KUBECTL_VERSION}"
|
||||
@@ -65,10 +50,10 @@ have() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
case "$stage" in
|
||||
lint) install_yamllint ;;
|
||||
l0) install_yamllint; install_check_jsonschema; install_kyverno; install_promtool ;; # schema (check-jsonschema) + Kyverno L0 + alert unit tests
|
||||
l0) install_yamllint; install_check_jsonschema; install_kyverno ;; # schema (check-jsonschema) + Kyverno L0
|
||||
l1) install_chainsaw ;; # L1 render via conformance runner (Chainsaw)
|
||||
conformance) install_chainsaw; install_kubectl ;; # trigger the in-cluster Job
|
||||
all) install_yamllint; install_check_jsonschema; install_kyverno; install_promtool; install_chainsaw; install_kubectl ;;
|
||||
all) install_yamllint; install_check_jsonschema; install_kyverno; install_chainsaw; install_kubectl ;;
|
||||
*) echo "unknown stage: $stage" >&2; exit 2 ;;
|
||||
esac
|
||||
log "stage '${stage}' tools ready"
|
||||
@@ -44,3 +44,6 @@ infra/tofu/.r2.env
|
||||
|
||||
# Vendored ansible collections (installed by ansible-galaxy for node re-provision; not tracked or linted)
|
||||
infra/ansible/collections/
|
||||
|
||||
# standup.sh DNS snapshots — run artifacts, not authored state
|
||||
.standup-dns/
|
||||
@@ -1,57 +0,0 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -1,95 +0,0 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|auth.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|conformance-dns.home.keithjackson.dev|192.168.1.56
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|crafty.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|gitea.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|prod.home.keithjackson.dev|192.168.2.20
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
A|unifi.prod.home.keithjackson.dev|192.168.2.21
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-conformance-dns.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/chainsaw-growing-grouse/dns-probe-svc
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -1,93 +0,0 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|auth.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|crafty.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|gitea.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|prod.home.keithjackson.dev|192.168.2.20
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
A|unifi.prod.home.keithjackson.dev|192.168.2.21
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
+268
-1231
File diff suppressed because it is too large.
Load diff
@@ -69,6 +69,17 @@ infra/devcontainer/dev.sh run make -C ci gate # the same gate CI runs
|
||||
infra/devcontainer/dev.sh # interactive shell (sops, flux, kubectl, tofu, ansible)
|
||||
```
|
||||
|
||||
`kubectl`/`flux`/`helm` in there talk to the **dev** cluster by default. Prefix `CLUSTER=prod` to
|
||||
target prod instead:
|
||||
|
||||
```bash
|
||||
infra/devcontainer/dev.sh run kubectl get nodes # dev (default)
|
||||
CLUSTER=prod infra/devcontainer/dev.sh run kubectl get nodes # prod
|
||||
```
|
||||
|
||||
Both clusters are always present in the container's `KUBECONFIG` as the contexts `k3s-dev` and
|
||||
`k3s-prod`; `CLUSTER` picks which one is current, and each run prints the cluster it is targeting.
|
||||
|
||||
The container mounts the repo at `/work`, your age keyring, and the sibling spec repo read-only
|
||||
so the spec-immutability check can actually run. See
|
||||
[`infra/devcontainer/README.md`](infra/devcontainer/README.md).
|
||||
|
||||
@@ -170,27 +170,6 @@ spec:
|
||||
- port: '3000'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy in-cluster client ingress allow (R-NET-02): gitea-allow-flux-ingress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: gitea-allow-flux-ingress
|
||||
namespace: gitea
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: flux-system
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '3000'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): gitea-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
containers:
|
||||
- name: jackett
|
||||
@@ -83,7 +83,7 @@ spec:
|
||||
- containerPort: 9117
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -97,7 +97,6 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -267,7 +266,7 @@ metadata:
|
||||
app.kubernetes.io/name: jackett
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -11,7 +11,6 @@ resources:
|
||||
- jackett-oauth2-proxy-credentials.sops.yaml
|
||||
- jackett.yaml
|
||||
- nextcloud-db-app.sops.yaml
|
||||
- nextcloud-db-backup-credentials-prod.sops.yaml
|
||||
- nextcloud-db-backup-credentials.sops.yaml
|
||||
- nextcloud-db.yaml
|
||||
- pihole-oauth2-proxy-credentials.sops.yaml
|
||||
@@ -20,6 +19,5 @@ resources:
|
||||
- qbittorrent-oauth2-proxy-credentials.sops.yaml
|
||||
- qbittorrent.yaml
|
||||
- semaphore-db-app.sops.yaml
|
||||
- semaphore-db-backup-credentials-prod.sops.yaml
|
||||
- semaphore-db-backup-credentials.sops.yaml
|
||||
- semaphore-db.yaml
|
||||
@@ -1,29 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: nextcloud-db-backup-credentials-prod
|
||||
namespace: nextcloud-db
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:4lx6et4HcXDegulyIoa7jZR/lVgapGPEkukajfCKjvk=,iv:p6V7tizbikk55tOvpPOsPHTe/p52QLX5A5xrwuB14rA=,tag:oNJuipT/Lb6zynJ2K2Q71A==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:V5bHPoPeBOVyLp1n1Jilyso4i5P2aPSH0hrryfweg2q/SctWZxRbLBQ/FLzOigbJsAhlwOGu069mQPma6xjfdw==,iv:8u4UOvI8dC/cm6mEWDlyN5E3SfRZmwKY9uZY/DjPEsQ=,tag:tw7tGs29TmUvMPBCtCszAA==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiSUt5N2k2N2lDU2ZsMy9m
|
||||
M1NFS0M2QjFsNFhnR0hCMzFyaTdiQWFZRFJZCmhBTWRXNmxLMGtsc3Rxd2FSMDZp
|
||||
aktjcTE2c1hmU1EzM3RWNksyb3Q1YVUKLS0tIDBFS1I5aDBER2cvcWtLZ3BXTEp5
|
||||
ZGI5ZmZIcStheUNQamxIVVVpc0ZMOHMKlE5aHewtHJpeU24F5sodc9CEm9KTjHQQ
|
||||
8ThEQ/8vyazL9V4ojlICxSHU/SHkHpD1eBaTKcAQd/LZzujIcJ+QfA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:8awz1Jh8t4zjDMyd93/v+6on9iiIaSi01293A7uYhxj9L1+g63zeO+DPmSn6+KbZvHdqU2A7up09H1NzfXAEdGuvVbB36cdGq0zKank3PUi8c7IT6qNuWydxRAcqLnbbD+c30Z6GB+gAuZHstNOe4Rgc4aJvMN6FuP6x6Z0870c=,iv:KGGKJq2QA5xztvoHs/SViXW72DYwT16i+W+dJBUeIJQ=,tag:3UpYqx2+RIK0QdMH/BSIpA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
+17
-294
@@ -37,53 +37,16 @@ metadata:
|
||||
name: nextcloud-db
|
||||
namespace: nextcloud-db
|
||||
spec:
|
||||
# THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db; see
|
||||
# platform/keycloak/postgres-cluster.yaml for the full rationale. Short version: Longhorn's 3-way
|
||||
# replication saves the DATA on node loss, not the SERVICE, and with `instances: 1` all three CNPG
|
||||
# primaries had drifted onto k3s-prod-2, so one node took Nextcloud down alongside SSO and
|
||||
# Semaphore. A streaming standby lets CNPG promote in seconds and repoint the -rw Service.
|
||||
# Gated on the Barman archive below actually working (ContinuousArchiving=True with a real
|
||||
# firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup.
|
||||
#
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
# Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as
|
||||
# keycloak-db). The storage is already 3-way replicated by Longhorn; raise to 3 for Postgres-level
|
||||
# HA as a future hardening.
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
# Stamp app.kubernetes.io/name=nextcloud-db onto every generated pod/service so the rendered
|
||||
# nextcloud egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud-db
|
||||
# HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the
|
||||
# primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and
|
||||
# recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short
|
||||
# version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance
|
||||
# after a node loss it may stack two instances on one node — which is how `instances: 3` decays
|
||||
# back into the single-node failure domain this change removes. `required` makes it a constraint;
|
||||
# the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still
|
||||
# serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes.
|
||||
#
|
||||
# APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one
|
||||
# switchover. Expect a single brief connection reset for Nextcloud.
|
||||
affinity:
|
||||
enablePodAntiAffinity: true
|
||||
podAntiAffinityType: required
|
||||
primaryUpdateMethod: switchover
|
||||
# SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full
|
||||
# rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the
|
||||
# last few transactions when a primary dies before its standbys catch up, and `method: any` +
|
||||
# `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`,
|
||||
# never `required` — `required` blocks writes outright when fewer than `number` healthy standbys
|
||||
# are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded
|
||||
# standby into a total write outage. `preferred` relaxes to async instead.
|
||||
#
|
||||
# NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter —
|
||||
# no restart, no switchover, no client disconnect. Land it after the anti-affinity roll.
|
||||
postgresql:
|
||||
synchronous:
|
||||
method: any
|
||||
number: 1
|
||||
dataDurability: preferred
|
||||
storage:
|
||||
storageClass: longhorn
|
||||
size: 5Gi
|
||||
@@ -105,38 +68,29 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
|
||||
# backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/nextcloud-db/ — dev
|
||||
# s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod
|
||||
# s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and
|
||||
# destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
|
||||
# Credentials: PER-ENVIRONMENT (2026-09-07). dev reads nextcloud-db-backup-credentials (the
|
||||
# scoped MinIO key), prod reads nextcloud-db-backup-credentials-prod (the Cloudflare R2
|
||||
# token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
|
||||
# generated golden — render.py places the -prod sidecar in this DB's wave); the
|
||||
# cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
|
||||
# one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
|
||||
# ONE shared Secret until the split, so giving prod its R2 token would have left dev
|
||||
# signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
|
||||
# disk-full clock below rather than a merely missing backup.
|
||||
#
|
||||
# archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
|
||||
# Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
|
||||
# the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
|
||||
# endpoint. Treat ContinuousArchiving=False here as an availability incident.
|
||||
# backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
|
||||
# (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP.
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials
|
||||
# (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
|
||||
# substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
|
||||
backup:
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
# PER-ENVIRONMENT PATH (cluster-vars). CNPG appends serverName, which defaults to the CLUSTER
|
||||
# NAME — and both environments have a `nextcloud-db`. Sharing one MinIO with a shared path
|
||||
# would interleave two Postgres servers' WAL into ONE Barman stream, corrupting the chain and
|
||||
# destroying PITR for both. Dev keeps `s3://cnpg` (its existing, PROVEN backups); prod nests.
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: nextcloud-db-backup-credentials
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: nextcloud-db-backup-credentials
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
@@ -158,234 +112,3 @@ spec:
|
||||
cluster:
|
||||
name: nextcloud-db
|
||||
method: barmanObjectStore
|
||||
---
|
||||
# =============================================================================
|
||||
# CNPG METRICS-EXPORTER CONNECT RECONCILER (R-OBS-01)
|
||||
#
|
||||
# THE BUG THIS CLOSES (true positive, verified live on prod 2026-09-12, and the
|
||||
# condition platform/cnpg/alerts.yaml already recorded on BOTH clusters 2026-09-06):
|
||||
# the `nextcloud` database's ACL had been narrowed out-of-band so that PUBLIC held
|
||||
# nothing, and CNPG's metrics exporter role therefore could not even CONNECT:
|
||||
#
|
||||
# nextcloud-db-1/-2/-3, every scrape:
|
||||
# FATAL: permission denied for database "nextcloud" (SQLSTATE 42501)
|
||||
# DETAIL: User does not have CONNECT privilege.
|
||||
# msg="Error collecting user query" targetDatabase=nextcloud
|
||||
#
|
||||
# CONSEQUENCE: `cnpg_last_error` = 1 on all three instances and the ENTIRE
|
||||
# cnpg_pg_stat_* / pg_settings / pg_postmaster family MISSING for this database,
|
||||
# while the scrape target still reported up=1 — a blind spot that looks like health.
|
||||
# Measured before the fix: cnpg_pg_settings_setting had 765 series for keycloak and
|
||||
# 765 for semaphore-db and ZERO for nextcloud-db. Nextcloud's database had never
|
||||
# been monitored.
|
||||
#
|
||||
# ROOT CAUSE, AND WHY ONLY THIS DATABASE. All three Cluster specs are identical in
|
||||
# their bootstrap; the difference is entirely in the live ACL:
|
||||
#
|
||||
# nextcloud : datacl = {nextcloud=CTc/nextcloud} <-- explicit, no PUBLIC entry
|
||||
# keycloak : datacl = NULL
|
||||
# semaphore : datacl = NULL
|
||||
#
|
||||
# A NULL datacl means "PostgreSQL defaults", which include CONNECT and TEMP for
|
||||
# PUBLIC — so keycloak and semaphore let the exporter in only by inheriting PUBLIC's
|
||||
# implicit CONNECT. On `nextcloud` something ran the equivalent of
|
||||
# `REVOKE ALL ON DATABASE nextcloud FROM PUBLIC`, which MATERIALISES datacl and
|
||||
# drops the implicit PUBLIC entry with it. That REVOKE is not in this repo and not
|
||||
# in DECISIONS-LOG.md — it was applied out of band, which is exactly why a
|
||||
# git-declared fix is needed rather than another hand-run psql.
|
||||
#
|
||||
# WHY THE GRANT IS EXPLICIT RATHER THAN "RE-GRANT CONNECT TO PUBLIC": restoring
|
||||
# PUBLIC's CONNECT would undo whatever hardening that REVOKE intended and would
|
||||
# leave the exporter depending on an implicit default again — the precise fragility
|
||||
# that broke here and that keycloak/semaphore are still relying on. Naming the
|
||||
# exporter role makes the dependency explicit and survives a future REVOKE FROM
|
||||
# PUBLIC on any of the three databases.
|
||||
#
|
||||
# WHY A CronJob AND NOT A CNPG FIELD — CNPG 1.30.0's API GENUINELY CANNOT EXPRESS
|
||||
# THIS. Checked against the live CRDs on prod:
|
||||
# * `Database` (databases.postgresql.cnpg.io) has no database-ACL field at all.
|
||||
# `allowConnections` is the pg_database.datallowconn FLAG (open/closed to
|
||||
# everyone), not a per-role GRANT; its only grant/revoke verbs are the `usage`
|
||||
# lists on `fdws` and `servers`.
|
||||
# * `spec.managed.roles` and `DatabaseRole` manage ROLE ATTRIBUTES (login,
|
||||
# superuser, connectionLimit, ...) plus `inRoles` MEMBERSHIP. Membership is the
|
||||
# wrong tool: the only local role holding CONNECT here is the OWNER, so
|
||||
# `inRoles: [nextcloud]` would hand the exporter the owner's CREATE and TEMP on
|
||||
# the database and read/write on every application table — the opposite of
|
||||
# least privilege, to buy one CONNECT.
|
||||
# * `bootstrap.initdb.postInitApplicationSQL` runs ONCE, at bootstrap. It cannot
|
||||
# repair these three already-bootstrapped instances (the actual bug), and on a
|
||||
# FRESH cluster the grant is unnecessary anyway (datacl starts NULL). Adding it
|
||||
# would be a second mechanism that fixes nothing this one does not.
|
||||
# So the SQL has to be applied by something, and this repo already has the pattern
|
||||
# for state Flux cannot reach: the level-triggered CronJob reconciler in
|
||||
# platform/keycloak/client-secret-reconciler.yaml, whose reasoning transfers
|
||||
# verbatim — the drifted state lives inside PostgreSQL, not in a Kubernetes object,
|
||||
# so no admission or mutation policy can see it. A one-shot Job is deliberately NOT
|
||||
# used: a completed Job is never re-run, so a DB rebuild or a PITR restore would
|
||||
# silently reopen the blind spot. A loop converges instead.
|
||||
#
|
||||
# PRIVILEGE GRANTED — EXACTLY ONE, AND NOTHING ELSE:
|
||||
# GRANT CONNECT ON DATABASE nextcloud TO cnpg_metrics_exporter;
|
||||
# Resulting ACL entry is `cnpg_metrics_exporter=c/nextcloud` — the `c` is CONNECT;
|
||||
# there is no `C` (CREATE) and no `T` (TEMPORARY). No ownership, no superuser, no
|
||||
# write, no table or schema grants. Nothing further is needed because CNPG already
|
||||
# creates this role as a member of the built-in `pg_monitor` role (verified on all
|
||||
# three prod clusters), and pg_monitor already carries the read access every
|
||||
# collector query wants — they read pg_catalog views (pg_stat_database,
|
||||
# pg_stat_archiver, pg_settings, pg_postmaster) and no application tables. Proof
|
||||
# that CONNECT alone is sufficient: after this grant, `Error collecting user query`
|
||||
# went to ZERO on all three pods, i.e. every collector query now succeeds.
|
||||
#
|
||||
# IT RUNS AS THE DATABASE OWNER, NOT AS A SUPERUSER. `enableSuperuserAccess` is off
|
||||
# on this Cluster (there is no nextcloud-db-superuser Secret), and it stays off: the
|
||||
# owner of a database holds the grant option on that database's privileges, so
|
||||
# `nextcloud` can issue this GRANT itself. Verified live before committing, by
|
||||
# running the statement under `SET ROLE nextcloud`. Credentials come from the
|
||||
# in-namespace SOPS Secret nextcloud-db-app that already exists for the app.
|
||||
#
|
||||
# NETWORKING: nextcloud-db is intentionally NOT an app-namespace (see this file's
|
||||
# header), so Kyverno generates no default-deny baseline here and this same-namespace
|
||||
# pod -> nextcloud-db-rw:5432 connection needs no rendered allow.
|
||||
#
|
||||
# THE POD LABEL IS DELIBERATELY *NOT* app.kubernetes.io/name=nextcloud-db. That
|
||||
# label is what `inheritedMetadata` stamps on the CNPG instance pods and what the
|
||||
# rendered nextcloud egress CiliumNetworkPolicy selects; reusing it here would quietly
|
||||
# make this reconciler a permitted destination for the Nextcloud app. It gets its own
|
||||
# name so that selector keeps meaning "the database".
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: nextcloud-db-exporter-grant
|
||||
namespace: nextcloud-db
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud-db-exporter-grant
|
||||
app.kubernetes.io/part-of: nextcloud-db
|
||||
spec:
|
||||
# Every 15 minutes, matching the keycloak reconciler's cadence. A converged
|
||||
# database makes this one idempotent GRANT and exits; the cost is negligible, and
|
||||
# after a rebuild or PITR restore monitoring is blind for at most one interval
|
||||
# (well inside the alert's `for: 6h`).
|
||||
schedule: "*/15 * * * *"
|
||||
# A run still waiting on the primary must not be lapped by the next tick.
|
||||
concurrencyPolicy: Forbid
|
||||
startingDeadlineSeconds: 300
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 300
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud-db-exporter-grant
|
||||
app.kubernetes.io/part-of: nextcloud-db
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
# Talks only to PostgreSQL; its one input is a mounted Secret, so it needs
|
||||
# no Kubernetes API access at all (R-SEC-02, least privilege).
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
# uid/gid 26 is `postgres` in the CNPG PostgreSQL image.
|
||||
runAsUser: 26
|
||||
runAsGroup: 26
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: grant
|
||||
# The SAME image the instances above run, pinned to the same tag. psql
|
||||
# is already in it and it is already present on every node, so this
|
||||
# adds no new image and no new supply-chain surface. Keep the tag in
|
||||
# step with spec.imageName above when Renovate bumps it.
|
||||
image: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- psql
|
||||
# -w never prompts; -X skips ~/.psqlrc; ON_ERROR_STOP makes the Job
|
||||
# FAIL LOUDLY rather than reporting success on a refused grant.
|
||||
- "-w"
|
||||
- "-X"
|
||||
- "-v"
|
||||
- "ON_ERROR_STOP=1"
|
||||
- "-f"
|
||||
- /etc/grant/grant.sql
|
||||
env:
|
||||
# libpq connection, assembled from the owner Secret. Target the -rw
|
||||
# Service: GRANT is a write and must reach the primary.
|
||||
- name: PGHOST
|
||||
value: nextcloud-db-rw.nextcloud-db.svc.cluster.local
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
# Connecting to the application database itself is deliberate: the
|
||||
# owner has CONNECT on it, and a successful connection is the same
|
||||
# check the exporter was failing.
|
||||
- name: PGDATABASE
|
||||
value: nextcloud
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nextcloud-db-app
|
||||
key: username
|
||||
# Read by libpq from the environment and never logged; psql echoes
|
||||
# no connection parameters.
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nextcloud-db-app
|
||||
key: password
|
||||
- name: PGCONNECT_TIMEOUT
|
||||
value: "10"
|
||||
volumeMounts:
|
||||
- name: sql
|
||||
mountPath: /etc/grant
|
||||
readOnly: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
volumes:
|
||||
- name: sql
|
||||
configMap:
|
||||
name: nextcloud-db-exporter-grant
|
||||
defaultMode: 0444
|
||||
---
|
||||
# The desired state, as SQL. Idempotent: re-granting an existing privilege is a
|
||||
# no-op, so a converged run changes nothing. The DO block is the ASSERTION — it
|
||||
# turns "the grant did not take" into a failed Job instead of a silent success.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nextcloud-db-exporter-grant
|
||||
namespace: nextcloud-db
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud-db-exporter-grant
|
||||
app.kubernetes.io/part-of: nextcloud-db
|
||||
data:
|
||||
grant.sql: |
|
||||
-- The ONE privilege the CNPG metrics exporter is missing on this database.
|
||||
-- CONNECT only: no CREATE, no TEMPORARY, no ownership, no table/schema grants.
|
||||
-- Read access for the collector's pg_catalog queries already comes from the
|
||||
-- built-in pg_monitor role, which CNPG grants this role when it creates it.
|
||||
GRANT CONNECT ON DATABASE nextcloud TO cnpg_metrics_exporter;
|
||||
|
||||
-- Fail the Job if the privilege is somehow still absent, so a broken
|
||||
-- reconciler surfaces as a failed Job rather than as a quiet blind spot.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT has_database_privilege('cnpg_metrics_exporter', 'nextcloud', 'CONNECT') THEN
|
||||
RAISE EXCEPTION 'cnpg_metrics_exporter still lacks CONNECT on database nextcloud';
|
||||
END IF;
|
||||
RAISE NOTICE 'converged: cnpg_metrics_exporter has CONNECT on database nextcloud';
|
||||
END
|
||||
$$;
|
||||
@@ -35,34 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume plex-media - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: plex-media
|
||||
labels:
|
||||
app.kubernetes.io/name: plex
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
readOnly: true
|
||||
claimRef:
|
||||
namespace: plex
|
||||
name: plex-media
|
||||
---
|
||||
# PVC plex-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -79,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 8Gi
|
||||
---
|
||||
# PVC plex-media (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: plex-media
|
||||
namespace: plex
|
||||
labels:
|
||||
app.kubernetes.io/name: plex
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
storageClassName: ''
|
||||
volumeName: plex-media
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -135,16 +90,10 @@ spec:
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: plex-config
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: plex-media
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
@@ -35,33 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume qbittorrent-downloads - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: qbittorrent-downloads
|
||||
labels:
|
||||
app.kubernetes.io/name: qbittorrent
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
claimRef:
|
||||
namespace: qbittorrent
|
||||
name: qbittorrent-downloads
|
||||
---
|
||||
# PVC qbittorrent-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -78,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
# PVC qbittorrent-downloads (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: qbittorrent-downloads
|
||||
namespace: qbittorrent
|
||||
labels:
|
||||
app.kubernetes.io/name: qbittorrent
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ''
|
||||
volumeName: qbittorrent-downloads
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -118,24 +74,16 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
containers:
|
||||
- name: qbittorrent
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
- name: PUID
|
||||
value: '1000'
|
||||
- name: PGID
|
||||
value: '3000'
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: downloads
|
||||
mountPath: /downloads
|
||||
subPath: downloads
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -149,7 +97,6 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -201,15 +148,10 @@ spec:
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
securityContext:
|
||||
fsGroup: 3000
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: qbittorrent-config
|
||||
- name: downloads
|
||||
persistentVolumeClaim:
|
||||
claimName: qbittorrent-downloads
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
@@ -324,7 +266,7 @@ metadata:
|
||||
app.kubernetes.io/name: qbittorrent
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: semaphore-db-backup-credentials-prod
|
||||
namespace: semaphore-db
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:OhWEEYqrvmDB40NmiETgeUisBvaO9ocevCZAsurNijw=,iv:kEjOIg9NHbRYm+Ul1WrBAzb3gLbIv+ipiYCxFK0QoVc=,tag:KAB1JAIw0ATwbjDEKZVT9A==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:C0PNKv07LCYpFONLqd55sz+ioyU/i2YJUkbZ8OX/dCErM/DilqchyUAT5EqnT/r3YJmdaA1pX85S3fxlVVHkbg==,iv:4ha28b+xIebvX2JPITmtTovIe4vdmj2XfI3cgkUHH4c=,tag:xbMyflcHojytPzgaoOYcXQ==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZVTJ3cXR6cks3Y0RNS2hx
|
||||
K3pjZEduYUxmL0V3SmFUcU9XbGVHSVFITWh3Cjl1eHlTLytoOW5PaVNFVkJoeTBo
|
||||
T1FnRHNDVzJ6dWpuR09WaElvbWF4YU0KLS0tIEJoQ1VsM1cxRlg0R1dtNGdOenNn
|
||||
aVNjMjVBTytLdHlFdmZCS2tPelo1Y2MKYt1G5t+sDiHgcg+Q9ECBzJn2gS/paSEC
|
||||
7aoX60SBrPgHvnGls5lzc3Tin9nH4jWYl6Nu8DAvIE0YPXDFr5a0UA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:mhVP7tjdZK2FR2imvpJbTyT7BigvcIKEpLSnBS1oSITUirB8xZC7aHBfyFuIoKOrpOJgEe40nHUdDYCHRJtb4dLeVja8+7meVaQWzxC8BWVHsksW+73hsqhjAzlHeobx6Irbk4PAMqIY0/jnf4MHryjnRn3dUuwu9U1CqL30JbE=,iv:57oCcp6FW9EAIVKnYGtkaTrnbijyv54T+vmZxMbMqnw=,tag:agrRP/JLypU60KOeN0s/5w==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -38,53 +38,16 @@ metadata:
|
||||
name: semaphore-db
|
||||
namespace: semaphore-db
|
||||
spec:
|
||||
# THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db /
|
||||
# nextcloud-db; see platform/keycloak/postgres-cluster.yaml for the full rationale. Short version:
|
||||
# Longhorn's 3-way replication saves the DATA on node loss, not the SERVICE, and with
|
||||
# `instances: 1` all three CNPG primaries had drifted onto k3s-prod-2, so one node took Semaphore
|
||||
# down alongside SSO and Nextcloud. A streaming standby lets CNPG promote in seconds and repoint
|
||||
# the -rw Service. Gated on the Barman archive below actually working (ContinuousArchiving=True
|
||||
# with a real firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup.
|
||||
#
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
# Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as
|
||||
# keycloak-db / nextcloud-db). The storage is already 3-way replicated by Longhorn; raise to 3 for
|
||||
# Postgres-level HA as a future hardening.
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
# Stamp app.kubernetes.io/name=semaphore-db onto every generated pod/service so the rendered
|
||||
# semaphore egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: semaphore-db
|
||||
# HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the
|
||||
# primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and
|
||||
# recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short
|
||||
# version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance
|
||||
# after a node loss it may stack two instances on one node — which is how `instances: 3` decays
|
||||
# back into the single-node failure domain this change removes. `required` makes it a constraint;
|
||||
# the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still
|
||||
# serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes.
|
||||
#
|
||||
# APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one
|
||||
# switchover. Expect a single brief connection reset for Semaphore.
|
||||
affinity:
|
||||
enablePodAntiAffinity: true
|
||||
podAntiAffinityType: required
|
||||
primaryUpdateMethod: switchover
|
||||
# SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full
|
||||
# rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the
|
||||
# last few transactions when a primary dies before its standbys catch up, and `method: any` +
|
||||
# `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`,
|
||||
# never `required` — `required` blocks writes outright when fewer than `number` healthy standbys
|
||||
# are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded
|
||||
# standby into a total write outage. `preferred` relaxes to async instead.
|
||||
#
|
||||
# NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter —
|
||||
# no restart, no switchover, no client disconnect. Land it after the anti-affinity roll.
|
||||
postgresql:
|
||||
synchronous:
|
||||
method: any
|
||||
number: 1
|
||||
dataDurability: preferred
|
||||
storage:
|
||||
storageClass: longhorn
|
||||
size: 5Gi
|
||||
@@ -106,38 +69,29 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
|
||||
# backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/semaphore-db/ — dev
|
||||
# s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod
|
||||
# s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and
|
||||
# destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
|
||||
# Credentials: PER-ENVIRONMENT (2026-09-07). dev reads semaphore-db-backup-credentials (the
|
||||
# scoped MinIO key), prod reads semaphore-db-backup-credentials-prod (the Cloudflare R2
|
||||
# token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
|
||||
# generated golden — render.py places the -prod sidecar in this DB's wave); the
|
||||
# cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
|
||||
# one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
|
||||
# ONE shared Secret until the split, so giving prod its R2 token would have left dev
|
||||
# signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
|
||||
# disk-full clock below rather than a merely missing backup.
|
||||
#
|
||||
# archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
|
||||
# Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
|
||||
# the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
|
||||
# endpoint. Treat ContinuousArchiving=False here as an availability incident.
|
||||
# backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
|
||||
# (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP.
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials
|
||||
# (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
|
||||
# substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
|
||||
backup:
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
# PER-ENVIRONMENT PATH (cluster-vars). CNPG appends serverName, which defaults to the CLUSTER
|
||||
# NAME — and both environments have a `semaphore-db`. Sharing one MinIO with a shared path
|
||||
# would interleave two Postgres servers' WAL into ONE Barman stream, corrupting the chain and
|
||||
# destroying PITR for both. Dev keeps `s3://cnpg` (its existing, PROVEN backups); prod nests.
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: semaphore-db-backup-credentials
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: semaphore-db-backup-credentials
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
---
|
||||
# Namespace (trust zone; default-deny + policies apply)
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: vm-demo
|
||||
labels:
|
||||
homelab.keithjackson.dev/app-namespace: 'true'
|
||||
annotations:
|
||||
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
||||
---
|
||||
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
homelab.keithjackson.dev/baseline: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# CDI DataVolume (standalone; survives rebuild, R-STATE-01)
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-demo-rootdisk
|
||||
namespace: vm-demo
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
url: docker://quay.io/containerdisks/fedora:41
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
---
|
||||
# KubeVirt VirtualMachine (R-PLAT-03)
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: vm-demo
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
running: false
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: vm-demo
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: none
|
||||
spec:
|
||||
domain:
|
||||
cpu:
|
||||
cores: 1
|
||||
memory:
|
||||
guest: 1536Mi
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk:
|
||||
bus: virtio
|
||||
- name: cloudinit
|
||||
disk:
|
||||
bus: virtio
|
||||
interfaces:
|
||||
- name: default
|
||||
masquerade: {}
|
||||
resources:
|
||||
requests:
|
||||
memory: 1536Mi
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
dataVolume:
|
||||
name: vm-demo-rootdisk
|
||||
- name: cloudinit
|
||||
cloudInitNoCloud:
|
||||
userData: "#cloud-config\npackage_update: true\npackages:\n - dnf-automatic\nwrite_files:\n - path: /etc/dnf/automatic.conf\n content: |\n [commands]\n upgrade_type = default\n random_sleep = 0\n download_updates = yes\n apply_updates = yes\nruncmd:\n - [ systemctl, enable, --now, dnf-automatic.timer ]\n"
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): vm-demo-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: vm-demo-allow-internet-egress
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
egress:
|
||||
- toEntities:
|
||||
- world
|
||||
---
|
||||
# ServiceMonitor (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: vm-demo
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- vm-demo
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
---
|
||||
# PrometheusRule SLI alerts (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: vm-demo-slis
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
groups:
|
||||
- name: vm-demo.slis
|
||||
rules:
|
||||
- alert: vm-demo-down
|
||||
expr: up{job="vm-demo"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: vm-demo
|
||||
annotations:
|
||||
summary: vm-demo is down
|
||||
description: 'vm-demo has been unreachable for 5m (SLI: up).'
|
||||
@@ -1,228 +0,0 @@
|
||||
---
|
||||
# Namespace (trust zone; default-deny + policies apply)
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: keycloak
|
||||
labels:
|
||||
homelab.keithjackson.dev/app-namespace: 'true'
|
||||
gateway-access: shared
|
||||
annotations:
|
||||
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
||||
---
|
||||
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: keycloak
|
||||
labels:
|
||||
homelab.keithjackson.dev/baseline: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PVC keycloak-db (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: keycloak-db
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: native
|
||||
spec:
|
||||
containers:
|
||||
- name: keycloak
|
||||
image: quay.io/keycloak/keycloak
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /data/db
|
||||
volumes:
|
||||
- name: db
|
||||
persistentVolumeClaim:
|
||||
claimName: keycloak-db
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: keycloak
|
||||
ports:
|
||||
- name: port-8080
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
---
|
||||
# CiliumNetworkPolicy egress allow (R-NET-02): keycloak-allow-keycloak-db-5432
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-keycloak-db-5432
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
app.kubernetes.io/name: keycloak-db
|
||||
k8s:io.kubernetes.pod.namespace: keycloak-db
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '5432'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): keycloak-allow-gateway-ingress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-gateway-ingress
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
ingress:
|
||||
- fromEntities:
|
||||
- ingress
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '8080'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): keycloak-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-internet-egress
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
egress:
|
||||
- toEntities:
|
||||
- world
|
||||
---
|
||||
# HTTPRoute keycloak-8080 (R-TLS/R-DNS)
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: keycloak-8080
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- keycloak.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- name: keycloak
|
||||
port: 8080
|
||||
---
|
||||
# ServiceMonitor (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- keycloak
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
---
|
||||
# PrometheusRule SLI alerts (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: keycloak-slis
|
||||
namespace: keycloak
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
groups:
|
||||
- name: keycloak.slis
|
||||
rules:
|
||||
- alert: keycloak-down
|
||||
expr: up{job="keycloak"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: keycloak
|
||||
annotations:
|
||||
summary: keycloak is down
|
||||
description: 'keycloak has been unreachable for 5m (SLI: up).'
|
||||
@@ -35,34 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume nextcloud-legacy-archive - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: nextcloud-legacy-archive
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud
|
||||
spec:
|
||||
capacity:
|
||||
storage: 270Gi
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/nextcloud
|
||||
readOnly: true
|
||||
claimRef:
|
||||
namespace: nextcloud
|
||||
name: nextcloud-legacy-archive
|
||||
---
|
||||
# PVC nextcloud-data (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -79,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
---
|
||||
# PVC nextcloud-legacy-archive (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: nextcloud-legacy-archive
|
||||
namespace: nextcloud
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
storageClassName: ''
|
||||
volumeName: nextcloud-legacy-archive
|
||||
resources:
|
||||
requests:
|
||||
storage: 270Gi
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -164,10 +119,6 @@ spec:
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/www/html
|
||||
- name: legacy-archive
|
||||
mountPath: /mnt/legacy-nextcloud
|
||||
subPath: nextcloud
|
||||
readOnly: true
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
@@ -181,9 +132,6 @@ spec:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: nextcloud-data
|
||||
- name: legacy-archive
|
||||
persistentVolumeClaim:
|
||||
claimName: nextcloud-legacy-archive
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
+3
-62
@@ -35,33 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume radarr-media - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: radarr-media
|
||||
labels:
|
||||
app.kubernetes.io/name: radarr
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
claimRef:
|
||||
namespace: radarr
|
||||
name: radarr-media
|
||||
---
|
||||
# PVC radarr-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -78,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
# PVC radarr-media (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: radarr-media
|
||||
namespace: radarr
|
||||
labels:
|
||||
app.kubernetes.io/name: radarr
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ''
|
||||
volumeName: radarr-media
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -118,25 +74,16 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
containers:
|
||||
- name: radarr
|
||||
image: lscr.io/linuxserver/radarr:5.21.1
|
||||
ports:
|
||||
- containerPort: 7878
|
||||
env:
|
||||
- name: PUID
|
||||
value: '1000'
|
||||
- name: PGID
|
||||
value: '3000'
|
||||
- name: TZ
|
||||
value: Etc/UTC
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -150,7 +97,6 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -202,15 +148,10 @@ spec:
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
securityContext:
|
||||
fsGroup: 3000
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: radarr-config
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: radarr-media
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
@@ -347,7 +288,7 @@ metadata:
|
||||
app.kubernetes.io/name: radarr
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -5,15 +5,6 @@
|
||||
# credential from its own namespace (a Pod cannot read a Secret in another namespace).
|
||||
# access-key-encryption: FIXED base64 32-byte AES key Semaphore uses to encrypt stored SSH keys /
|
||||
# credentials; MUST stay stable across restarts or stored secrets become undecryptable.
|
||||
# oidc-client-secret: the Keycloak `semaphore` CONFIDENTIAL client secret, and ONLY that. It MIRRORS
|
||||
# `semaphore-client-secret` in platform/keycloak/keycloak-realm-clients.sops.yaml (the realm import
|
||||
# injects that copy into the client), same cross-namespace mirror pattern as db-password above.
|
||||
# It replaced the former `oidc-providers-json` key (2026-09-07): that key held the WHOLE
|
||||
# SEMAPHORE_OIDC_PROVIDERS blob, which buried the environment-specific realm-issuer and redirect
|
||||
# URLs inside ciphertext where Flux's ${BASE_DOMAIN} postBuild substitution could not reach them,
|
||||
# so prod ran with dev's URLs. Those URLs are PUBLIC and now live in plaintext in
|
||||
# catalog/semaphore.yaml (rendered to apps/wave-1/semaphore.yaml); the kubelet splices THIS value
|
||||
# back into the blob via $(SEMAPHORE_OIDC_CLIENT_SECRET). Only the secret half is encrypted.
|
||||
# Recover: infra/devcontainer/dev.sh run sops -d apps/semaphore-bootstrap-secret.sops.yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
@@ -24,11 +15,11 @@ metadata:
|
||||
app.kubernetes.io/name: semaphore
|
||||
type: Opaque
|
||||
stringData:
|
||||
admin-username: ENC[AES256_GCM,data:ju3wVdjNzRUP/w==,iv:d/5ZxgQZ5bympu3gT2ck3DJy9iUPPBAtZYsJvD3To7I=,tag:0UtBaDVqNSK4RCJrQicchw==,type:str]
|
||||
admin-password: ENC[AES256_GCM,data:pxIwpgh3BoND2TI0mPTixOTJoictzT77dqtuRdMKgY/2WaW4wM1Ra6H6n/dxnyR2,iv:eYbFQGFYVJsnYGwn6cOI3BgAdzMcin9YggAhiuL+xqM=,tag:/sPWGv4gue0vjc2nMk7wWA==,type:str]
|
||||
db-password: ENC[AES256_GCM,data:hBFpWXdpk1R5a23Pntup03t56NBRPS+SZU8CrHVb/ypIaIpgf7kjYtVy6nhTF6u0,iv:eBzoQgH5ZhWWpfKZ6AQmwaSal5QaiuN9gfexQ7GsVFs=,tag:j2k8EwhKDlj/Ud6O39ioVQ==,type:str]
|
||||
access-key-encryption: ENC[AES256_GCM,data:SKtAV4HxJ9pNYDqUK/1uOdI1rtMji3pTfprI2rhQxFQyhc6dFlhqI+OUEcY=,iv:rKCP+FLidcOltOwmMMcO5M8tz4qiifIMI+W5soGO2M8=,tag:HaHzkX/JI+9wmcg2SD9GAQ==,type:str]
|
||||
oidc-client-secret: ENC[AES256_GCM,data:uCo7TNzbM2QKw9ie+xzyUG9MFKco1OAlwr5qPn/AGpNvo1YyXSfbZyyM0CLJ1NS821hGg5Zr/m3R/xpy84s3FQ==,iv:O8a4haskufpmorPaxOkJ1Q99Y5GTlzmDFJAsHS+irLk=,tag:VojJpA9jNMbNQmIe1Qr44w==,type:str]
|
||||
admin-username: ENC[AES256_GCM,data:uD/vO9A9MjerFg==,iv:i/NAqVD6rWg9oWWSGBrRkxzwcPp2IW1HLIKY0pdcPPg=,tag:r27tKLAJ+ydE7Ty3ccMNUw==,type:str]
|
||||
admin-password: ENC[AES256_GCM,data:544nWY1UmBnlsdPJAQZJMXRumu4W9X7UXZvZ6vHN/ArjRgLXKTArOe5JkT/nNrNw,iv:AC1DDbZj2HfpU19EjzA7T+WpIaR4nZc3PNN61y8bAzU=,tag:r28LFi5w6ZeCCaRaX2jRdg==,type:str]
|
||||
db-password: ENC[AES256_GCM,data:ySkdvaGRCW9fMQqChs5l8dY4zCG19i4HOEinRC48mgLuh9WOrjd16T6jwFgiwB55,iv:+1gztJ22HbwgQJiolZMMWOcKIsxrBJpnruJdtxdooOU=,tag:8CfJGwyPTjDGyr0CvalXMQ==,type:str]
|
||||
access-key-encryption: ENC[AES256_GCM,data:1YAMLo1nVkpJWZc+KBZ7B4ia+Y8P+xMyOhHMcEGGJOXd3o+WC5FqirLBVh4=,iv:j28ozeaDhGWDjoaxFOhn2t9YbYE7L3ah/Mfi+i2DWwY=,tag:czG9Wxc/7Nsuh1W5iATOww==,type:str]
|
||||
oidc-providers-json: ENC[AES256_GCM,data:90yt5mIVWLVI2KrlJWpxhL/BMDpEgRhv7+BL+mBRt79diRvU7pN9vtzhZprAKzcU1sEywlT4EwFlsvJpYyo5Qbkp+ZVknbyhAlGvHhJOBHukOPWgf4LignLerO2GzhXjOOzFqWfZ4EyUxNMWTbSe/xc0zIVHDDNQiYplGVRK+E/ukiI7sD/Tn2RrbrdvpC5GyK0NXWRsXn9ffhE3fCoiRHXagj9i/q+uUrQI5EKpPobrDUAGqvKoBXTl/wJ4phnWg9IqtCTE/c3yA7fLLz4j5dCtPZNSVPT4TKDx8Mdv8CfCgziVicjG2s+yHuWpsQn8lf6nvdJmpYPvIqTtLWdxFWuD8Vfo6Qbq2mqPbKR+KmDOGUnjl4gbZdL1SkE9JiqBUobz7UXeLXwYzxMIsinOfYZLcCk7alrD5Yndj9JDxDJZQ54sMMBfVbxDl94W1VHy03RxAR46KKAbPaxOP5wH4Q4z3SDceNwmuGCDRGChRXiHhN5X2X0GqYCbc8E6GdkIMcpmD80e26ydEmAjw7F89mCF4F/yMzJNiG82qaZIn8HeqgGA8urous9r9bYJNlitYTYQ,iv:jbfVdrrZ9fGiOxhQeoTJfFg1QwiOMkoQyaadiVfxX/w=,tag:JkepcRs34EuEgNb4buTTDQ==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
@@ -38,14 +29,14 @@ sops:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTzF6NTBtcFVWK1Vtc0I2
|
||||
VGdySHBsRGgvc1h0VzgvQ01xYVBJV3g2Q1NRCjJyZHVYZGtuamo1MFJHWlBwWkxt
|
||||
cHA4VkY1TkRpN3M4UTdMOWZTbFVGemMKLS0tIGVNdDBrWVJveU56VjVBU2JWdHF4
|
||||
RGtaYStXYUpONTZjZCtmM0NyZ3Rka1UKIQpmQai8dz+uRAsXxyeuGaRlRwsFqIPQ
|
||||
9AfrZJcslYwlaK3ptm1oqaCdrlxcMar2nI7thSqzbLe+e1nzA2z7vQ==
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuRDlqRFJ0dk14UHluSGRP
|
||||
S1pESy9PWlpMakU0aTkzNVhqUGdtbWNEdTJjCm5RNG14azJHc28rdEJ4L0RzR0xO
|
||||
MGRocGZ1V1JZTENWQkN3VGZzamp2eTgKLS0tIGt6QkNNaW5hdTBJVHY0SVo5d3JX
|
||||
elp0SWxHcjg0RWF3aW9mWTJHZ0lhcUEKc0rU5VilAVDleCwqq4SOF+Y9sniu1Pka
|
||||
w/MGDA9THSB0b+YgWRWiX/KiimFqp0ZFODOJ9cHVwFGY71+VadPwOg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T07:01:50Z"
|
||||
mac: ENC[AES256_GCM,data:Xutm1rQt57J4RILElD8FkNOsQT1HK5jZlPtqqLyZ/VqOoDwo+IS1wp7hROprlUvQC0bUXy79tRRIl5Q0CCOD+fvAGIdmade+vB7qXlYN6Q1441dGhuSTPvckso6FaB2ddRjKhKGLc/Z4bJoz9VaRNNxJNpWXIrENTg+I+QhNhlo=,iv:klJOrEQ6QN5dTGkQfGnbMbgCWsgWwZPb5YXDMeWeE34=,tag:fuJr4t3Tt3dT9u3/0SFo8Q==,type:str]
|
||||
lastmodified: "2026-08-27T22:12:32Z"
|
||||
mac: ENC[AES256_GCM,data:nHftJTGnTr0O5h8yGXwgz3eflggXJfIXM3m8n3bzDJE/P4nvSpZzFeMb+sc2on+AL+t+SeKrIl8blzm33XsQ1lkaY85Zm+bc5cnED1g04BMDVmmK3brh/8w50Bp2wPEVzb4I01MDzhhyIbOYg7pFvZAH3/VuqvM1t6XmfF1NrOA=,iv:4KBa9PH/dfdGwLC16RenZIVB6BtSoGo6x3qt3h1R43I=,tag:KULiA1klP/RZMNigkloGDA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -118,13 +118,11 @@ spec:
|
||||
key: access-key-encryption
|
||||
- name: SEMAPHORE_PORT
|
||||
value: :3000
|
||||
- name: SEMAPHORE_OIDC_CLIENT_SECRET
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: semaphore-bootstrap
|
||||
key: oidc-client-secret
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
|
||||
key: oidc-providers-json
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/semaphore
|
||||
|
||||
+3
-62
@@ -35,33 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume sonarr-media - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: sonarr-media
|
||||
labels:
|
||||
app.kubernetes.io/name: sonarr
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
claimRef:
|
||||
namespace: sonarr
|
||||
name: sonarr-media
|
||||
---
|
||||
# PVC sonarr-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -78,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
# PVC sonarr-media (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: sonarr-media
|
||||
namespace: sonarr
|
||||
labels:
|
||||
app.kubernetes.io/name: sonarr
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ''
|
||||
volumeName: sonarr-media
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -118,25 +74,16 @@ spec:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
containers:
|
||||
- name: sonarr
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15
|
||||
ports:
|
||||
- containerPort: 8989
|
||||
env:
|
||||
- name: PUID
|
||||
value: '1000'
|
||||
- name: PGID
|
||||
value: '3000'
|
||||
- name: TZ
|
||||
value: Etc/UTC
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -150,7 +97,6 @@ spec:
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
@@ -202,15 +148,10 @@ spec:
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
securityContext:
|
||||
fsGroup: 3000
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: sonarr-config
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: sonarr-media
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
@@ -347,7 +288,7 @@ metadata:
|
||||
app.kubernetes.io/name: sonarr
|
||||
annotations:
|
||||
homelab.keithjackson.dev/auth-mode: forward-auth
|
||||
homelab.keithjackson.dev/auth-groups: media-users,platform-admins
|
||||
homelab.keithjackson.dev/auth-groups: media-users
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -11,7 +11,8 @@ Spec: [ADR-0004 (Flux)](../../../home-network-k3s/decisions/adr-0004-gitops-flux
|
||||
## Files
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `bootstrap.sh` | **The single bootstrap entrypoint** the Infra worker's final Ansible play calls. render+apply the Flux controllers (`components/`) → verify `sops-age` Secret → `kubectl apply -k clusters/dev`. |
|
||||
| `bootstrap.sh` | **The single bootstrap entrypoint** the Infra worker's final Ansible play calls. render+apply the Flux controllers (`components/`) → **pre-apply chart CRDs (`preapply-crds.sh`)** → verify `sops-age` Secret → `kubectl apply -k ${FLUX_ENV_PATH}`. |
|
||||
| `preapply-crds.sh` | **Server-side pre-apply of the Kyverno CRD bundle** — the CRDs `platform/kyverno/admission/helmrelease.yaml` deliberately tells Helm *not* to install (`crds: {install: false}`, because the ~5.6 MiB bundle exceeds kubectl's 256 KiB last-applied annotation limit and stalls the release). Nothing else installs them, so without this step `platform` never goes Ready on a cold cluster. Chart name/version/repo are **derived from that HelmRelease** — no second pin. Idempotent; runnable standalone. |
|
||||
| `components/kustomization.yaml` | **Customization of the Flux CONTROLLERS themselves.** Overlays `flux install --export` output; currently enables `StrictPostBuildSubstitutions` on kustomize-controller. See "Customizing the controllers" below. |
|
||||
| `patches/platform-sops-decryption.yaml` | Strategic-merge patch enabling `decryption.provider: sops` + `secretRef: sops-age` on the `platform` Kustomization (realises the base TODO). |
|
||||
| `patches/apps-sops-decryption.yaml` | Same, for the app-wave Kustomizations. There is **no single `apps` Kustomization** any more — the apps tier is now `apps-wave-0/1/2` + `apps-vms`, and SOPS decryption is layered on **per wave** (each wave Kustomization is patched by name in `clusters/dev`). |
|
||||
@@ -74,6 +75,49 @@ flux get kustomizations -A # all 11 must return
|
||||
**Rollback** is the same command with `FLUX_INSTALL_PLAIN=1 bash bootstrap/flux/bootstrap.sh`, or
|
||||
simply `flux install --namespace=flux-system --components-extra=…` (drops the arg back off).
|
||||
|
||||
## The CRD pre-apply step (`preapply-crds.sh`) — why bootstrap has to do this
|
||||
|
||||
`platform/kyverno/admission/helmrelease.yaml` sets `crds: {install: false}`. That is correct and must
|
||||
stay: the Kyverno CRD bundle is **5,636,155 bytes across 22 CRDs** at chart 3.9.0 (measured with
|
||||
`helm template … --include-crds`, 2026-09-04), so Helm's client-side apply hits kubectl's 256 KiB
|
||||
`last-applied-configuration` annotation limit — `metadata.annotations: Too long` — which stalls the
|
||||
HelmRelease and blocks `platform` plus every tier that `dependsOn` it.
|
||||
|
||||
The part that was never automated: **with CRD install off, nothing installs the CRDs.** Kyverno's
|
||||
Deployments start, its API types do not exist, `platform` never reaches Ready. Dev has them only
|
||||
because somebody once ran the command in a shell. `preapply-crds.sh` is that command, in git:
|
||||
|
||||
```
|
||||
helm template <chart> --repo <url> --version <constraint> --include-crds --set crds.install=true
|
||||
| yq 'select(.kind == "CustomResourceDefinition")'
|
||||
| kubectl apply --server-side --force-conflicts
|
||||
```
|
||||
|
||||
- **Version derivation, not a second pin.** `chart`, `version`, `namespace` and the repo `url` are
|
||||
read out of the HelmRelease/HelmRepository pair with `yq`. `spec.chart.spec.version` is a semver
|
||||
**constraint** (`3.9.x`), and `helm template --version` accepts the same constraint syntax
|
||||
helm-controller resolves — so the rendered CRDs are the ones Flux will install, and a Renovate bump
|
||||
of that one line carries this step with it. There is nothing here that can drift.
|
||||
- **CRDs only.** Piping the *whole* `helm template` output into kubectl (the by-hand command in the
|
||||
HelmRelease comment) would create Deployments/webhooks outside Helm's ownership and then fight the
|
||||
HelmRelease over field managers. The `yq` filter is load-bearing.
|
||||
- **Idempotent.** Server-side apply is an upsert; `--force-conflicts` adopts the fields dev's
|
||||
hand-applied CRDs left owned by `kubectl-client-side-apply`. Re-running prints
|
||||
`serverside-applied` and exits 0.
|
||||
- **Cannot degrade to a no-op.** Three guards: every derived coordinate must be non-empty and not
|
||||
`null`; the rendered CRD count must be `> 0` (an empty stream would otherwise `apply` cleanly and
|
||||
exit 0 — the exact silent-success bug this repo keeps hitting); and after the apply every CRD is
|
||||
waited on for `condition=Established`. There is **no skip flag** on purpose.
|
||||
- **Environment-agnostic (R-ENV-02).** It names no environment and takes no overlay input — it reads
|
||||
repo files and acts on `$KUBECONFIG`. Dev and prod run the identical step.
|
||||
|
||||
Standalone use (an already-bootstrapped cluster whose `platform` is blocked on the missing CRDs):
|
||||
|
||||
```bash
|
||||
# inside infra/devcontainer, KUBECONFIG pointed at the target cluster
|
||||
bash bootstrap/flux/preapply-crds.sh
|
||||
```
|
||||
|
||||
## Why `flux install` + apply, not `flux bootstrap git` (Phase 1)
|
||||
|
||||
The reconciliation graph is **already authored** in `clusters/base` (read-only to this worker):
|
||||
@@ -98,7 +142,7 @@ tofu apply → cloud-init → Ansible (k3s + Cilium) [Infra worker, T-1
|
||||
▼ (final Ansible play calls the entrypoints below)
|
||||
1. bootstrap/sops/age-keygen.sh # operator generates age key OUTSIDE repo
|
||||
2. bootstrap/sops/age-key-secret.md # operator installs private key as sops-age Secret
|
||||
3. bootstrap/flux/bootstrap.sh # flux install → verify sops-age → apply -k clusters/dev
|
||||
3. bootstrap/flux/bootstrap.sh # flux install → preapply-crds.sh → verify sops-age → apply -k ${FLUX_ENV_PATH}
|
||||
│
|
||||
▼
|
||||
Flux reconciles: GitRepository(flux-system) → Kustomization(platform) → Kustomization(platform-config)
|
||||
@@ -118,8 +162,10 @@ live in `infra/devcontainer/versions.env`).
|
||||
template). `bootstrap.sh` **fails fast** if this Secret is absent.
|
||||
3. `bootstrap/flux/bootstrap.sh` (this entrypoint).
|
||||
|
||||
`bootstrap.sh` is idempotent-friendly (`flux install` and `kubectl apply -k` re-run safely) and
|
||||
never handles key material — it only *verifies* the Secret exists.
|
||||
`bootstrap.sh` is idempotent-friendly (`flux install`, `preapply-crds.sh`'s server-side apply, and
|
||||
`kubectl apply -k` all re-run safely) and never handles key material — it only *verifies* the Secret
|
||||
exists. Beyond `flux`/`kubectl` it now also needs **`helm` + `yq`** on PATH (step 2's CRD render;
|
||||
both are in `infra/devcontainer`) and outbound network to the Kyverno chart repo.
|
||||
|
||||
## Handoff — clusters owner / PM (files this worker may not edit)
|
||||
|
||||
|
||||
@@ -33,10 +33,19 @@
|
||||
# the single apps.yaml was retired). Letting
|
||||
# bootstrap regenerate them would DUPLICATE/conflict. `flux install` +
|
||||
# applying the authored clusters/dev overlay is the reconciling path.
|
||||
# 2. sops-age Secret — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so
|
||||
# 2. CRD pre-apply — ./preapply-crds.sh applies the Kyverno CRD bundle SERVER-SIDE, because
|
||||
# platform/kyverno/admission/helmrelease.yaml sets `crds: {install: false}`
|
||||
# (the ~5.6 MiB bundle blows kubectl's 256 KiB last-applied annotation limit,
|
||||
# which STALLS the HelmRelease and blocks the `platform` Kustomization and
|
||||
# every tier below it). With CRD install off, NOTHING else installs them, so
|
||||
# a cold cluster can never reconcile `platform` without this step. The chart
|
||||
# version is DERIVED from that same HelmRelease — no second pin. Dev only
|
||||
# ever had these CRDs because they were applied BY HAND; this is that step,
|
||||
# in git, idempotent and version-derived. See ./preapply-crds.sh.
|
||||
# 3. sops-age Secret — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so
|
||||
# decryption works on first pass (../sops/age-key-secret.md). This script
|
||||
# only VERIFIES it exists; it never handles key material.
|
||||
# 3. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph:
|
||||
# 4. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph:
|
||||
# GitRepository `flux-system` (source) -> Kustomization `platform` ->
|
||||
# `platform-config` -> the app-wave Kustomizations `apps-wave-0/1/2`
|
||||
# (chained dependsOn) + `apps-vms`, with cluster-vars overlay
|
||||
@@ -71,7 +80,10 @@ repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd
|
||||
echo "==> Flux + SOPS bootstrap — path=${FLUX_ENV_PATH}"
|
||||
|
||||
# --- 0. tooling present (from infra/devcontainer; versions pinned there) ------
|
||||
for bin in flux kubectl; do
|
||||
# helm + yq are needed by step 2 (./preapply-crds.sh renders the CRD bundle from the authored
|
||||
# HelmRelease). Checked HERE as well as in that script so a missing binary stops the run BEFORE the
|
||||
# Flux controllers are installed, instead of half-way through.
|
||||
for bin in flux kubectl helm yq; do
|
||||
command -v "$bin" >/dev/null 2>&1 || { echo "ERROR: '$bin' not found (use infra/devcontainer)." >&2; exit 1; }
|
||||
done
|
||||
|
||||
@@ -99,7 +111,7 @@ done
|
||||
#
|
||||
# ESCAPE HATCH: FLUX_INSTALL_PLAIN=1 falls back to a bare `flux install` (no customization, so NO
|
||||
# strict substitution). Use only to unblock a bring-up if the render path misbehaves, and say so.
|
||||
echo "==> [1/3] installing Flux controllers${FLUX_COMPONENTS_EXTRA:+ (+ extra: ${FLUX_COMPONENTS_EXTRA})}"
|
||||
echo "==> [1/4] installing Flux controllers${FLUX_COMPONENTS_EXTRA:+ (+ extra: ${FLUX_COMPONENTS_EXTRA})}"
|
||||
flux_install_args=(--namespace="${FLUX_NAMESPACE}")
|
||||
[ -n "${FLUX_COMPONENTS_EXTRA}" ] && flux_install_args+=(--components-extra="${FLUX_COMPONENTS_EXTRA}")
|
||||
|
||||
@@ -130,18 +142,29 @@ else
|
||||
kubectl apply -f "${render_dir}/rendered.yaml"
|
||||
fi
|
||||
|
||||
# --- 2. verify the out-of-band age key Secret exists -------------------------
|
||||
# --- 2. pre-apply the CRDs Helm is not allowed to install ---------------------
|
||||
# platform/kyverno/admission/helmrelease.yaml sets `crds: {install: false}`, so the 22 Kyverno CRDs
|
||||
# must exist BEFORE the platform Kustomization reconciles that HelmRelease — otherwise Kyverno's
|
||||
# controllers come up with no API types, `platform` never goes Ready, and all of its dependents stall.
|
||||
# ./preapply-crds.sh derives the chart version from the HelmRelease itself (no second pin), applies
|
||||
# only the CRD documents SERVER-SIDE (no 256 KiB annotation limit), and is idempotent. It FAILS HARD
|
||||
# rather than skipping — there is deliberately NO opt-out flag: a skipped CRD apply produces a
|
||||
# cluster that looks bootstrapped and reconciles nothing.
|
||||
echo "==> [2/4] pre-applying chart CRDs (server-side)"
|
||||
bash "${script_dir}/preapply-crds.sh"
|
||||
|
||||
# --- 3. verify the out-of-band age key Secret exists -------------------------
|
||||
# We DO NOT create it here (it holds the private key). The operator installs it per
|
||||
# ../sops/age-key-secret.md. Fail fast if missing so decryption can't silently no-op.
|
||||
echo "==> [2/3] verifying ${SOPS_SECRET_NAME} Secret exists in ${FLUX_NAMESPACE}"
|
||||
echo "==> [3/4] verifying ${SOPS_SECRET_NAME} Secret exists in ${FLUX_NAMESPACE}"
|
||||
if ! kubectl -n "${FLUX_NAMESPACE}" get secret "${SOPS_SECRET_NAME}" >/dev/null 2>&1; then
|
||||
echo "ERROR: Secret ${FLUX_NAMESPACE}/${SOPS_SECRET_NAME} not found." >&2
|
||||
echo " Install the age private key first: bootstrap/sops/age-key-secret.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- 3. apply the authored environment entrypoint ----------------------------
|
||||
echo "==> [3/3] kubectl apply -k ${FLUX_ENV_PATH}"
|
||||
# --- 4. apply the authored environment entrypoint ----------------------------
|
||||
echo "==> [4/4] kubectl apply -k ${FLUX_ENV_PATH}"
|
||||
kubectl apply -k "${repo_root}/${FLUX_ENV_PATH}"
|
||||
|
||||
echo
|
||||
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# preapply-crds.sh — server-side pre-apply of the CRDs that Helm is NOT allowed to install.
|
||||
# ENVIRONMENT-AGNOSTIC: it reads the chart coordinates out of the authored HelmRelease and talks to
|
||||
# whatever cluster $KUBECONFIG points at, so dev and prod run the SAME step with no fork (R-ENV-02).
|
||||
# Called by ./bootstrap.sh (step 2/4); also safe to run STANDALONE against an already-bootstrapped
|
||||
# cluster (that is how prod gets unblocked without re-running the whole bootstrap).
|
||||
#
|
||||
# R-GITOPS-01 (the graph must reconcile from git, not from someone's shell history) · R-ENV-02.
|
||||
# =============================================================================
|
||||
# *** AUTHORED, NOT RUN. Agents must not execute it — it MUTATES a cluster. ***
|
||||
# =============================================================================
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
#
|
||||
# platform/kyverno/admission/helmrelease.yaml sets `crds: {install: false}`. That is deliberate:
|
||||
# the Kyverno CRD bundle is ~5.6 MiB, far past kubectl's 256 KiB last-applied-configuration
|
||||
# annotation limit, so Helm's client-side apply fails with "metadata.annotations: Too long" and
|
||||
# STALLS the HelmRelease — which blocks the `platform` Kustomization and every tier that dependsOn
|
||||
# it (the kps-cascade failure mode; see the HelmRelease's own comment).
|
||||
#
|
||||
# The consequence nobody automated: with CRD install off, NOTHING installs those CRDs. Kyverno's
|
||||
# Deployments come up but its API types do not exist, so `platform` never goes Ready on a cold
|
||||
# cluster. The dev cluster only has them because they were once applied BY HAND — exactly the
|
||||
# undocumented manual step this repo exists to eliminate. This script is that step, in git,
|
||||
# version-derived and idempotent.
|
||||
#
|
||||
# HOW THE VERSION IS DERIVED (there is deliberately NO second copy of it)
|
||||
#
|
||||
# Everything comes out of the HelmRelease/HelmRepository pair that Flux itself reconciles:
|
||||
#
|
||||
# chart name spec.chart.spec.chart (HelmRelease)
|
||||
# chart version spec.chart.spec.version (HelmRelease) e.g. "3.9.x"
|
||||
# chart repo url spec.url (HelmRepository)
|
||||
# namespace metadata.namespace (HelmRelease)
|
||||
#
|
||||
# The version is a semver CONSTRAINT, not a fixed tag. `helm template --version` accepts the same
|
||||
# constraint syntax Flux's helm-controller resolves (both use Masterminds/semver), so passing it
|
||||
# through UNCHANGED renders the very chart version Flux will install. Renovate bumps one line in
|
||||
# one file and this step follows automatically — it cannot drift, because there is nothing here to
|
||||
# drift from.
|
||||
#
|
||||
# WHY ONLY THE CRD DOCUMENTS ARE APPLIED
|
||||
#
|
||||
# The by-hand command recorded in the HelmRelease comment piped the WHOLE `helm template` output
|
||||
# into kubectl. Do not do that: it would create Deployments/Services/webhooks outside Helm's
|
||||
# ownership, which then fight the HelmRelease over field managers and leave orphans behind. We
|
||||
# filter to `kind: CustomResourceDefinition` and apply nothing else. Values are the chart DEFAULTS
|
||||
# (plus an explicit crds.install=true) — NOT the repo's values, which switch CRD rendering off.
|
||||
#
|
||||
# IDEMPOTENCY
|
||||
#
|
||||
# `kubectl apply --server-side --force-conflicts` is a declarative upsert: the first run creates,
|
||||
# every later run reports "serverside-applied" and exits 0, and field ownership is taken from
|
||||
# whatever previously wrote those objects (that is what --force-conflicts is for — it resolves the
|
||||
# hand-applied dev CRDs' `kubectl-client-side-apply` manager without a human deciding anything).
|
||||
# Server-side apply does not write the last-applied annotation, which is the whole reason the
|
||||
# 5.6 MiB bundle can be applied at all.
|
||||
#
|
||||
# LOUD FAILURE (this script must never degrade to a successful no-op)
|
||||
#
|
||||
# `set -euo pipefail`, plus three explicit guards, because a silent skip here reproduces the exact
|
||||
# class of bug this repo has been bitten by more than once:
|
||||
# 1. every derived coordinate must be non-empty and not "null" — a moved YAML path fails here;
|
||||
# 2. the rendered CRD count must be > 0 — if a future chart moves/renames its CRD templates the
|
||||
# filter would match nothing and `kubectl apply` on an empty stream EXITS 0. Guarded;
|
||||
# 3. after applying, every CRD is waited on for condition=Established — so "applied" is proven
|
||||
# to mean "the API type exists", not merely "kubectl returned 0".
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# --- config (env-overridable) ------------------------------------------------
|
||||
# The single source of truth for the chart coordinates. Repo-relative on purpose: this is the file
|
||||
# Flux reconciles, so it is the file this step must agree with.
|
||||
CRD_HELMRELEASE="${CRD_HELMRELEASE:-platform/kyverno/admission/helmrelease.yaml}"
|
||||
# How long to wait for the applied CRDs to become Established (apiextensions publishes the type).
|
||||
CRD_ESTABLISH_TIMEOUT="${CRD_ESTABLISH_TIMEOUT:-120s}"
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/../.." && pwd))"
|
||||
|
||||
echo "==> CRD pre-apply — source of truth: ${CRD_HELMRELEASE}"
|
||||
|
||||
# --- 0. tooling present (from infra/devcontainer; versions pinned there) ------
|
||||
# helm renders the bundle, yq selects the CRD documents, kubectl applies them server-side. All three
|
||||
# are baked into infra/devcontainer (HELM_VERSION / YQ_VERSION / KUBECTL_VERSION in versions.env).
|
||||
for bin in helm yq kubectl; do
|
||||
command -v "$bin" >/dev/null 2>&1 || { echo "ERROR: '$bin' not found (use infra/devcontainer)." >&2; exit 1; }
|
||||
done
|
||||
|
||||
helmrelease="${repo_root}/${CRD_HELMRELEASE}"
|
||||
[ -f "${helmrelease}" ] || {
|
||||
echo "ERROR: ${helmrelease} not found — CRD_HELMRELEASE points nowhere." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- 1. derive the chart coordinates from the authored manifests --------------
|
||||
chart_name="$(yq eval-all --no-doc 'select(.kind == "HelmRelease") | .spec.chart.spec.chart' "${helmrelease}")"
|
||||
chart_version="$(yq eval-all --no-doc 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "${helmrelease}")"
|
||||
chart_ns="$(yq eval-all --no-doc 'select(.kind == "HelmRelease") | .metadata.namespace' "${helmrelease}")"
|
||||
chart_repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' "${helmrelease}")"
|
||||
|
||||
# Guard 1: a moved/renamed YAML path must STOP the bootstrap, not render a bogus chart. yq prints
|
||||
# "null" for a missing path and an empty string for a missing document, so both are rejected.
|
||||
for pair in "chart:${chart_name}" "version:${chart_version}" "namespace:${chart_ns}" "repo:${chart_repo}"; do
|
||||
key="${pair%%:*}"; val="${pair#*:}"
|
||||
case "${val}" in
|
||||
""|null|*$'\n'*)
|
||||
echo "ERROR: could not derive a single '${key}' from ${CRD_HELMRELEASE} (got: '${val}')." >&2
|
||||
echo " The HelmRelease/HelmRepository shape changed — fix the yq paths in $(basename "${BASH_SOURCE[0]}")." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo " chart=${chart_name} version=${chart_version} repo=${chart_repo} namespace=${chart_ns}"
|
||||
echo " (version is the constraint Flux resolves — NOT a second copy pinned here)"
|
||||
|
||||
# --- 2. render the chart and keep ONLY the CRD documents ----------------------
|
||||
render_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "${render_dir}"' EXIT
|
||||
crd_file="${render_dir}/crds.yaml"
|
||||
|
||||
# --include-crds covers charts that ship a native crds/ dir; crds.install=true covers Kyverno, whose
|
||||
# CRDs are subchart TEMPLATES that the repo's own values switch off. Chart defaults otherwise.
|
||||
echo "==> rendering ${chart_name} ${chart_version} (CRD documents only)"
|
||||
helm template "${chart_name}" "${chart_name}" \
|
||||
--repo "${chart_repo}" \
|
||||
--version "${chart_version}" \
|
||||
--namespace "${chart_ns}" \
|
||||
--include-crds \
|
||||
--set crds.install=true \
|
||||
| yq eval-all 'select(.kind == "CustomResourceDefinition")' - > "${crd_file}"
|
||||
|
||||
# Blank/"null" lines are dropped on purpose: an EMPTY render makes `yq .metadata.name` emit one
|
||||
# empty line, which would otherwise count as a CRD and sail straight past guard 2 below.
|
||||
mapfile -t crd_names < <(yq eval-all --no-doc '.metadata.name' "${crd_file}" | grep -vx -e '' -e 'null' || true)
|
||||
crd_count="${#crd_names[@]}"
|
||||
|
||||
# Guard 2: an empty stream applies cleanly and exits 0 — the silent-no-op trap. Refuse it.
|
||||
[ "${crd_count}" -gt 0 ] || {
|
||||
echo "ERROR: rendered 0 CustomResourceDefinitions from ${chart_name} ${chart_version}." >&2
|
||||
echo " The chart's CRD layout or the crds.install value key changed. Refusing to continue:" >&2
|
||||
echo " applying nothing would look like success and leave 'platform' blocked forever." >&2
|
||||
exit 1
|
||||
}
|
||||
echo " ${crd_count} CRDs, $(wc -c < "${crd_file}") bytes (>256 KiB — this is why it must be server-side)"
|
||||
|
||||
# --- 3. server-side apply (idempotent upsert) ---------------------------------
|
||||
# --server-side: no last-applied annotation, so the size limit does not apply.
|
||||
# --force-conflicts: adopt fields owned by an earlier manager (e.g. dev's hand-applied CRDs, or
|
||||
# Helm's own manager if a past chart version did install them). Makes a re-run a plain no-op.
|
||||
echo "==> kubectl apply --server-side --force-conflicts (${crd_count} CRDs)"
|
||||
kubectl apply --server-side --force-conflicts -f "${crd_file}"
|
||||
|
||||
# --- 4. prove the API types actually exist -----------------------------------
|
||||
# Without this, "kubectl returned 0" is the only evidence. Establishment is what Kyverno's
|
||||
# controllers and the platform Kustomization actually need.
|
||||
echo "==> waiting for condition=Established (timeout ${CRD_ESTABLISH_TIMEOUT})"
|
||||
crd_refs=()
|
||||
for n in "${crd_names[@]}"; do crd_refs+=("crd/${n}"); done
|
||||
kubectl wait --for=condition=Established --timeout="${CRD_ESTABLISH_TIMEOUT}" \
|
||||
"${crd_refs[@]}" >/dev/null || {
|
||||
echo "ERROR: not all ${crd_count} CRDs reached Established within ${CRD_ESTABLISH_TIMEOUT}." >&2
|
||||
echo " Inspect: kubectl get crd | grep ${chart_name}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "==> ${crd_count} ${chart_name} CRDs applied and Established."
|
||||
Executable
+988
@@ -0,0 +1,988 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# standup.sh — bring a bootstrapped cluster's reconciliation graph all the way up.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): it reads everything it needs from the cluster it is pointed at and
|
||||
# from the authored manifests. There is no dev/prod fork. What makes it safe to run against prod is
|
||||
# the MANDATORY --expect-server guard, not a name in the filename.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# `bootstrap/flux/bootstrap.sh` installs Flux and applies the graph. It does NOT finish the job when
|
||||
# tiers have been deliberately suspended (a DNS/TLS migration window, an incident recovery) or when
|
||||
# a HelmRelease has wedged. Those last steps were living in shell history and in a human's head,
|
||||
# which is the same failure the Kyverno CRD pre-apply had. This is that sequence, in git, ordered,
|
||||
# idempotent, and gated.
|
||||
#
|
||||
# THE ORDERING IS LOAD-BEARING. Read infra/PROD-STANDUP.md before deviating from it.
|
||||
#
|
||||
# USAGE
|
||||
# bootstrap/standup.sh --kubeconfig <path> --expect-server <url> [--confirm-destructive] <step...>
|
||||
#
|
||||
# steps: preflight crds unwedge externaldns s3check resume verify (or: all)
|
||||
#
|
||||
# --expect-server is REQUIRED and is compared against the live API server before ANY command runs.
|
||||
# Pointing this at the wrong cluster is the one mistake with unbounded blast radius: dev is live and
|
||||
# shares a Cloudflare zone with prod. The guard is not optional and there is no override flag.
|
||||
#
|
||||
# --confirm-destructive is required by `unwedge` only. Everything else is additive or reversible.
|
||||
#
|
||||
# --s3-endpoint <url> makes `s3check` probe a CANDIDATE backup endpoint instead of the live
|
||||
# cluster-vars value, so a proposed target can be PROVEN before it is authored into cluster-vars.
|
||||
# `s3check` writes and then deletes ONE throwaway object under `<prefix>/.s3check/`; it changes
|
||||
# nothing else and applies nothing. See infra/BACKUP-R2.md.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------- args
|
||||
KUBECONFIG_ARG="" EXPECT_SERVER="" CONFIRM_DESTRUCTIVE=0 S3_ENDPOINT_OVERRIDE=""
|
||||
STEPS=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--kubeconfig) KUBECONFIG_ARG="${2:?--kubeconfig needs a value}"; shift 2 ;;
|
||||
--expect-server) EXPECT_SERVER="${2:?--expect-server needs a value}"; shift 2 ;;
|
||||
--confirm-destructive) CONFIRM_DESTRUCTIVE=1; shift ;;
|
||||
--s3-endpoint) S3_ENDPOINT_OVERRIDE="${2:?--s3-endpoint needs a value}"; shift 2 ;;
|
||||
-h|--help) sed -n '2,30p' "$0"; exit 0 ;;
|
||||
-*) echo "ERROR: unknown flag '$1'" >&2; exit 2 ;;
|
||||
*) STEPS+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "${KUBECONFIG_ARG}" ] || { echo "ERROR: --kubeconfig is required." >&2; exit 2; }
|
||||
[ -n "${EXPECT_SERVER}" ] || { echo "ERROR: --expect-server is required (safety guard)." >&2; exit 2; }
|
||||
[ "${#STEPS[@]}" -gt 0 ] || { echo "ERROR: no step given. One of: preflight crds unwedge externaldns s3check resume verify all" >&2; exit 2; }
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/.." && pwd))"
|
||||
cd "${repo_root}"
|
||||
|
||||
export KUBECONFIG="${KUBECONFIG_ARG}"
|
||||
[ -f "${KUBECONFIG}" ] || { echo "ERROR: kubeconfig not found: ${KUBECONFIG}" >&2; exit 1; }
|
||||
|
||||
# Where DNS snapshots go. This MUST be a path that survives between invocations: each `dev.sh run` is
|
||||
# a FRESH container, so a $TMPDIR default silently threw the baseline away and made `verify`'s
|
||||
# before/after comparison — the check that detects damage to the OTHER cluster's records — impossible
|
||||
# to run across steps. The repo is bind-mounted, so a repo-relative dir persists on the host.
|
||||
# Gitignored: these are run artifacts, not authored state.
|
||||
SNAP_DIR="${STANDUP_SNAP_DIR:-${repo_root}/.standup-dns}"
|
||||
mkdir -p "${SNAP_DIR}"
|
||||
|
||||
say() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
warn() { printf '\033[33mWARN: %s\033[0m\n' "$*" >&2; }
|
||||
|
||||
for bin in kubectl flux helm yq sops; do
|
||||
command -v "$bin" >/dev/null 2>&1 || die "'$bin' not found — run this inside infra/devcontainer."
|
||||
done
|
||||
|
||||
# ------------------------------------------------------------------ THE SAFETY GUARD
|
||||
# Checked once, before any step, and re-checked by every mutating helper. The failure mode this
|
||||
# prevents is running a prod bring-up against the LIVE dev cluster, whose public DNS records prod
|
||||
# has already destroyed once (2026-09-04).
|
||||
live_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
|
||||
[ "${live_server}" = "${EXPECT_SERVER}" ] || die \
|
||||
"TARGET MISMATCH — refusing to run.
|
||||
--expect-server : ${EXPECT_SERVER}
|
||||
live API server : ${live_server}
|
||||
The kubeconfig does not point where you think it does. Nothing has been changed."
|
||||
say "target confirmed: ${live_server}"
|
||||
|
||||
# Dependency order. Flux enforces the edges itself via dependsOn; resuming in this order simply means
|
||||
# each wait is meaningful instead of blocking on a dependency that is still suspended.
|
||||
# flux-system -> cilium -> platform -> platform-config -> platform-config-cdi
|
||||
# -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2 -> apps-vms
|
||||
#
|
||||
# `platform-config-cdi` MUST stay in this list and MUST stay before apps-vms. It carries the CDI
|
||||
# StorageProfiles, which were split out of platform-config because StorageProfile's CRD does not exist
|
||||
# until cdi-operator has reconciled the CDI CR (clusters/base/platform-config-cdi.yaml). apps-vms
|
||||
# dependsOn it: apps/vms/unifi provisions on `longhorn-vm`, and without the profile applied CDI infers
|
||||
# ReadWriteOnce instead of the authored ReadWriteMany — permanently non-live-migratable for that disk.
|
||||
# Omitting it here would make `resume` return success without ever waiting on the tier that guarantees it.
|
||||
TIER_ORDER=(platform platform-config platform-config-cdi oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms)
|
||||
# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which is
|
||||
# an intentionally-invalid placeholder until the operator supplies the real R2 endpoint.
|
||||
|
||||
cvar() { kubectl -n flux-system get cm cluster-vars -o jsonpath="{.data.$1}" 2>/dev/null || true; }
|
||||
|
||||
k_ready() { kubectl -n flux-system get kustomization "$1" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true; }
|
||||
k_msg() { kubectl -n flux-system get kustomization "$1" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true; }
|
||||
k_susp() { kubectl -n flux-system get kustomization "$1" -o jsonpath='{.spec.suspend}' 2>/dev/null || true; }
|
||||
|
||||
wait_ready() { # <kustomization> <timeout-seconds>
|
||||
local n="$1" t="${2:-600}" i=0
|
||||
while [ "$i" -lt "$t" ]; do
|
||||
[ "$(k_ready "$n")" = "True" ] && { info "$n Ready (${i}s)"; return 0; }
|
||||
sleep 10; i=$((i+10))
|
||||
done
|
||||
warn "$n did NOT reach Ready in ${t}s: $(k_msg "$n")"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Snapshot the Cloudflare zone. This is how damage to the OTHER cluster's records becomes detectable.
|
||||
# Queried against the Cloudflare API, never a local resolver: the workstation's own dnsmasq wildcard
|
||||
# (address=/.home.keithjackson.dev/) answers the whole tree locally and produced a false
|
||||
# healthy reading during the 2026-09-04 incident and hid the damage for a while.
|
||||
dns_snapshot() { # <output-file>
|
||||
local out="$1" tok zid page tp
|
||||
tok="$(sops -d platform/externaldns/cloudflare-api-token.sops.yaml \
|
||||
| yq '.stringData."api-token"' | tr -d '"')"
|
||||
[ -n "${tok}" ] && [ "${tok}" != "null" ] || die "could not read the Cloudflare token (SOPS/age key available?)"
|
||||
zid="$(curl -sS -H "Authorization: Bearer ${tok}" \
|
||||
'https://api.cloudflare.com/client/v4/zones?name=keithjackson.dev' | yq '.result[0].id' | tr -d '"')"
|
||||
[ -n "${zid}" ] && [ "${zid}" != "null" ] || die "Cloudflare zone lookup failed"
|
||||
page=1; : > "${out}"
|
||||
while :; do
|
||||
local r; r="$(curl -sS -H "Authorization: Bearer ${tok}" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?per_page=100&page=${page}")"
|
||||
printf '%s' "${r}" | yq -o=json '.result' \
|
||||
| yq -r '.[] | (.type + "|" + .name + "|" + .content)' | tr -d '"' >> "${out}"
|
||||
tp="$(printf '%s' "${r}" | yq '.result_info.total_pages' | tr -d '"')"
|
||||
[ "${page}" -ge "${tp:-1}" ] && break
|
||||
page=$((page+1))
|
||||
done
|
||||
# Strip CRs, and sort under a FIXED collation. Both matter, and both were learned the hard way:
|
||||
# - a stray \r makes every line differ from a baseline captured without one;
|
||||
# - locale-dependent sort ordering puts `*.keithjackson.dev` in a different position on the host
|
||||
# than in the container, so a diff reports pairs of IDENTICAL lines as changed. That produced a
|
||||
# "RECORDS OUTSIDE ... CHANGED" alarm on a zone where nothing had changed at all.
|
||||
# A regression check that cries wolf is worse than none — it trains you to skip the one alarm that
|
||||
# matters. LC_ALL=C makes the ordering reproducible everywhere.
|
||||
tr -d '\r' < "${out}" | grep -vE '^\s*$' | LC_ALL=C sort -o "${out}"
|
||||
info "$(wc -l < "${out}") records -> ${out}"
|
||||
}
|
||||
|
||||
dns_diff() { # <before> <after> — returns non-zero if anything changed
|
||||
if diff -u "$1" "$2" > "${SNAP_DIR}/dns.diff"; then
|
||||
info "Cloudflare zone IDENTICAL — nothing created, modified or deleted."
|
||||
return 0
|
||||
fi
|
||||
warn "Cloudflare zone CHANGED:"; cat "${SNAP_DIR}/dns.diff"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ======================================================================= s3check plumbing
|
||||
# WHY THIS IS THIS BIG. The previous version of `s3check` curl'd /minio/health/live and reported
|
||||
# REACHABLE. That probe could not pass against Cloudflare R2 (no such path), and reachability is not
|
||||
# the property anyone cares about: a reachable endpoint with a wrong key, a missing bucket, or R2's
|
||||
# checksum rejection still yields a velero tier that reports Ready while NO backup exists. What must
|
||||
# be proven is the write path Velero and CNPG-barman actually use, with the credentials they actually
|
||||
# hold, from the network position they actually occupy.
|
||||
|
||||
# Pinned. The AWS CLI is a deliberate choice over `minio/mc`:
|
||||
# - it prints the S3 ERROR CODE (NoSuchBucket / AccessDenied / InvalidAccessKeyId / InvalidArgument),
|
||||
# so a failure can be NAMED and turned into a fix instead of guessed at from an HTTP status;
|
||||
# - it exposes the FLEXIBLE-CHECKSUM behaviour that is the R2 trap
|
||||
# (AWS_REQUEST_CHECKSUM_CALCULATION), so the trap can be reproduced on purpose here instead of
|
||||
# discovered by a silently-failing backup at 03:00;
|
||||
# - its image carries bash + coreutils. `minio/mc` ships no sed/awk/diff — that minimalism has
|
||||
# already broken one probe in this repo — and it cannot be made to send the checksum headers, so
|
||||
# it cannot test for the trap at all.
|
||||
S3CHECK_IMAGE="amazon/aws-cli:2.36.40"
|
||||
S3CHECK_RUN_ID="$(date +%s)"
|
||||
|
||||
# Throwaway probe pods, tracked so they are removed on EVERY exit path (including `die`). A probe
|
||||
# must never leave a pod behind on a live cluster.
|
||||
S3CHECK_PODS=()
|
||||
s3_cleanup_pods() {
|
||||
local p
|
||||
for p in ${S3CHECK_PODS[@]+"${S3CHECK_PODS[@]}"}; do
|
||||
kubectl -n "${p%%/*}" delete pod "${p#*/}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
||||
done
|
||||
S3CHECK_PODS=()
|
||||
}
|
||||
trap s3_cleanup_pods EXIT
|
||||
|
||||
# TRUE when the endpoint is not a usable target. `TBD-*` alone was WRONG once the placeholder became
|
||||
# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`: that does not START with TBD-, so the old
|
||||
# prefix pattern matched nothing and `resume` would have resumed velero straight onto a placeholder
|
||||
# hostname — the exact silent-no-backups failure the guard exists to prevent. Match anywhere.
|
||||
s3_is_placeholder() { # <endpoint>
|
||||
case "$1" in
|
||||
""|*TBD*|*tbd-*|*REPLACE-ME*|*replace-me*|*CHANGE-ME*|*change-me*) return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
# Refuse any value that is not a plain URL/name fragment. This is BOTH a safety guard — these values
|
||||
# are interpolated into the probe pod's JSON overrides AND into a shell script that runs inside it, so
|
||||
# a quote, a backslash or a newline could rewrite either — and a correctness guard: a cluster-vars
|
||||
# value with a trailing space or a stray quote is a defect, not something to paper over.
|
||||
s3_safe_token() { # <label> <value>
|
||||
case "$2" in
|
||||
*[!A-Za-z0-9:/._-]*) die "refusing to probe: $1 contains a character outside [A-Za-z0-9:/._-].
|
||||
That is either a corrupt cluster-vars value or an injection attempt. Value (quoted): $(printf '%q' "$2")" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- the in-pod probe
|
||||
# Emitted to the pod base64-encoded, so nothing in it has to survive JSON or shell quoting.
|
||||
# CONTRACT WITH THE CALLER (this is what makes a silent no-op impossible):
|
||||
# - every proven property prints exactly one `S3CHECK-OK[n] ...` line and increments n;
|
||||
# - every failure prints `S3CHECK-FAIL ...` and is counted;
|
||||
# - the LAST line is `S3CHECK-RESULT PASS|FAIL assertions=<n> failures=<n>`.
|
||||
# The caller independently knows how many assertions this environment must produce and re-counts the
|
||||
# OK lines itself, so neither a truncated log, a skipped bucket, an empty log, nor a lying result line
|
||||
# can be read as success. (A CRD guard in this repo once counted a blank line as "1 CRD" and sailed
|
||||
# past; that class of bug is designed out here rather than commented about.)
|
||||
s3_probe_script() {
|
||||
cat <<'PROBE'
|
||||
set -u
|
||||
umask 077
|
||||
A=0; F=0
|
||||
ok() { A=$((A+1)); printf 'S3CHECK-OK[%d] %s\n' "$A" "$*"; }
|
||||
bad() { F=$((F+1)); printf 'S3CHECK-FAIL %s\n' "$*"; }
|
||||
note() { printf 'S3CHECK-NOTE %s\n' "$*"; }
|
||||
verdict() {
|
||||
if [ "${F}" -eq 0 ] && [ "${A}" -gt 0 ]; then
|
||||
printf 'S3CHECK-RESULT PASS assertions=%d failures=0\n' "${A}"; exit 0
|
||||
fi
|
||||
printf 'S3CHECK-RESULT FAIL assertions=%d failures=%d\n' "${A}" "${F}"; exit 1
|
||||
}
|
||||
|
||||
for v in S3EP S3REGION S3TARGETS S3KEY CRED_MODE; do
|
||||
eval "x=\${${v}-}"
|
||||
[ -n "${x}" ] || { bad "required env ${v} is empty — the probe was launched wrong"; verdict; }
|
||||
done
|
||||
|
||||
export AWS_EC2_METADATA_DISABLED=true
|
||||
export AWS_PAGER=""
|
||||
export AWS_DEFAULT_REGION="${S3REGION}"
|
||||
export AWS_REGION="${S3REGION}"
|
||||
export AWS_MAX_ATTEMPTS=2
|
||||
export AWS_RETRY_MODE=standard
|
||||
export AWS_CONFIG_FILE=/tmp/s3check-aws-config
|
||||
# Path-style addressing, because both targets need it: MinIO on a bare IP has no virtual-host names,
|
||||
# and R2's S3 API is documented path-style. Checksums OFF, mirroring `checksumAlgorithm: ""` on the
|
||||
# Velero BSL — this run must exercise the same wire shape the real backup will use. Written to the
|
||||
# CONFIG FILE as well as the environment so the setting holds even if one channel is ignored by a
|
||||
# future CLI version.
|
||||
{
|
||||
echo '[default]'
|
||||
echo 'request_checksum_calculation = when_required'
|
||||
echo 'response_checksum_validation = when_required'
|
||||
echo 's3 ='
|
||||
echo ' addressing_style = path'
|
||||
} > "${AWS_CONFIG_FILE}"
|
||||
|
||||
# ---- (1) the credential loads at all -------------------------------------------------
|
||||
# NEVER print a credential value, in either `key: value` or `key=value` shape. Only booleans,
|
||||
# byte counts and lengths leave this pod.
|
||||
case "${CRED_MODE}" in
|
||||
file)
|
||||
cf="${AWS_SHARED_CREDENTIALS_FILE-}"
|
||||
if [ -z "${cf}" ] || [ ! -s "${cf}" ]; then
|
||||
bad "credential file '${cf:-<unset>}' is missing or empty — the Secret key did not mount"
|
||||
elif ! grep -Eq '^[[:space:]]*\[default\][[:space:]]*$' "${cf}"; then
|
||||
bad "credential file has no [default] profile. velero-plugin-for-aws reads an AWS INI file from
|
||||
the 'cloud' key of velero-cloud-credentials; without [default] every backup fails at RUNTIME"
|
||||
elif ! grep -Eq '^[[:space:]]*aws_access_key_id[[:space:]]*=[[:space:]]*[^[:space:]]' "${cf}" \
|
||||
|| ! grep -Eq '^[[:space:]]*aws_secret_access_key[[:space:]]*=[[:space:]]*[^[:space:]]' "${cf}"; then
|
||||
bad "credential file [default] lacks a non-empty aws_access_key_id / aws_secret_access_key"
|
||||
else
|
||||
ok "credential LOADS: AWS INI with [default] + both keys ($(wc -c < "${cf}") bytes; values never printed)"
|
||||
fi
|
||||
;;
|
||||
env)
|
||||
if [ -z "${AWS_ACCESS_KEY_ID-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY-}" ]; then
|
||||
bad "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY did not arrive from the Secret (missing ACCESS_KEY_ID / ACCESS_SECRET_KEY key?)"
|
||||
else
|
||||
ok "credential LOADS: access key id present (${#AWS_ACCESS_KEY_ID} chars; values never printed)"
|
||||
fi
|
||||
;;
|
||||
*) bad "unknown CRED_MODE '${CRED_MODE}'" ;;
|
||||
esac
|
||||
[ "${F}" -eq 0 ] || verdict # no point talking to S3 with no credential
|
||||
|
||||
OUT=""; RC=0
|
||||
awsx() { OUT="$(aws --endpoint-url "${S3EP}" "$@" 2>&1)"; RC=$?; return 0; }
|
||||
|
||||
detail() { printf '%s\n' "${OUT}" | head -n 12 | while IFS= read -r l; do printf 'S3CHECK-DETAIL %s\n' "${l}"; done; }
|
||||
|
||||
# Every failure is NAMED and given the fix. An unclassified failure is still a failure.
|
||||
diagnose() { # <operation label>
|
||||
local op="$1"
|
||||
case "${OUT}" in
|
||||
*InvalidArgument*|*checksum*|*Checksum*)
|
||||
bad "${op}: 400 InvalidArgument — THE R2 CHECKSUM TRAP. velero-plugin-for-aws v1.14+ bundles an
|
||||
AWS SDK that sends x-amz-sdk-checksum-algorithm / x-amz-checksum-crc32 on every PUT, and
|
||||
Cloudflare R2 REJECTS those headers with HTTP 400 InvalidArgument, so every Velero backup and
|
||||
every kopia blob upload fails. FIX: checksumAlgorithm: \"\" must be present in the
|
||||
backupStorageLocation config in platform/velero/helmrelease.yaml. This probe already disables
|
||||
checksums, so seeing this here means the client sent one anyway — treat it as a hard blocker." ;;
|
||||
*"Could not connect"*|*EndpointConnectionError*|*ConnectTimeoutError*|*"Failed to establish"*|\
|
||||
*"Name or service not known"*|*"nodename nor servname"*|*"Temporary failure in name resolution"*|\
|
||||
*"Connection reset"*|*"Read timeout"*)
|
||||
bad "${op}: this POD cannot reach ${S3EP} (connect / DNS / timeout). Velero and CNPG would
|
||||
install perfectly clean and then fail every backup at RUNTIME — a tier that looks healthy
|
||||
while no backup exists. This is the reading that matters: the workstation's own reachability
|
||||
is irrelevant (prod sits on the SOHO net, pfSense's WAN; the dev LAN's MinIO sits INSIDE
|
||||
pfSense) and a host-side probe has already misled this project once. Fix pod egress/routing,
|
||||
or give this environment its own endpoint." ;;
|
||||
*NoSuchBucket*|*"(404)"*|*"Not Found"*)
|
||||
bad "${op}: the bucket does not exist at this endpoint. Create it before enabling velero/CNPG
|
||||
(R2: dashboard -> R2 -> Create bucket; see infra/BACKUP-R2.md)." ;;
|
||||
*InvalidAccessKeyId*|*SignatureDoesNotMatch*|*"(401)"*|*Unauthorized*)
|
||||
bad "${op}: the endpoint REJECTED these credentials. Wrong key, or a key belonging to a
|
||||
different R2 account / MinIO. Re-encrypt the credential Secrets — infra/BACKUP-R2.md." ;;
|
||||
*AccessDenied*|*"(403)"*|*Forbidden*)
|
||||
bad "${op}: authenticated but FORBIDDEN. The token is not scoped to this bucket. An R2 API token
|
||||
needs Object Read & Write on BOTH the velero and cnpg buckets." ;;
|
||||
*) bad "${op}: unclassified failure (rc=${RC}). Treated as a FAILURE, never as unknown-so-fine." ;;
|
||||
esac
|
||||
detail
|
||||
}
|
||||
|
||||
PAY=/tmp/s3check.payload
|
||||
GOT=/tmp/s3check.got
|
||||
head -c 4096 /dev/urandom | base64 > "${PAY}"
|
||||
[ -s "${PAY}" ] || { bad "could not generate a probe payload inside the pod"; verdict; }
|
||||
|
||||
TRAP_DONE=0
|
||||
for t in ${S3TARGETS}; do
|
||||
B="${t%%:*}"; P="${t#*:}"
|
||||
[ "${P}" = "${B}" ] && P=""
|
||||
if [ -n "${P}" ]; then K="${P}/.s3check/${S3KEY}"; else K=".s3check/${S3KEY}"; fi
|
||||
|
||||
# ---- (2) the bucket exists and is LISTABLE. This is the operation Velero's BSL sync and Barman's
|
||||
# WAL scan both perform; ListBuckets deliberately is NOT used, because neither consumer
|
||||
# needs it and a correctly bucket-scoped R2 token may not grant it.
|
||||
args=(s3api list-objects-v2 --bucket "${B}" --max-keys 1)
|
||||
[ -n "${P}" ] && args+=(--prefix "${P}/")
|
||||
awsx "${args[@]}"
|
||||
if [ "${RC}" -eq 0 ]; then
|
||||
ok "${B}: bucket EXISTS, AUTHENTICATES and is LISTABLE (prefix '${P:-<bucket root>}')"
|
||||
else
|
||||
diagnose "LIST ${B}"
|
||||
continue # a short assertion count is how the caller learns this target was abandoned
|
||||
fi
|
||||
|
||||
# ---- (3) PUT -> GET -> byte-compare -> DELETE. A real write, not a HEAD: HEAD on a bucket proves
|
||||
# nothing about whether an object can be STORED (wrong scope, checksum rejection and
|
||||
# read-only tokens all pass a HEAD and fail a backup).
|
||||
awsx s3api put-object --bucket "${B}" --key "${K}" --body "${PAY}"
|
||||
if [ "${RC}" -eq 0 ]; then ok "${B}: PUT ${K} accepted (genuine write path)"
|
||||
else diagnose "PUT ${B}/${K}"; continue; fi
|
||||
|
||||
rm -f "${GOT}"
|
||||
awsx s3api get-object --bucket "${B}" --key "${K}" "${GOT}"
|
||||
if [ "${RC}" -eq 0 ]; then ok "${B}: GET ${K} returned an object"
|
||||
else diagnose "GET ${B}/${K}"
|
||||
aws --endpoint-url "${S3EP}" s3api delete-object --bucket "${B}" --key "${K}" >/dev/null 2>&1 || true
|
||||
continue; fi
|
||||
|
||||
sp="$(wc -c < "${PAY}")"; sg="$(wc -c < "${GOT}" 2>/dev/null || echo 0)"
|
||||
hp="$(sha256sum < "${PAY}")"; hp="${hp%% *}"
|
||||
hg="$(sha256sum < "${GOT}")"; hg="${hg%% *}"
|
||||
if [ "${sp}" = "${sg}" ] && [ -n "${hp}" ] && [ "${hp}" = "${hg}" ]; then
|
||||
ok "${B}: round trip is BYTE-IDENTICAL (${sp} bytes, sha256 matches)"
|
||||
else
|
||||
bad "${B}: round trip CORRUPTED — wrote ${sp} bytes (sha256 ${hp}), read back ${sg} bytes (sha256 ${hg})"
|
||||
fi
|
||||
|
||||
awsx s3api delete-object --bucket "${B}" --key "${K}"
|
||||
if [ "${RC}" -eq 0 ]; then ok "${B}: DELETE ${K} accepted"
|
||||
else diagnose "DELETE ${B}/${K}"; fi
|
||||
|
||||
awsx s3api head-object --bucket "${B}" --key "${K}"
|
||||
if [ "${RC}" -ne 0 ]; then ok "${B}: scratch object is GONE — the probe left nothing behind"
|
||||
else bad "${B}: scratch object ${K} STILL EXISTS after DELETE. Remove it by hand; the probe leaked."; fi
|
||||
|
||||
# ---- (4) the R2 checksum trap, probed ON PURPOSE. Informational by design: the authored BSL sets
|
||||
# checksumAlgorithm: "", so a rejection here does NOT mean backups are broken — it means
|
||||
# that line is LOAD-BEARING. Reported either way so the operator learns which world this
|
||||
# endpoint lives in before, not after, enabling velero.
|
||||
if [ "${TRAP_DONE}" -eq 0 ]; then
|
||||
TRAP_DONE=1
|
||||
TK="${K}.checksum-trap-probe"
|
||||
OUT="$(AWS_REQUEST_CHECKSUM_CALCULATION=when_supported \
|
||||
aws --endpoint-url "${S3EP}" s3api put-object --bucket "${B}" --key "${TK}" --body "${PAY}" 2>&1)"; RC=$?
|
||||
if [ "${RC}" -eq 0 ]; then
|
||||
note "checksum probe: this endpoint ACCEPTS a checksum-bearing PUT — the R2 checksum trap is NOT
|
||||
present here (expected for MinIO). checksumAlgorithm: \"\" stays harmless."
|
||||
aws --endpoint-url "${S3EP}" s3api delete-object --bucket "${B}" --key "${TK}" >/dev/null 2>&1 \
|
||||
|| note "checksum probe: could NOT delete ${TK} — remove it by hand"
|
||||
else
|
||||
case "${OUT}" in
|
||||
*InvalidArgument*)
|
||||
note "checksum probe: R2 CHECKSUM TRAP CONFIRMED at this endpoint — a PUT carrying
|
||||
x-amz-sdk-checksum-algorithm / x-amz-checksum-crc32 is rejected with 400 InvalidArgument.
|
||||
checksumAlgorithm: \"\" in platform/velero/helmrelease.yaml is LOAD-BEARING: delete it, or bump
|
||||
velero-plugin-for-aws without re-checking it, and EVERY backup breaks." ;;
|
||||
*) note "checksum probe: inconclusive (rc=${RC}) — neither confirms nor rules out the trap." ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
verdict
|
||||
PROBE
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- run one probe pod
|
||||
# <ns> <file|env> <secret> <space-separated bucket:prefix targets> <endpoint> <region>
|
||||
s3_probe() {
|
||||
local ns="$1" mode="$2" sec="$3" targets="$4" ep="$5" region="$6"
|
||||
# $$ alone is NOT unique here: inside the dev container this script is frequently PID 1, so two runs
|
||||
# would reuse the same pod name AND the same scratch object key. Mix in the epoch second.
|
||||
local pod="s3check-$$-${S3CHECK_RUN_ID}-${mode}"
|
||||
|
||||
# What this environment MUST produce: 1 credential assertion + 6 per bucket (list, put, get,
|
||||
# byte-compare, delete, delete-verified). Computed HERE, by the caller, from the target list — the
|
||||
# pod's own claim is never the only source of the expected count.
|
||||
local t n=0
|
||||
for t in ${targets}; do n=$((n+1)); done
|
||||
local min=$((1 + 6 * n))
|
||||
|
||||
local volsjson="" mountsjson="" creds=""
|
||||
case "${mode}" in
|
||||
file) # exactly how velero consumes it: the `cloud` key as an AWS INI credentials FILE
|
||||
volsjson='"volumes":[{"name":"s3creds","secret":{"secretName":"'"${sec}"'","items":[{"key":"cloud","path":"credentials"}],"defaultMode":256}}],'
|
||||
mountsjson='"volumeMounts":[{"name":"s3creds","mountPath":"/s3creds","readOnly":true}],'
|
||||
creds='{"name":"AWS_SHARED_CREDENTIALS_FILE","value":"/s3creds/credentials"},{"name":"CRED_MODE","value":"file"}' ;;
|
||||
env) # exactly how CNPG/barman consumes it: discrete ACCESS_KEY_ID / ACCESS_SECRET_KEY keys
|
||||
creds='{"name":"AWS_ACCESS_KEY_ID","valueFrom":{"secretKeyRef":{"name":"'"${sec}"'","key":"ACCESS_KEY_ID"}}},{"name":"AWS_SECRET_ACCESS_KEY","valueFrom":{"secretKeyRef":{"name":"'"${sec}"'","key":"ACCESS_SECRET_KEY"}}},{"name":"CRED_MODE","value":"env"}' ;;
|
||||
*) die "internal: unknown probe mode '${mode}'" ;;
|
||||
esac
|
||||
|
||||
local b64; b64="$(s3_probe_script | base64 | tr -d '\n')"
|
||||
[ -n "${b64}" ] || die "internal: probe script encoded to nothing"
|
||||
|
||||
# egress-class label: harmless here (the velero namespace is not an app namespace) but required by
|
||||
# the require-egress-class-label ClusterPolicy in any namespace labelled app-namespace=true, which
|
||||
# the CNPG credential namespaces are. Without it the probe pod would be REJECTED by admission and
|
||||
# the check would report "unknown" instead of a result.
|
||||
local overrides
|
||||
overrides='{"apiVersion":"v1","metadata":{"labels":{"app.kubernetes.io/name":"s3check","homelab.keithjackson.dev/egress-class":"internet"}},"spec":{"restartPolicy":"Never","automountServiceAccountToken":false,'"${volsjson}"'"containers":[{"name":"s3check","image":"'"${S3CHECK_IMAGE}"'","imagePullPolicy":"IfNotPresent","command":["/bin/bash","-c","printf %s \"$PROBE_B64\" | base64 -d > /tmp/probe.sh && exec bash /tmp/probe.sh"],'"${mountsjson}"'"env":['"${creds}"',{"name":"PROBE_B64","value":"'"${b64}"'"},{"name":"S3EP","value":"'"${ep}"'"},{"name":"S3REGION","value":"'"${region}"'"},{"name":"S3TARGETS","value":"'"${targets}"'"},{"name":"S3KEY","value":"'"${pod}"'.probe"},{"name":"AWS_REQUEST_CHECKSUM_CALCULATION","value":"when_required"},{"name":"AWS_RESPONSE_CHECKSUM_VALIDATION","value":"when_required"}],"resources":{"requests":{"cpu":"50m","memory":"64Mi"},"limits":{"memory":"256Mi"}}}]}}'
|
||||
|
||||
say "probe: ns=${ns} secret=${sec} (${mode}) targets='${targets}' → ${min} assertions required"
|
||||
kubectl -n "${ns}" delete pod "${pod}" --ignore-not-found >/dev/null 2>&1 || true
|
||||
S3CHECK_PODS+=("${ns}/${pod}")
|
||||
kubectl -n "${ns}" run "${pod}" --restart=Never --image="${S3CHECK_IMAGE}" \
|
||||
--overrides="${overrides}" >/dev/null 2>&1 \
|
||||
|| die "could NOT create the probe pod ${ns}/${pod} (admission policy? RBAC? quota?).
|
||||
That is a FAILURE, not an unknown: nothing about ${ep} has been proven."
|
||||
|
||||
local i=0 phase=""
|
||||
while [ "${i}" -lt 300 ]; do
|
||||
phase="$(kubectl -n "${ns}" get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
case "${phase}" in Succeeded|Failed) break ;; esac
|
||||
sleep 5; i=$((i+5))
|
||||
done
|
||||
case "${phase}" in
|
||||
Succeeded|Failed) ;;
|
||||
*) local reason
|
||||
reason="$(kubectl -n "${ns}" get pod "${pod}" \
|
||||
-o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
|
||||
die "probe pod ${ns}/${pod} did not finish in ${i}s (phase=${phase:-<none>}, waiting=${reason:-none}).
|
||||
NOT a pass. If waiting=ErrImagePull/ImagePullBackOff this cluster cannot pull ${S3CHECK_IMAGE}." ;;
|
||||
esac
|
||||
|
||||
local log; log="$(kubectl -n "${ns}" logs "${pod}" 2>/dev/null || true)"
|
||||
[ -n "${log}" ] || die "probe pod ${ns}/${pod} produced NO output (phase=${phase}). Failing closed —
|
||||
an empty log is the shape of a probe that never ran, not of a healthy backup target."
|
||||
printf '%s\n' "${log}" | while IFS= read -r l; do printf ' %s\n' "${l}"; done
|
||||
|
||||
# ---- the four independent accounting checks that make a silent no-op impossible ----
|
||||
local okcount failcount resline nres verd a f
|
||||
okcount="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-OK\[' || true)"
|
||||
failcount="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-FAIL' || true)"
|
||||
nres="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-RESULT ' || true)"
|
||||
[ "${nres}" = "1" ] || die "expected exactly ONE S3CHECK-RESULT line from ${ns}/${pod}, found ${nres}.
|
||||
A missing verdict line means the probe died mid-way; it is never a pass."
|
||||
resline="$(printf '%s\n' "${log}" | grep '^S3CHECK-RESULT ')"
|
||||
read -r _ verd a f <<<"${resline}"
|
||||
a="${a#assertions=}"; f="${f#failures=}"
|
||||
case "${a}" in ''|*[!0-9]*) die "malformed verdict from ${ns}/${pod}: '${resline}'" ;; esac
|
||||
case "${f}" in ''|*[!0-9]*) die "malformed verdict from ${ns}/${pod}: '${resline}'" ;; esac
|
||||
[ "${verd}" = "PASS" ] || die "S3 ROUND TRIP FAILED against ${ep} (${ns}/${pod}: ${f} failure(s)).
|
||||
Read the S3CHECK-FAIL lines above — each one names the cause and the fix. Do NOT enable velero
|
||||
or CNPG backups against this endpoint until they are all gone."
|
||||
[ "${f}" = "0" ] || die "${ns}/${pod} claimed PASS while reporting ${f} failures — refusing to believe it."
|
||||
[ "${failcount}" = "0" ] || die "${ns}/${pod} claimed PASS but the log carries ${failcount} S3CHECK-FAIL line(s)."
|
||||
[ "${a}" -ge "${min}" ] || die "${ns}/${pod} claimed PASS with only ${a} assertions; ${min} are required
|
||||
(1 credential + 6 per bucket). A short count means steps were SKIPPED — exactly the
|
||||
degrade-to-a-successful-no-op this check exists to prevent."
|
||||
[ "${okcount}" = "${a}" ] || die "assertion accounting mismatch for ${ns}/${pod}: ${okcount} S3CHECK-OK
|
||||
lines in the log vs a claimed ${a}. Refusing to pass on a self-report that does not add up."
|
||||
[ "${phase}" = "Succeeded" ] || die "${ns}/${pod} printed PASS but the pod ended ${phase}."
|
||||
info "PROVEN: ${a} assertions, 0 failures, pod ${phase}, no object left behind."
|
||||
}
|
||||
|
||||
# ============================================================================ steps
|
||||
step_preflight() {
|
||||
say "PREFLIGHT (read-only)"
|
||||
info "nodes Ready: $(kubectl get nodes --no-headers | awk '$2=="Ready"' | wc -l)/$(kubectl get nodes --no-headers | wc -l)"
|
||||
info "flux controllers Running: $(kubectl -n flux-system get pods --no-headers | awk '$3=="Running"' | wc -l)"
|
||||
# Count BOTH groups the Kyverno chart ships. Grepping only 'kyverno.io' undercounts by 2 — the
|
||||
# policy-report types are in the wgpolicyk8s.io group — which reads as "CRDs are missing" when the
|
||||
# pre-apply in fact succeeded. Expect 22 for kyverno 3.9.x.
|
||||
info "kyverno CRDs present: $(kubectl get crd --no-headers 2>/dev/null | grep -cE 'kyverno\.io|wgpolicyk8s\.io' || true) (expect 22 for kyverno 3.9.x)"
|
||||
|
||||
local base owner filter s3 acme
|
||||
base="$(cvar BASE_DOMAIN)"; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
|
||||
filter="$(cvar EXTERNALDNS_DOMAIN_FILTER)"; s3="$(cvar S3_BACKUP_ENDPOINT)"; acme="$(cvar ACME_CLUSTER_ISSUER)"
|
||||
say "cluster-vars guards"
|
||||
info "BASE_DOMAIN = ${base}"
|
||||
info "EXTERNALDNS_TXT_OWNER_ID = ${owner}"
|
||||
info "EXTERNALDNS_DOMAIN_FILTER = ${filter}"
|
||||
info "ACME_CLUSTER_ISSUER = ${acme}"
|
||||
info "S3_BACKUP_ENDPOINT = ${s3}"
|
||||
|
||||
# THE GUARD THAT MATTERS. Two clusters sharing one Cloudflare zone are isolated by txtOwnerId and
|
||||
# by serving disjoint hostnames — NOT by domainFilters (finding F-DNS1: a subdomain filter drops the
|
||||
# parent zone and publishes nothing). Both clusters inheriting the chart default `default` is the
|
||||
# single root cause of the 2026-09-04 incident, in which prod deleted every one of dev's records.
|
||||
# preflight is a DIAGNOSTIC: it must always finish and report. The only thing it refuses over is the
|
||||
# target guard, because that one means you are pointed at the wrong cluster. Everything else is
|
||||
# reported and warned about here, and REFUSED on the mutating path (step_externaldns) where acting
|
||||
# on a bad value is what actually causes harm. Dying here instead hid the rest of dev's health
|
||||
# report behind a condition that was, for dev, simply the long-standing status quo.
|
||||
if [ -z "${owner}" ]; then
|
||||
warn "EXTERNALDNS_TXT_OWNER_ID is EMPTY — external-dns falls back to the chart default 'default'.
|
||||
Safe only if this is the sole cluster publishing into the zone. 'externaldns' will refuse."
|
||||
elif [ "${owner}" = "unset-see-cluster-vars" ]; then
|
||||
warn "EXTERNALDNS_TXT_OWNER_ID is still the base sentinel — the overlay did not apply.
|
||||
'externaldns' will refuse until a real per-environment value is set."
|
||||
elif [ "${owner}" = "default" ]; then
|
||||
# NOT fatal here, because `default` is DEV's legitimate, established owner id — its live records
|
||||
# are registered under it and cannot be renamed without orphaning them. Dying on it would make
|
||||
# this script refuse to run against dev at all, which would be an environment fork (R-ENV-02) in
|
||||
# a script whose whole claim is that it has none. preflight only READS, so it warns; the
|
||||
# MUTATING path (step_externaldns) still refuses outright.
|
||||
warn "EXTERNALDNS_TXT_OWNER_ID is 'default' — the Helm chart's default value.
|
||||
That is correct for DEV (its records are registered under it) and WRONG for any second cluster
|
||||
sharing this zone: two clusters both defaulting to 'default' is the exact root cause of the
|
||||
2026-09-04 incident, in which prod treated dev's TXT registry entries as its own and
|
||||
policy: sync deleted the records behind them."
|
||||
else
|
||||
info "owner id is distinct and non-default: OK"
|
||||
fi
|
||||
|
||||
if s3_is_placeholder "${s3}"; then
|
||||
warn "S3_BACKUP_ENDPOINT is unset/placeholder ('${s3}'). velero AND apps-wave-0 consume it.
|
||||
velero will be SKIPPED by the resume step. apps-wave-0 will still apply (the token IS defined,
|
||||
so strict substitution succeeds) but anything writing to that endpoint fails at RUNTIME, not at
|
||||
apply time — so a green apps-wave-0 does NOT mean backups work. Operator decision required."
|
||||
else
|
||||
info "S3 backup endpoint set: OK"
|
||||
fi
|
||||
|
||||
# COLD-START CRD ORDERING. Flux server-side dry-runs EVERY object in a Kustomization before applying
|
||||
# any of them, so a single unknown `kind` aborts the whole tier. platform-config applies the CDI CR
|
||||
# *and* CDI StorageProfiles in the same pass — but the StorageProfile CRD does not exist until the
|
||||
# cdi-operator has reconciled that CR. On an established cluster the CRDs are long since there and
|
||||
# this is invisible; on a COLD one it is a deadlock, and it is the same class of bug as Kyverno's
|
||||
# 5.7 MiB CRD bundle. Report it here so `resume` does not burn 15 minutes discovering it.
|
||||
say "CRDs that later tiers apply resources against"
|
||||
local crd missing=0
|
||||
for crd in storageprofiles.cdi.kubevirt.io cdis.cdi.kubevirt.io kubevirts.kubevirt.io \
|
||||
volumesnapshotclasses.snapshot.storage.k8s.io; do
|
||||
if kubectl get crd "${crd}" >/dev/null 2>&1; then
|
||||
printf ' %-52s present\n' "${crd}"
|
||||
else
|
||||
printf ' \033[33m%-52s MISSING\033[0m\n' "${crd}"; missing=1
|
||||
fi
|
||||
done
|
||||
if [ "${missing}" = "1" ]; then
|
||||
warn "a MISSING crd above will fail platform-config's dry-run and block every tier below it.
|
||||
CDI's own CRDs (StorageProfile, DataVolume, ...) are created by cdi-operator only AFTER it
|
||||
reconciles the CDI CR. Check that the CDI CR is applied and cdi-operator is Available; if the
|
||||
CRD appeared only recently this can also be a stale discovery cache in kustomize-controller,
|
||||
which a forced reconcile clears."
|
||||
kubectl -n cdi get cdi,deploy 2>/dev/null | head -12 || info "(no cdi namespace yet)"
|
||||
fi
|
||||
|
||||
say "tiers"
|
||||
local n
|
||||
for n in flux-system cilium "${TIER_ORDER[@]}" velero; do
|
||||
printf ' %-16s suspend=%-6s ready=%-6s %s\n' "$n" "$(k_susp "$n")" "$(k_ready "$n")" "$(k_msg "$n" | head -c 90)"
|
||||
done
|
||||
|
||||
say "HelmReleases not Ready"
|
||||
kubectl get helmrelease -A \
|
||||
-o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,MSG:.status.conditions[?\(@.type==\"Ready\"\)].message \
|
||||
--no-headers 2>/dev/null | grep -v ' True ' || info "(all Ready)"
|
||||
|
||||
# SUSPENDED releases are invisible above: a suspended HelmRelease keeps whatever Ready condition it
|
||||
# last had, so a STALE `Ready=True` hides the fact that Flux is not managing it at all. external-dns
|
||||
# is deliberately in this state until `standup.sh externaldns` runs.
|
||||
say "HelmReleases SUSPENDED (Flux is not reconciling these — a stale Ready=True means nothing)"
|
||||
kubectl get helmrelease -A \
|
||||
-o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend --no-headers 2>/dev/null \
|
||||
| awk '$3=="true"' || true
|
||||
}
|
||||
|
||||
step_crds() {
|
||||
say "CRD PRE-APPLY (idempotent)"
|
||||
# Charts whose CRD bundle exceeds kubectl's 256 KiB annotation limit set crds.install=false, so
|
||||
# nothing installs them and `platform` can never go Ready. See bootstrap/flux/preapply-crds.sh.
|
||||
bash "${script_dir}/flux/preapply-crds.sh"
|
||||
|
||||
# ---------------------------------------------------------------- the CDI cold-start deadlock
|
||||
# SAME CLASS OF BUG, DIFFERENT OPERATOR. `platform-config` applies the CDI CR *and*
|
||||
# platform/kubevirt/cdi-storageprofiles.yaml in ONE pass. But StorageProfile's CRD does not exist
|
||||
# until cdi-operator has reconciled that CR — and Flux server-side dry-runs EVERY object before
|
||||
# applying ANY of them, so the unknown kind aborts the whole tier, the CR is never applied, the CRD
|
||||
# is never created, and platform-config can never converge. A true deadlock, invisible on an
|
||||
# established cluster (the CRDs are long since there) and fatal on a cold one.
|
||||
#
|
||||
# Breaking it needs exactly one thing: get the CDI CR in. Then cdi-operator creates the remaining
|
||||
# CDI CRDs and platform-config's next reconcile dry-runs clean. We apply ONLY the CR — the minimum
|
||||
# — and leave every other object to Flux. The CR carries no ${} tokens, so no substitution is
|
||||
# needed and applying it here cannot diverge from what Flux would render.
|
||||
local cdi_cr="${repo_root}/platform/kubevirt/cdi-cr.yaml"
|
||||
if kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1; then
|
||||
info "storageprofiles CRD already present — CDI is past its cold start"
|
||||
elif [ ! -f "${cdi_cr}" ]; then
|
||||
warn "expected ${cdi_cr} and it is missing — skipping the CDI unblock"
|
||||
else
|
||||
say "CDI cold-start unblock: applying the CDI CR so cdi-operator can create its CRDs"
|
||||
kubectl get ns cdi >/dev/null 2>&1 || die "namespace 'cdi' absent — the platform tier has not applied the CDI operator yet. Run 'resume' far enough to make platform Ready first."
|
||||
kubectl apply --server-side --force-conflicts -f "${cdi_cr}"
|
||||
local i=0
|
||||
while [ "$i" -lt 300 ]; do
|
||||
kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 && break
|
||||
sleep 10; i=$((i+10))
|
||||
done
|
||||
kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 \
|
||||
|| die "cdi-operator did not create storageprofiles.cdi.kubevirt.io within ${i}s.
|
||||
Inspect: kubectl -n cdi get cdi,pods,deploy — platform-config stays blocked until it exists."
|
||||
kubectl wait --for=condition=Established --timeout=60s crd/storageprofiles.cdi.kubevirt.io >/dev/null
|
||||
info "storageprofiles.cdi.kubevirt.io created and Established (${i}s)"
|
||||
fi
|
||||
}
|
||||
|
||||
step_unwedge() {
|
||||
say "UNWEDGE STUCK HELM RELEASES"
|
||||
# A Helm release left in uninstalling/pending-* cannot be advanced by helm-controller: it reports
|
||||
# StateError ("unable to determine state for release with status 'uninstalling'") forever, which
|
||||
# keeps `platform` (wait: true) not-Ready and every tier below it blocked.
|
||||
#
|
||||
# How prod got here: Kyverno's install timed out because its pods were crash-looping with no CRDs;
|
||||
# install.remediation.retries:3 then fired an uninstall, and that uninstall timed out too.
|
||||
#
|
||||
# Pre-applied CRDs live OUTSIDE the Helm release and SURVIVE this — that is precisely why the
|
||||
# pre-apply is a separate step and must run before this one.
|
||||
local found=0 ns rel st
|
||||
while read -r ns rel st; do
|
||||
[ -n "${ns:-}" ] || continue
|
||||
found=1
|
||||
warn "${ns}/${rel} is '${st}'"
|
||||
[ "${CONFIRM_DESTRUCTIVE}" = "1" ] || {
|
||||
info "DRY RUN. This DELETES the release's Kubernetes objects and lets Flux install cold."
|
||||
info "Re-run with --confirm-destructive to act."
|
||||
continue
|
||||
}
|
||||
say "clearing ${ns}/${rel}"
|
||||
# --no-hooks and --wait=false: the WAIT is what hung the original remediation, so do not repeat it.
|
||||
helm uninstall "${rel}" -n "${ns}" --no-hooks --wait=false || warn "helm uninstall returned non-zero; continuing to check state"
|
||||
if helm list -n "${ns}" --all -q 2>/dev/null | grep -qx "${rel}"; then
|
||||
warn "${ns}/${rel} still present after uninstall — inspect manually, NOT deleting release secrets by hand."
|
||||
else
|
||||
info "${ns}/${rel} cleared"
|
||||
flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile of ${ns}/${rel} did not complete in 10m"
|
||||
fi
|
||||
done < <(
|
||||
for ns in $(kubectl get helmrelease -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"\n"}{end}' | sort -u); do
|
||||
helm list -n "${ns}" --all -o json 2>/dev/null \
|
||||
| yq -r '.[] | select(.status | test("uninstalling|pending-install|pending-upgrade|pending-rollback")) | "'"${ns}"' " + .name + " " + .status' 2>/dev/null || true
|
||||
done
|
||||
)
|
||||
[ "${found}" = "1" ] || info "no wedged releases found."
|
||||
}
|
||||
|
||||
step_externaldns() {
|
||||
say "EXTERNAL-DNS — SAFE UPGRADE (prove the owner id BEFORE the pod can run)"
|
||||
# THE HAZARD: a Helm upgrade sets replicas back to the chart default (1) and applies the new args in
|
||||
# ONE operation. There is no window in which to check the result first. If --txt-owner-id is missing
|
||||
# the pod starts as owner `default`, claims the other cluster's TXT registry entries, and under
|
||||
# `policy: sync` DELETES the records behind them. That is the 2026-09-04 incident.
|
||||
#
|
||||
# THE FIX IS TO VERIFY IN ADVANCE, NOT AFTER: render the chart with the LIVE HelmRelease values and
|
||||
# assert the flag appears. Rendering is deterministic and touches nothing, so this converts a race
|
||||
# into a precondition.
|
||||
local ns=external-dns rel=external-dns
|
||||
local owner chart ver repo vals
|
||||
owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
|
||||
[ -n "${owner}" ] && [ "${owner}" != "default" ] && [ "${owner}" != "unset-see-cluster-vars" ] \
|
||||
|| die "refusing: EXTERNALDNS_TXT_OWNER_ID is '${owner}'"
|
||||
|
||||
chart="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.chart}')"
|
||||
ver="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.version}')"
|
||||
repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' platform/externaldns/helmrepository.yaml | tr -d '"')"
|
||||
[ -n "${repo}" ] || die "could not derive the external-dns chart repo url"
|
||||
|
||||
vals="${SNAP_DIR}/ed-values.yaml"
|
||||
kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.values}' | yq -P > "${vals}"
|
||||
info "chart=${chart} version=${ver} repo=${repo}"
|
||||
info "live spec.values.txtOwnerId = $(yq '.txtOwnerId' "${vals}")"
|
||||
|
||||
say "rendering the chart with the LIVE values to prove the flag"
|
||||
local rendered="${SNAP_DIR}/ed-rendered.yaml"
|
||||
helm template "${rel}" "${chart}" --repo "${repo}" --version "${ver}" -n "${ns}" -f "${vals}" > "${rendered}"
|
||||
local args; args="$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].args[]' "${rendered}")"
|
||||
printf '%s\n' "${args}" | sed 's/^/ /'
|
||||
printf '%s\n' "${args}" | grep -qx -- "--txt-owner-id=${owner}" \
|
||||
|| die "the render does NOT contain --txt-owner-id=${owner}.
|
||||
An upgrade would start external-dns as owner 'default' and delete the other cluster's records.
|
||||
REFUSING. Fix the chart values key before retrying."
|
||||
info "PROVEN: the upgrade will carry --txt-owner-id=${owner}"
|
||||
|
||||
say "snapshotting the Cloudflare zone (baseline for the post-check)"
|
||||
dns_snapshot "${SNAP_DIR}/dns-before.txt"
|
||||
|
||||
say "resuming + forcing the upgrade"
|
||||
kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.suspend}' | grep -q true \
|
||||
&& flux resume helmrelease "${rel}" -n "${ns}" || info "(HelmRelease was not suspended)"
|
||||
flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile did not complete in 10m"
|
||||
|
||||
say "verifying the LIVE deployment args"
|
||||
local live; live="$(kubectl -n "${ns}" get deploy "${rel}" -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null || true)"
|
||||
printf ' %s\n' "${live}"
|
||||
printf '%s' "${live}" | grep -q -- "--txt-owner-id=${owner}" \
|
||||
|| { kubectl -n "${ns}" scale deploy "${rel}" --replicas=0 || true
|
||||
die "live args LACK --txt-owner-id=${owner}. Scaled to 0. Investigate before scaling up."; }
|
||||
info "live deployment carries --txt-owner-id=${owner}"
|
||||
|
||||
say "post-check: did anything in the shared zone change?"
|
||||
sleep 45 # external-dns --interval=1m; give it a full cycle to act before judging it safe
|
||||
dns_snapshot "${SNAP_DIR}/dns-after.txt"
|
||||
dns_diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-after.txt" \
|
||||
|| warn "records changed — expected ONLY additions under this cluster's own BASE_DOMAIN ($(cvar BASE_DOMAIN)).
|
||||
Any change to a name outside it, or to a TXT entry owned by another cluster, is a REGRESSION."
|
||||
}
|
||||
|
||||
step_s3check() {
|
||||
say "S3 BACKUP TARGET — PROVE A REAL ROUND TRIP FROM INSIDE THE CLUSTER"
|
||||
# Velero and CNPG run in-cluster, so pod egress is the only reachability that counts: prod sits on
|
||||
# the SOHO network (pfSense's WAN) while the dev LAN's MinIO sits INSIDE pfSense, so host and pod
|
||||
# egress differ — and a healthy host-side reading has already misled this project once.
|
||||
#
|
||||
# WHAT THIS PROVES (and it fails closed on every one of them):
|
||||
# 1. the committed credential loads and AUTHENTICATES, in the exact shape each consumer reads it
|
||||
# (velero: an AWS INI file from the `cloud` key; CNPG: discrete ACCESS_KEY_ID/ACCESS_SECRET_KEY);
|
||||
# 2. each required bucket EXISTS and is LISTABLE under this environment's own prefix;
|
||||
# 3. PUT -> GET -> byte-compare -> DELETE of a throwaway object under that prefix — the genuine
|
||||
# write path, not a HEAD (a HEAD passes with a read-only token and with R2 rejecting writes);
|
||||
# 4. whether the R2 CHECKSUM TRAP is present at this endpoint, named explicitly either way.
|
||||
#
|
||||
# WHAT IT STILL DOES NOT PROVE: that Velero's kopia uploader and CNPG's barman-cloud themselves work
|
||||
# against this target. Both are UNPROVEN against R2 in this repo. See infra/BACKUP-R2.md.
|
||||
local ep live region
|
||||
live="$(cvar S3_BACKUP_ENDPOINT)"
|
||||
info "live S3_BACKUP_ENDPOINT = ${live:-<unset>}"
|
||||
if [ -n "${S3_ENDPOINT_OVERRIDE}" ]; then
|
||||
# Probe a CANDIDATE endpoint before adopting it. The whole point is to find out whether a proposed
|
||||
# backup target works from this cluster's pods BEFORE authoring it into cluster-vars — otherwise
|
||||
# the first evidence is silently-missing backups.
|
||||
ep="${S3_ENDPOINT_OVERRIDE}"
|
||||
info "probing CANDIDATE endpoint from --s3-endpoint: ${ep}"
|
||||
else
|
||||
ep="${live}"
|
||||
fi
|
||||
if s3_is_placeholder "${ep}"; then
|
||||
die "no usable endpoint to probe (endpoint='${ep:-<unset>}').
|
||||
cluster-vars still carries the loud placeholder. Pass --s3-endpoint https://<account-id>.r2.cloudflarestorage.com
|
||||
to test a CANDIDATE before authoring it. See infra/BACKUP-R2.md."
|
||||
fi
|
||||
s3_safe_token S3_BACKUP_ENDPOINT "${ep}"
|
||||
case "${ep}" in
|
||||
http://*|https://*) ;;
|
||||
*) die "endpoint '${ep}' carries no http:// or https:// scheme — the AWS SDK needs one." ;;
|
||||
esac
|
||||
|
||||
# Region is a knob, not a constant: MinIO ignores it, R2 requires exactly `auto`, and an EMPTY value
|
||||
# in cluster-vars silently builds a BSL with no region.
|
||||
region="$(cvar S3_BACKUP_REGION)"
|
||||
case "${ep}" in
|
||||
*.r2.cloudflarestorage.com*)
|
||||
if [ "${region}" != "auto" ]; then
|
||||
warn "endpoint is Cloudflare R2 but cluster-vars S3_BACKUP_REGION='${region:-<unset>}'. R2 requires
|
||||
exactly 'auto'. Probing with 'auto' anyway; FIX clusters/base/cluster-vars.yaml before enabling
|
||||
velero, or the BackupStorageLocation is built with the wrong region."
|
||||
fi
|
||||
region=auto ;;
|
||||
*)
|
||||
if [ -z "${region}" ]; then
|
||||
region="us-east-1"
|
||||
warn "cluster-vars S3_BACKUP_REGION is EMPTY on this cluster. Probing with 'us-east-1' (MinIO
|
||||
ignores it) — but the BSL would be rendered with region: \"\". Set it explicitly."
|
||||
fi ;;
|
||||
esac
|
||||
s3_safe_token S3_BACKUP_REGION "${region}"
|
||||
|
||||
# Buckets come from the AUTHORED manifests + cluster-vars, never from a hardcoded name here: the
|
||||
# whole point is to probe what this environment will really write to (R-ENV-02, no dev/prod fork).
|
||||
local vbucket vprefix cpath cbucket cprefix rest
|
||||
vbucket="$(yq -r '.spec.values.configuration.backupStorageLocation[0].bucket' \
|
||||
platform/velero/helmrelease.yaml 2>/dev/null | tr -d '"')"
|
||||
[ -n "${vbucket}" ] && [ "${vbucket}" != "null" ] \
|
||||
|| die "could not read the Velero bucket name from platform/velero/helmrelease.yaml"
|
||||
vprefix="$(cvar VELERO_BACKUP_PREFIX)"
|
||||
|
||||
cpath="$(cvar CNPG_BACKUP_PATH)"
|
||||
if [ -z "${cpath}" ]; then
|
||||
warn "cluster-vars CNPG_BACKUP_PATH is EMPTY on this cluster. barmanObjectStore.destinationPath would
|
||||
substitute to the empty string and EVERY WAL archive would fail at runtime. Probing the historical
|
||||
default s3://cnpg so the round trip is still proven — but SET THIS VALUE."
|
||||
cpath="s3://cnpg"
|
||||
fi
|
||||
case "${cpath}" in s3://*) ;; *) die "CNPG_BACKUP_PATH='${cpath}' is not an s3:// URL." ;; esac
|
||||
rest="${cpath#s3://}"; cbucket="${rest%%/*}"
|
||||
if [ "${rest}" = "${cbucket}" ]; then cprefix=""; else cprefix="${rest#*/}"; fi
|
||||
cprefix="${cprefix%/}"
|
||||
[ -n "${cbucket}" ] || die "CNPG_BACKUP_PATH='${cpath}' names no bucket."
|
||||
|
||||
s3_safe_token velero-bucket "${vbucket}"; s3_safe_token VELERO_BACKUP_PREFIX "${vprefix}"
|
||||
s3_safe_token cnpg-bucket "${cbucket}"; s3_safe_token CNPG_BACKUP_PATH-prefix "${cprefix}"
|
||||
|
||||
say "what is about to be proven"
|
||||
info "endpoint ${ep}"
|
||||
info "region ${region}"
|
||||
info "velero bucket '${vbucket}' prefix '${vprefix:-<bucket root>}' (authored bucket + VELERO_BACKUP_PREFIX)"
|
||||
info "cnpg bucket '${cbucket}' prefix '${cprefix:-<bucket root>}' (from CNPG_BACKUP_PATH)"
|
||||
info "scratch key <prefix>/.s3check/<pod>.probe — written, read back, byte-compared, DELETED"
|
||||
|
||||
local ran=0 keys
|
||||
# (a) Velero's own credential, from its own namespace, mounted the way the aws plugin reads it.
|
||||
if kubectl get ns velero >/dev/null 2>&1 \
|
||||
&& kubectl -n velero get secret velero-cloud-credentials >/dev/null 2>&1; then
|
||||
# List KEY NAMES only. Never pipe or print a value (a careless redaction has leaked a password
|
||||
# into a transcript on this project already). go-template, not jsonpath: kubectl's jsonpath
|
||||
# cannot range over a map's keys (`range $k,$v :=` is rejected, U+002C) and returns EMPTY, which
|
||||
# silently looked like "the key is missing".
|
||||
keys="$(kubectl -n velero get secret velero-cloud-credentials \
|
||||
-o go-template='{{range $k,$v := .data}}{{$k}}{{"\n"}}{{end}}' 2>/dev/null || true)"
|
||||
printf '%s\n' "${keys}" | grep -qx 'cloud' \
|
||||
|| die "Secret velero/velero-cloud-credentials carries no 'cloud' key (keys: $(printf '%s' "${keys}" | tr '\n' ' ')).
|
||||
velero-plugin-for-aws reads exactly that key as an AWS INI file; without it every backup fails
|
||||
at RUNTIME while the HelmRelease reports Ready."
|
||||
s3_probe velero file velero-cloud-credentials "${vbucket}:${vprefix}" "${ep}" "${region}"
|
||||
ran=$((ran+1))
|
||||
else
|
||||
warn "no Secret velero/velero-cloud-credentials on this cluster — Velero's own credential could NOT
|
||||
be proven. That is expected before the velero tier has ever reconciled here; it is not a pass."
|
||||
fi
|
||||
|
||||
# (b) A CNPG/barman credential, discovered rather than hardcoded (R-ENV-02: no per-environment list
|
||||
# of app names in this script). Any one of them proves the key + the cnpg bucket + the discrete
|
||||
# ACCESS_KEY_ID/ACCESS_SECRET_KEY shape; they are all kept in sync by design.
|
||||
local pick cns csec
|
||||
pick="$(kubectl get secret -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}' 2>/dev/null \
|
||||
| grep -- '-db-backup-credentials$' | head -n 1 || true)"
|
||||
if [ -n "${pick}" ]; then
|
||||
cns="${pick%%/*}"; csec="${pick#*/}"
|
||||
s3_probe "${cns}" env "${csec}" "${cbucket}:${cprefix}" "${ep}" "${region}"
|
||||
ran=$((ran+1))
|
||||
else
|
||||
warn "no *-db-backup-credentials Secret found in any namespace — the CNPG/barman credential could
|
||||
NOT be proven."
|
||||
fi
|
||||
|
||||
[ "${ran}" -gt 0 ] || die "NO backup credential Secret exists on this cluster, so NOTHING could be
|
||||
proven about ${ep}. That is a FAILURE, not a skip: a green s3check must never mean 'I found
|
||||
nothing to check'."
|
||||
|
||||
say "S3 ROUND TRIP PROVEN from inside the cluster: ${ep} (${ran} credential path(s))"
|
||||
info "STILL UNPROVEN by this step: Velero's kopia uploader and CNPG barman-cloud against this target,"
|
||||
info "and a RESTORE. A backup that has never been restored is not a backup — see infra/BACKUP-R2.md."
|
||||
}
|
||||
|
||||
step_resume() {
|
||||
say "RESUME TIERS (dependency order)"
|
||||
local n s3; s3="$(cvar S3_BACKUP_ENDPOINT)"
|
||||
for n in "${TIER_ORDER[@]}"; do
|
||||
if [ "$(k_susp "$n")" = "true" ]; then
|
||||
info "resuming $n"; flux resume kustomization "$n" --timeout=1s >/dev/null 2>&1 || true
|
||||
else
|
||||
info "$n already unsuspended"
|
||||
fi
|
||||
wait_ready "$n" 900 || { warn "stopping at $n — a partial, understood bring-up beats a forced green."; return 1; }
|
||||
done
|
||||
|
||||
# The placeholder test MUST be substring-based. It was `TBD-*` (a PREFIX match) while the placeholder
|
||||
# was `TBD-PROD-…`; the R2 move changed it to `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`,
|
||||
# which does not START with TBD- — so this case arm silently stopped matching and `resume` would have
|
||||
# RESUMED velero onto a placeholder hostname on prod. That is the precise failure this guard exists
|
||||
# to prevent, reintroduced by a value change in another file. s3_is_placeholder matches anywhere.
|
||||
if s3_is_placeholder "${s3}"; then
|
||||
warn "velero left SUSPENDED: S3_BACKUP_ENDPOINT='${s3}' is not a usable endpoint.
|
||||
Resuming it would create a BackupStorageLocation that can never reach a bucket, i.e. a tier
|
||||
that reports Ready-ish while backups silently do not exist. Set the real endpoint first, prove
|
||||
it with \`s3check\`, and only then resume (infra/BACKUP-R2.md)."
|
||||
else
|
||||
info "resuming velero"; flux resume kustomization velero --timeout=1s >/dev/null 2>&1 || true
|
||||
wait_ready velero 900 || warn "velero did not converge"
|
||||
fi
|
||||
}
|
||||
|
||||
step_verify() {
|
||||
say "VERIFY"
|
||||
local base; base="$(cvar BASE_DOMAIN)"
|
||||
info "BASE_DOMAIN = ${base}"
|
||||
say "kustomizations"
|
||||
kubectl get kustomization -A \
|
||||
-o custom-columns=NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status --no-headers
|
||||
say "certificates (must be Ready before any hostname is trusted)"
|
||||
kubectl get certificate -A --no-headers 2>/dev/null || info "(none yet)"
|
||||
say "pods not Running/Completed"
|
||||
kubectl get pods -A --no-headers | awk '$4!="Running" && $4!="Completed"' || true
|
||||
info "total pods: $(kubectl get pods -A --no-headers | wc -l)"
|
||||
say "gateway / LoadBalancer VIPs"
|
||||
kubectl get svc -A --field-selector spec.type=LoadBalancer --no-headers 2>/dev/null || true
|
||||
say "published DNS for this cluster's own domain"
|
||||
local owner; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
|
||||
dns_snapshot "${SNAP_DIR}/dns-verify.txt"
|
||||
info "records under ${base}: $(grep -c "${base}" "${SNAP_DIR}/dns-verify.txt" || true)"
|
||||
|
||||
# Every TXT registry entry this cluster publishes must carry OUR owner id. One reading another
|
||||
# cluster's id means we have claimed its records and `policy: sync` will delete them next cycle.
|
||||
local foreign
|
||||
foreign="$(grep "${base}" "${SNAP_DIR}/dns-verify.txt" | grep 'heritage=external-dns' \
|
||||
| grep -v "external-dns/owner=${owner}" || true)"
|
||||
if [ -n "${foreign}" ]; then
|
||||
warn "TXT entries under ${base} NOT owned by '${owner}' — this cluster has claimed records it does not own:"
|
||||
printf '%s\n' "${foreign}" | sed 's/^/ /'
|
||||
else
|
||||
info "every TXT registry entry under ${base} reads owner=${owner}: OK"
|
||||
fi
|
||||
|
||||
# THE REGRESSION CHECK. Additions under our own BASE_DOMAIN are expected. ANY change to a name
|
||||
# outside it means we have touched the other cluster's zone — the 2026-09-04 incident.
|
||||
if [ -f "${SNAP_DIR}/dns-before.txt" ]; then
|
||||
say "regression check against the pre-standup baseline"
|
||||
local outside
|
||||
outside="$(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" \
|
||||
| grep -E '^[<>]' | grep -v "${base}" || true)"
|
||||
if [ -n "${outside}" ]; then
|
||||
warn "RECORDS OUTSIDE ${base} CHANGED — investigate before doing anything else:"
|
||||
printf '%s\n' "${outside}" | sed 's/^/ /'
|
||||
else
|
||||
info "no record outside ${base} was added, changed or removed: OK"
|
||||
info "added under ${base}: $(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" | grep -c '^>' || true)"
|
||||
fi
|
||||
else
|
||||
warn "no ${SNAP_DIR}/dns-before.txt baseline — cannot prove the other cluster's records are untouched.
|
||||
Take one BEFORE the next standup so this check can run."
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================ dispatch
|
||||
for s in "${STEPS[@]}"; do
|
||||
case "$s" in
|
||||
preflight) step_preflight ;;
|
||||
s3check) step_s3check ;;
|
||||
crds) step_crds ;;
|
||||
unwedge) step_unwedge ;;
|
||||
externaldns) step_externaldns ;;
|
||||
resume) step_resume ;;
|
||||
verify) step_verify ;;
|
||||
all) step_preflight; step_crds; step_unwedge; step_externaldns; step_resume; step_verify ;;
|
||||
*) die "unknown step '$s' (preflight crds unwedge externaldns s3check resume verify all)" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
say "done: ${STEPS[*]}"
|
||||
@@ -180,34 +180,6 @@ dependencies: [] # default-deny: reaches only DNS; git clients re
|
||||
ingressRoutes:
|
||||
- host: gitea.${BASE_DOMAIN}
|
||||
port: 3000 # Gitea HTTP web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
# THIS IS WHAT KEEPS PROD'S GITOPS OUT OF A DEADLOCK — it is not an incidental allowance. Remove it
|
||||
# and prod cannot reconcile AT ALL, in a way that is very hard to diagnose from cold.
|
||||
#
|
||||
# Prod's Flux source is THIS SERVICE, not a public hostname:
|
||||
# clusters/prod/kustomization.yaml -> GitRepository.spec.url =
|
||||
# http://gitea.gitea.svc.cluster.local:3000/Home-Network/home-network-k3s-impl.git
|
||||
# chosen deliberately so prod's GitOps depends on NEITHER public DNS, Cloudflare, prod's own
|
||||
# Gateway, NOR ${BASE_DOMAIN} — a future domain change cannot sever prod from its own source. That
|
||||
# file documents the other half of this pair at length; the two must be read together.
|
||||
#
|
||||
# The only other ingress this entry renders is gitea-allow-gateway-ingress, which admits
|
||||
# `fromEntities: [ingress]` — the shared Gateway's Envoy proxy — and nothing else. Flux's
|
||||
# source-controller dials the ClusterIP DIRECTLY, so under the namespace default-deny (R-NET-01) its
|
||||
# clone was DROPPED (`wget` from source-controller timed out; srcReady=False, "unable to list
|
||||
# remote"). The deadlock that made this urgent: the HTTPRoute that would have served the new public
|
||||
# name after the 2026-09-13 apex flip lives in a DOWNSTREAM tier, which cannot apply while the
|
||||
# source is down — source down -> routes never rename -> source stays down. It was broken on
|
||||
# 2026-09-13 with a live `kubectl apply` of a hand-written gitea-allow-flux-ingress, i.e. with a
|
||||
# cluster object that existed OUTSIDE git. This declaration renders that object, so one
|
||||
# `kubectl delete` or one cluster rebuild can no longer put prod back in the deadlock.
|
||||
inClusterClients:
|
||||
# `name` is pinned to `flux` (rather than defaulting to the namespace) so the rendered policy name
|
||||
# equals the live object's, `gitea-allow-flux-ingress`, and Flux ADOPTS it in place: no
|
||||
# near-duplicate policy beside it, and no window in which prod's own source has no allow.
|
||||
- name: flux
|
||||
namespace: flux-system # matched by NAMESPACE alone — the flux controllers' pod labels belong
|
||||
# to upstream's manifests, not to this platform's contract
|
||||
port: 3000 # ONLY the Gitea HTTP port — the same single port the Gateway allow opens
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02)
|
||||
groups:
|
||||
|
||||
@@ -23,17 +23,6 @@ storage:
|
||||
- name: config
|
||||
size: 2Gi
|
||||
class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: jackett.${BASE_DOMAIN}
|
||||
@@ -42,15 +31,6 @@ auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in an authorized group (R-IDP-06)
|
||||
# platform-admins is a SUPERSET, not a widening. --allowed-group is repeatable and OR-ed, so
|
||||
# listing both means "media consumers OR platform operators". Without it the media UIs are
|
||||
# reachable by NOBODY the moment media-users is empty: on prod the realm's only user is the
|
||||
# operator, who is in platform-admins only, so jackett/qbittorrent/radarr/sonarr answered an
|
||||
# immediate 403 at the oauth2-proxy callback (no login prompt, because the Keycloak SSO session
|
||||
# already existed) while pihole/crafty — which already gate on platform-admins — worked.
|
||||
# This grants no real privilege: platform-admins already maps to cluster-admin via
|
||||
# k8s-oidc-rbac.yaml (oidc:platform-admins), so a member can already exec into these pods.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=0.22.0" # stay on the Jackett 0.22.x stable line
|
||||
|
||||
@@ -223,44 +223,6 @@ storage:
|
||||
mountPath: /var/www/html # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02).
|
||||
# The whole html dir is persisted so config.php + user data survive a
|
||||
# rebuild (the default /data/<name> would strand them on ephemeral rootfs).
|
||||
# THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND
|
||||
# NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision.
|
||||
#
|
||||
# WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed):
|
||||
# /mnt/Storage/nextcloud/nextcloud/config 103 M
|
||||
# /mnt/Storage/nextcloud/nextcloud/config_old 3.4 G
|
||||
# /mnt/Storage/nextcloud/nextcloud/config_backup 267 G <- 99% of the dataset
|
||||
# There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup
|
||||
# blob — an ARCHIVE, not a live user-data tree.
|
||||
#
|
||||
# WHY READ-ONLY. Two independent reasons, either sufficient:
|
||||
# 1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and
|
||||
# 192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two
|
||||
# Nextcloud instances writing one instance's config directory, each with its own database and
|
||||
# therefore its own oc_filecache, is a corruption scenario with no clean recovery.
|
||||
# 2. Nothing here needs writing. An archive is read.
|
||||
# "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a
|
||||
# MIGRATION (read this data in, then decommission the old instance), that is a different and
|
||||
# larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the
|
||||
# export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount.
|
||||
#
|
||||
# WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own
|
||||
# state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does
|
||||
# not own. A neutral path keeps the two completely separate. To expose it to users, add it as a
|
||||
# Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision.
|
||||
#
|
||||
# subPath skips the redundant single-child directory: the dataset root holds exactly one entry,
|
||||
# `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top.
|
||||
#
|
||||
# UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data
|
||||
# (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.)
|
||||
- name: legacy-archive
|
||||
class: bulk
|
||||
export: /mnt/Storage/nextcloud # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER}
|
||||
subPath: nextcloud
|
||||
mountPath: /mnt/legacy-nextcloud
|
||||
readOnly: true
|
||||
size: 270Gi # ADVISORY ONLY on a static NFS PV — the dataset's used size.
|
||||
dependencies:
|
||||
- to: nextcloud-db # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02).
|
||||
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
|
||||
|
||||
+15
-30
@@ -4,11 +4,21 @@
|
||||
# auth: native (Plex accounts, NOT Keycloak — docs/13 + docs/15 documented exception).
|
||||
# Validates against schemas/servicedefinition.schema.json.
|
||||
#
|
||||
# BULK MEDIA — NOW ATTACHED (docs/04 bulk tier; the long-standing OQ-2 prerequisite, resolved for
|
||||
# the STATIC case). The library is 8.33 T of ZFS on the Proxmox host at /mnt/Storage/plex, and it
|
||||
# already exists: nothing is provisioned, an existing dataset is mounted. See
|
||||
# docs/BULK-NFS-STORAGE.md for the layout, the export change and the uid analysis. (The bulk-tier
|
||||
# requirement ID is deliberately still not typed here — see that document's "gate" note.)
|
||||
# BULK MEDIA (spec bulk-tier requirement, docs/04) — OPEN PREREQUISITE, intentionally NOT declared
|
||||
# here yet (the requirement ID is cited in the WS-APPS report, not as a token here: this change does
|
||||
# NOT exercise the bulk tier, so it must not be selected as a "touched" requirement whose in-cluster
|
||||
# conformance test is still a Phase-1 gap).
|
||||
# The Plex library belongs on the `bulk` tier (docs/04 / docs/13 cross-cutting note: metadata DB
|
||||
# on replicated Longhorn, large library on bulk). The `bulk` StorageClass exists
|
||||
# (platform/longhorn/storageclasses.yaml) and the schema+renderer both support `class: bulk`, but
|
||||
# that class is a documented STUB: OQ-2 (external NFS/SMB backing) is UNRESOLVED and the NFS CSI
|
||||
# driver is not installed, so a `bulk` PVC will not bind and would leave the Plex pod Pending.
|
||||
# To onboard the running Plex shell now (compute + config + public exposure), the media volume is
|
||||
# omitted. Once OQ-2 is resolved and `bulk` provisions, attach the library with a one-line add:
|
||||
# storage:
|
||||
# - name: media
|
||||
# class: bulk # external mount; size omitted (sized outside the cluster)
|
||||
# then re-render. See the WS-APPS report for the full bulk-media finding.
|
||||
name: plex
|
||||
kind: container
|
||||
source:
|
||||
@@ -60,31 +70,6 @@ storage:
|
||||
# UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token
|
||||
# (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`,
|
||||
# so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
|
||||
# THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies,
|
||||
# /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`:
|
||||
# downloads/ movies/ photos/ tv/).
|
||||
#
|
||||
# WHY THE ROOT AND NOT THREE subPath MOUNTS. Nothing is gained by narrowing: the mount is `ro`,
|
||||
# so no subPath could remove a privilege that matters, while three mounts would triple the NFS
|
||||
# mounts per pod and mean a manifest change every time a top-level library is added.
|
||||
#
|
||||
# WHY READ-ONLY, when the export is `rw` and the other consumers write. Plex is a SERVER: it
|
||||
# indexes and streams. Every write on that tree belongs to Sonarr and Radarr. Marking it here
|
||||
# costs nothing and removes Plex from the set of things that can damage 8.33 T. Enforced by the
|
||||
# PV's nfs.readOnly (a real `ro` mount), not only by the volumeMount flag.
|
||||
#
|
||||
# UID: no change needed. The tree is 0755 owned 1000:3000 throughout, so it is world-readable and
|
||||
# world-traversable, and PLEX_UID/PLEX_GID above already make the process uid 1000 in any case.
|
||||
# (fsGroup would have been a no-op here regardless — the in-tree NFS plugin does no ownership
|
||||
# management. That is the trap the arrs' PUID/PGID exists to avoid; Plex never hits it.)
|
||||
- name: media
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
|
||||
mountPath: /media
|
||||
readOnly: true
|
||||
size: 9Ti # ADVISORY ONLY (nothing enforces a static NFS PV's capacity): the
|
||||
# dataset's 8.33 T used, rounded up, so `kubectl get pv` reports
|
||||
# something true. The real limit is the ZFS pool.
|
||||
dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: plex.${BASE_DOMAIN}
|
||||
|
||||
+14
-127
@@ -4,18 +4,19 @@
|
||||
# internal download client | egress VPN + kill-switch | config on Longhorn | auth forward-auth
|
||||
# (bare web UI, no native OIDC).
|
||||
#
|
||||
# BULK DOWNLOADS TIER — DELIVERED 2026-09-12 (the deferral below is CLOSED; kept for the record).
|
||||
# This entry used to ship config-only, with a comment promising to attach the downloads volume "once
|
||||
# OQ-2 is resolved". OQ-2 was resolved for STATIC pre-existing datasets (schema 1.5.0 / renderer
|
||||
# `class: bulk` + `export` -> a static, pre-bound NFS PV), and plex/sonarr/radarr/nextcloud all took
|
||||
# their media mounts — but qBittorrent's was never added, so the DOWNLOAD CLIENT was the one member
|
||||
# of the chain with no media volume at all. Measured on prod 2026-09-12:
|
||||
# Session\DefaultSavePath=/downloads/ Session\TempPath=/downloads/incomplete/
|
||||
# Downloads\SavePath=/downloads/ `ls /downloads` -> No such file or directory
|
||||
# i.e. the app was configured to write a path that DID NOT EXIST, in a container with no volume there.
|
||||
# Anything it did manage to create would have been invisible to sonarr/radarr, lost on every pod
|
||||
# recreation, and filling the NODE's ephemeral disk in the meantime. The `downloads` entry under
|
||||
# storage: below closes that. See docs/BULK-NFS-STORAGE.md and the bulk-tier L0 conformance test.
|
||||
# BULK DOWNLOADS TIER — OPEN PREREQUISITE, intentionally NOT declared here (mirrors plex + sonarr/radarr):
|
||||
# docs/13 row 13 places qBittorrent's downloads/staging volume on the `bulk` tier (large, non-3x-
|
||||
# replicated media handoff to the arr stack). The `bulk` StorageClass exists
|
||||
# (platform/longhorn/storageclasses.yaml) and both the schema and renderer support `class: bulk`, BUT
|
||||
# it is a NON-PROVISIONING STUB pending OQ-2 (its NFS/SMB backing is REPLACE-ME and the NFS CSI driver
|
||||
# is not installed) — a `bulk` PVC would never bind and would leave the pod Pending. Declaring it now
|
||||
# would also pull the bulk-tier requirement into the change's "touched" set, whose in-cluster
|
||||
# conformance test is not yet scaffolded, blocking the merge gate. So the downloads volume is DEFERRED
|
||||
# and this onboarding ships config-only. Once OQ-2 is resolved and `bulk` provisions, attach downloads
|
||||
# with a one-line add under storage:
|
||||
# - name: downloads
|
||||
# class: bulk # external mount; size omitted (sized outside the cluster)
|
||||
# then re-render. See the WS-APPS report for the full bulk-downloads finding.
|
||||
#
|
||||
# VPN-class (docs/13 row 13; docs/03 "VPN-class set"): ALL of this pod's public-internet egress
|
||||
# (peer/tracker/DHT traffic) MUST leave via the node PIA tunnel, never the WAN. `egress: vpn` renders
|
||||
@@ -31,122 +32,13 @@ name: qbittorrent
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING FOR THE BULK MOUNT, not cosmetic — exactly as on sonarr/radarr. Measured on prod
|
||||
# 2026-09-12 BEFORE this change: `id` inside the container reported the qbittorrent-nox process as
|
||||
# uid=911 gid=911, and /mnt/Storage/plex/downloads is 0755 owned 1000:1000. 0755 grants write to the
|
||||
# OWNER ONLY, so uid 911 would have mounted the export cleanly and then failed every write with
|
||||
# EACCES — the "mounts fine, cannot write" failure the sonarr entry warns about at length.
|
||||
env:
|
||||
- name: PUID
|
||||
value: "1000" # owner uid of /mnt/Storage/plex/downloads (and of every dir under the export)
|
||||
- name: PGID
|
||||
value: "3000" # the export's dominant group, matching sonarr/radarr: files this writes stay
|
||||
# readable to the EXISTING non-k3s consumers of the same dataset, and land
|
||||
# group-owned the same way the arrs' own writes do
|
||||
podSecurityContext:
|
||||
# For the LONGHORN config PVC only, and for the same reason radarr/sonarr set it: fsGroup is a NO-OP
|
||||
# on the NFS volume (the in-tree plugin does no ownership management), but on a Longhorn filesystem
|
||||
# the kubelet applies it, so /config becomes group-writable by the now-uid-1000 process without
|
||||
# depending on the image chowning it. (This image DOES chown: init-qbittorrent-config runs
|
||||
# `lsiown -R abc:abc /config`. Belt and braces — the live /config was 911:911 before this change,
|
||||
# and losing write access to it would lose the WebUI password hash and the session state.)
|
||||
fsGroup: 3000
|
||||
exposure: internal # LAN-only web UI at qbittorrent.${BASE_DOMAIN} (docs/13 row 13; R-DNS/R-NET)
|
||||
egress: vpn # ALL public egress via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE DOWNLOADS AREA — ONE mount of the SAME export sonarr/radarr/plex already consume
|
||||
# (192.168.2.102:/mnt/Storage/plex), narrowed by subPath to the `downloads` subtree and presented at
|
||||
# /downloads, which is precisely where this app is ALREADY configured to write.
|
||||
#
|
||||
# WHY subPath: downloads AND NOT THE EXPORT ROOT AT /media (the shape sonarr/radarr use):
|
||||
# * LEAST PRIVILEGE, AND IT COSTS NOTHING HERE. A download client has no business writing the
|
||||
# LIBRARY. The arrs need the root mount because THEY hardlink downloads/ -> tv/ or movies/ and
|
||||
# link() is a same-MOUNT operation (EXDEV otherwise) — their smallest common ancestor genuinely
|
||||
# IS the export root. qBittorrent performs no such cross-directory link: it writes into the
|
||||
# downloads tree and nothing else, so narrowing to that subtree removes movies/, tv/ and photos/
|
||||
# from this pod's mount namespace without removing any operation it actually performs.
|
||||
# * HARDLINKS STILL WORK, because they are not done here. The arrs still see ONE mount containing
|
||||
# both /media/downloads and /media/tv, so their link() stays intra-mount. Proven live 2026-09-12:
|
||||
# a file created from this pod under /downloads was hardlinked from the sonarr pod between
|
||||
# /media/downloads/... and /media/tv/... with st_nlink 1 -> 2 and no EXDEV.
|
||||
# * IT IS ONE MOUNT, so the L0 test `bulk-tier-distinct-from-replicated` assertion (6) — no
|
||||
# workload mounts one export twice — holds. That assertion is about TWO mounts inside ONE pod;
|
||||
# a single subPath mount is not the EXDEV trap, two are.
|
||||
# * The narrowing is a CLIENT/kubelet guarantee, stated honestly: the NFS server still exports the
|
||||
# whole dataset to this node, so the containment is the mount namespace, not the server's ACLs —
|
||||
# the same class of caveat docs/BULK-NFS-STORAGE.md §5 records for plex's readOnly.
|
||||
#
|
||||
# NOT readOnly — this is the one workload whose whole job is WRITING here.
|
||||
#
|
||||
# KNOWN, DELIBERATE SIDE EFFECT ON SHARED STATE: the image's init-qbittorrent-config runs
|
||||
# `lsiown abc:abc /downloads` (non-recursive) whenever /downloads is a mount, so starting this pod
|
||||
# chgrps /mnt/Storage/plex/downloads from 1000:1000 to 1000:3000 on the SERVER. Benign and verified:
|
||||
# the directory is mode 0755, whose group bits (r-x) are identical to its other bits, so no consumer
|
||||
# — k3s or legacy — gains or loses any access. Recorded because it is a write to state shared with
|
||||
# non-cluster consumers, which should never be a surprise.
|
||||
#
|
||||
# PATH AGREEMENT WITH THE ARRS IS STILL *NOT* CLOSED — see DECISIONS-LOG 2026-09-12 "Remote Path
|
||||
# Mappings". This pod calls the finished file /downloads/<name>; sonarr/radarr call the same bytes
|
||||
# /media/downloads/<name>. Same inode, different absolute path, so Completed Download Handling needs
|
||||
# ONE Remote Path Mapping per arr (remote host `qbittorrent.qbittorrent.svc.cluster.local`, remote
|
||||
# path /downloads/, local path /media/downloads/).
|
||||
#
|
||||
# THOSE MAPPINGS ARE NOW CONFIGURED AND VERIFIED ON PROD (2026-09-12), together with the qBittorrent
|
||||
# download client itself — which did not exist in EITHER arr until then (both
|
||||
# `GET /api/v3/downloadclient` returned `[]`, so completed-download handling had never been wired up
|
||||
# at all). `downloadclient/testall` reports isValid=true in both, and Sonarr's queue was observed
|
||||
# reporting outputPath=/downloads/... with the mapping deleted and /media/downloads/... with it
|
||||
# present — same torrent, same client.
|
||||
#
|
||||
# BUT THEY LIVE IN THE ARRS' OWN DATABASES (app state on their config PVCs), NOT IN GIT: a rebuild
|
||||
# from git alone does NOT restore them and no gate here notices their absence. The alternative —
|
||||
# repointing qBittorrent's SavePath at /media/downloads and mounting this subPath there instead —
|
||||
# buys path agreement at the cost of editing THIS app's config PVC, which is also app state; it was
|
||||
# not chosen because it changes a running download client's save path, which the deferral above
|
||||
# already shows is easy to get wrong. It is nonetheless the RECOMMENDED end state (it makes the
|
||||
# mappings unnecessary and puts the mountPath half of the fix in git) and is awaiting an operator
|
||||
# decision, not a technical blocker.
|
||||
#
|
||||
# RELATED OPEN BLOCKER: this app has no durable WebUI credential (no WebUI\Username /
|
||||
# WebUI\Password_PBKDF2 in qBittorrent.conf), so v5 issues a NEW temporary `admin` password on every
|
||||
# start. The arrs' saved download-client password therefore goes stale on every pod restart. See the
|
||||
# same DECISIONS-LOG entry for the two fixes, neither of which was taken unilaterally.
|
||||
- name: downloads
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER} — the SAME export
|
||||
# plex (ro) and sonarr/radarr (rw) mount; one dataset, one set of inodes
|
||||
subPath: downloads # the handoff subtree ONLY: no movies/, no tv/, no photos/
|
||||
mountPath: /downloads # matches the app's EXISTING Session\DefaultSavePath / TempPath, so no
|
||||
# change to config-PVC app state is needed to make this take effect
|
||||
size: 9Ti # ADVISORY ONLY on a static NFS PV (the ZFS pool is the real limit): the
|
||||
# dataset's 8.4 T used rounded up, stated the same way plex/sonarr/radarr
|
||||
# state it so `kubectl get pv` reports something true
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02).
|
||||
# STILL EMPTY WITH THE NFS MOUNT ABOVE, AND THAT IS CORRECT, NOT AN
|
||||
# OVERSIGHT. An NFS PersistentVolume is mounted by the NODE KERNEL in the
|
||||
# HOST netns, not by this pod: there is no pod-side socket to 2049, so no
|
||||
# CiliumNetworkPolicy can permit or deny it and none is needed. Verified by
|
||||
# construction — sonarr and radarr have mounted this same export read-write
|
||||
# for weeks and neither has any CNP naming the NFS server or port 2049
|
||||
# (grep apps/wave-1/sonarr.yaml: none). The SAME fact is what keeps the VPN
|
||||
# egress gateway off this traffic (its selector is a podSelector), and
|
||||
# 192.168.2.102 is inside the 192.168.0.0/16 excludedCIDR besides — see
|
||||
# platform/vpn-egress/egress-steering.yaml, "BULK NFS".
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: qbittorrent.${BASE_DOMAIN}
|
||||
port: 8080 # qBittorrent WebUI; protected at the Gateway by forward-auth (docs/09, docs/15)
|
||||
@@ -154,11 +46,6 @@ auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in an authorized group (R-IDP-06)
|
||||
# See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
|
||||
# platform-admins here is a superset, not a widening (that group already carries cluster-admin
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=5.0.0 <6.0.0" # stay on the qBittorrent 5.x stable line
|
||||
|
||||
+5
-82
@@ -9,23 +9,6 @@ name: radarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
env:
|
||||
- name: PUID
|
||||
value: "1000" # owner uid of every directory under /mnt/Storage/plex
|
||||
- name: PGID
|
||||
value: "3000" # dominant group there; keeps files this writes readable to the
|
||||
# EXISTING non-k3s consumers of the same export, which stay live
|
||||
- name: TZ
|
||||
value: Etc/UTC # same safe default plex uses; the arrs schedule by local time
|
||||
podSecurityContext:
|
||||
# For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
|
||||
# does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
|
||||
# what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
|
||||
# /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
|
||||
fsGroup: 3000
|
||||
exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
@@ -33,66 +16,11 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
|
||||
# /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
|
||||
#
|
||||
# *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Radarr imports by HARDLINKING or
|
||||
# atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
|
||||
# and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
|
||||
# /mnt/Storage/plex/movies/ the library
|
||||
# /mnt/Storage/plex/downloads/complete/radarr/ where finished releases land
|
||||
# — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
|
||||
# them as two subPath volumes (subPath: movies at /media/movies, subPath: downloads elsewhere) creates
|
||||
# TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Radarr then
|
||||
# silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
|
||||
# seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
|
||||
# is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
|
||||
# requirement, not a shortcut. (The cost is stated plainly: Radarr can also write /media/tv,
|
||||
# which is sonarr's. Narrowing that would cost hardlinks, which is a worse trade.)
|
||||
#
|
||||
# *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
|
||||
# renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
|
||||
# config volume entirely; the pod would come up with its app database shadowed by an 8 T media
|
||||
# tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
|
||||
# the same export, so when qbittorrent is eventually given this mount too, the arrs and the
|
||||
# download client will already agree on absolute paths — which is the other precondition for
|
||||
# hardlinked imports (otherwise every import needs a Remote Path Mapping).
|
||||
#
|
||||
# SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
|
||||
# is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
|
||||
# with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
|
||||
# Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
|
||||
# documents having already been bitten by. Fixing it means starting from an empty config volume, so
|
||||
# it belongs in its own change with the operator present.
|
||||
#
|
||||
# UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
|
||||
# + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
|
||||
# the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
|
||||
# write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
|
||||
# every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
|
||||
#
|
||||
# AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
|
||||
# ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
|
||||
# lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
|
||||
# that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
|
||||
# the image re-chowning /config for itself.
|
||||
- name: media
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
|
||||
mountPath: /media # => /media/movies (Root Folder) and /media/downloads (import source)
|
||||
size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
|
||||
# rounded up, so `kubectl get pv` reports something true.
|
||||
# NOTE (bulk-media PREREQUISITE): docs/13 row 3 places the media library / downloads volume on the
|
||||
# `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
|
||||
# but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
|
||||
# volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
|
||||
# therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Radarr deployable today.
|
||||
dependencies:
|
||||
- to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
|
||||
port: 8080
|
||||
@@ -107,11 +35,6 @@ auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in the media group (R-IDP-06)
|
||||
# See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
|
||||
# platform-admins here is a superset, not a widening (that group already carries cluster-admin
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=5.0.0 <6.0.0" # stay on the Radarr v5 stable line
|
||||
|
||||
+9
-374
@@ -4,8 +4,7 @@
|
||||
Consumes ONE ServiceDefinition (schemas/servicedefinition.schema.json, catalog/*.yaml)
|
||||
and emits the concrete manifest set the renderer contract specifies
|
||||
(schemas/renderer-contract.md): Namespace + default-deny baseline, Deployment/Service or
|
||||
KubeVirt VirtualMachine + CDI DataVolume, PVCs, per-dependency and per-in-cluster-client
|
||||
CiliumNetworkPolicy allows,
|
||||
KubeVirt VirtualMachine + CDI DataVolume, PVCs, per-dependency CiliumNetworkPolicy allows,
|
||||
egress-class steering label, HTTPRoute(s), ServiceMonitor/alerts, Flux image-automation,
|
||||
and the auth-mode / backup wiring hints.
|
||||
|
||||
@@ -54,30 +53,6 @@ LBL = "homelab.keithjackson.dev"
|
||||
# Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
|
||||
# and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10).
|
||||
ZONE = "${BASE_DOMAIN}"
|
||||
# The bulk (external NFS) tier's SERVER, emitted as a Flux post-build TOKEN for exactly the same
|
||||
# reason as ZONE: dev and prod reach the same Proxmox ZFS host by DIFFERENT addresses (dev's nodes
|
||||
# are on the host's own LAN and dial it directly; prod's nodes are on the far side of pfSense and
|
||||
# must be given whichever address the still-pending routing decision produces). Hardcoding either
|
||||
# one authors a manifest that is silently wrong in the other environment. Declared in
|
||||
# clusters/base/cluster-vars.yaml and patched by BOTH overlays; written BARE (it is a real knob,
|
||||
# not a shell expansion), so StrictPostBuildSubstitutions fails the tier if an overlay forgets it.
|
||||
BULK_NFS_SERVER = "${BULK_NFS_SERVER}"
|
||||
# Mount options for every bulk NFS volume. These are a PLATFORM decision, not a per-service one,
|
||||
# so they live here rather than on the ServiceDefinition surface:
|
||||
# nfsvers=4.1 NFSv4 carries mount+lock IN the protocol, so it needs ONE port (tcp/2049) and no
|
||||
# rpcbind/111, mountd or statd. That is what makes the prod path a single pfSense
|
||||
# rule instead of four, and it is why v4 is chosen over v3. The server advertises
|
||||
# +3 +4 +4.1 +4.2; 4.1 is the conservative floor every node kernel already speaks.
|
||||
# hard A `soft` mount turns a server hiccup into a short read / EIO handed to the
|
||||
# application, which for a media library and an archive is silent corruption.
|
||||
# `hard` blocks instead. The cost is explicit: an NFS outage HANGS I/O rather than
|
||||
# failing it, so an outage shows up as stuck pods, not as damaged files. Chosen
|
||||
# deliberately -- hung is recoverable, truncated is not.
|
||||
# noatime The datasets are relatime, so every read would otherwise generate a metadata
|
||||
# WRITE back to the pool. Plex walking an 8 T library must not do that.
|
||||
# timeo=600/retrans=2 the TCP defaults, stated explicitly so a future reader knows the retry
|
||||
# budget was considered rather than inherited.
|
||||
BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retrans=2"]
|
||||
GATEWAY_NAME = "shared"
|
||||
GATEWAY_NS = "gateway"
|
||||
KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label
|
||||
@@ -117,7 +92,6 @@ def apply_defaults(sd: dict) -> dict:
|
||||
s.setdefault("storage", [])
|
||||
s.setdefault("dependencies", [])
|
||||
s.setdefault("ingressRoutes", [])
|
||||
s.setdefault("inClusterClients", [])
|
||||
|
||||
# auth.mode default posture: forward-auth for any exposed UI, else none (internal, no
|
||||
# UI). An explicit auth block always wins (R-IDP-02).
|
||||
@@ -137,19 +111,10 @@ def apply_defaults(sd: dict) -> dict:
|
||||
obs.setdefault("slis", [])
|
||||
s["observability"] = obs
|
||||
|
||||
# storage entry class default is longhorn (replicated tier); readOnly default is False
|
||||
# (grant-nothing-extra is expressed by SETTING it, so the default stays back-compatible and
|
||||
# every pre-existing golden renders byte-identical).
|
||||
s["storage"] = [{**{"class": "longhorn", "readOnly": False}, **v} for v in s["storage"]]
|
||||
# storage entry class default is longhorn (replicated tier).
|
||||
s["storage"] = [{**{"class": "longhorn"}, **v} for v in s["storage"]]
|
||||
# dependency protocol default is tcp.
|
||||
s["dependencies"] = [{**{"protocol": "tcp"}, **d} for d in s["dependencies"]]
|
||||
# in-cluster client protocol default is tcp (as for a dependency); its `name` — which only
|
||||
# ever appears in the rendered POLICY name — defaults to the namespace it admits, so the
|
||||
# common case declares just `namespace` + `port` and reads like a dependency edge.
|
||||
s["inClusterClients"] = [
|
||||
{**{"protocol": "tcp", "name": c.get("namespace", "")}, **c}
|
||||
for c in s["inClusterClients"]
|
||||
]
|
||||
return s
|
||||
|
||||
|
||||
@@ -235,139 +200,18 @@ def _service_ports(s: dict) -> list:
|
||||
return [{"name": f"port-{p}", "port": p, "targetPort": p} for p in ports]
|
||||
|
||||
|
||||
def _bulk_access_mode(vol: dict) -> str:
|
||||
"""A bulk volume's access mode follows its read/write posture.
|
||||
|
||||
Every bulk volume is `...Many`: the SAME export is consumed by several namespaces at once
|
||||
(plex, sonarr and radarr all sit on /mnt/Storage/plex) and by several nodes at once, which is
|
||||
precisely what ReadWriteOnce cannot express. WHICH `Many` it is says whether this consumer
|
||||
writes -- so `kubectl get pv` reports the intent, not merely the topology.
|
||||
"""
|
||||
return "ReadOnlyMany" if vol.get("readOnly") else "ReadWriteMany"
|
||||
|
||||
|
||||
def r_bulk_pvs(s: dict) -> list:
|
||||
"""One STATIC PersistentVolume per `class: bulk` storage entry (docs/04 bulk tier).
|
||||
|
||||
WHY A PV AT ALL, RATHER THAN AN INLINE `nfs:` POD VOLUME
|
||||
-------------------------------------------------------
|
||||
An inline volume is fewer objects and it would work. It is rejected for four reasons, in
|
||||
descending order of what they would cost to discover later:
|
||||
|
||||
1. RECLAIM POLICY. `persistentVolumeReclaimPolicy: Retain` is a declared, reviewable
|
||||
property of a PV. There is no equivalent for an inline volume -- and the data behind
|
||||
these mounts is 8.33 TiB of media and a 267 GiB archive that ALSO has live non-k3s
|
||||
consumers. A statement in git that nothing in this cluster may reclaim it is worth an
|
||||
object.
|
||||
2. READ-ONLY LIVES IN ONE PLACE. `spec.nfs.readOnly: true` becomes a real `ro` mount option
|
||||
for every pod that ever binds this volume. The same intent expressed on an inline volume
|
||||
is one Deployment edit away from being dropped silently, with no diff anywhere near the
|
||||
storage. (It is still a CLIENT-side guarantee -- the export cannot distinguish plex from
|
||||
sonarr, since both may run on any node -- but it is one that lives in one reviewable
|
||||
place.)
|
||||
3. VISIBILITY. `kubectl get pv,pvc` is where an operator looks for "what external storage is
|
||||
attached, and who holds it". An inline volume is invisible there, and invisible to
|
||||
anything that reasons over PVCs.
|
||||
4. IT IS THE CONTRACT THE REPO ALREADY HAS. Every other storage entry is a PVC; the
|
||||
renderer, the Kyverno ServiceDefinition policy and the backup wiring all key on that. An
|
||||
inline volume would be a second, parallel storage model inside the same renderer.
|
||||
|
||||
CROSS-NAMESPACE. A PV binds 1:1 to a PVC, so "plex, sonarr and radarr share one export" is NOT
|
||||
one shared PV -- it is three PVs, one per namespace, each naming the same server+path. That is
|
||||
the correct model rather than a workaround: the sharing happens on the NFS server, which is
|
||||
built for concurrent clients, and each namespace keeps its own independently reviewable access
|
||||
mode (plex ReadOnlyMany, the arrs ReadWriteMany). One shared PV would have forced a single
|
||||
posture on all three.
|
||||
|
||||
BINDING IS PINNED FROM BOTH ENDS -- `claimRef` here and `volumeName` on the PVC. Either alone
|
||||
leaves a race in which some other PVC of matching size and mode grabs the volume first;
|
||||
together the binding is deterministic. `storageClassName: ""` (explicitly empty, NOT the `bulk`
|
||||
StorageClass) is what tells the binder this volume is statically provisioned: it suppresses the
|
||||
default-StorageClass annotation and guarantees no provisioner is ever asked to create anything.
|
||||
It also sidesteps a trap -- a StorageClass's `provisioner` is IMMUTABLE, so re-pointing the
|
||||
existing `bulk` class (whose stub names an NFS CSI driver that is not installed) at this data
|
||||
would make Flux fail forever on that one object.
|
||||
"""
|
||||
out = []
|
||||
for vol in s["storage"]:
|
||||
if vol.get("class") != "bulk":
|
||||
continue
|
||||
name = f"{s['name']}-{vol['name']}"
|
||||
# `export` and `size` are both structurally required of a bulk entry, and the JSON
|
||||
# Schema only enforces the first. A PersistentVolume with no `capacity` is rejected by
|
||||
# the API server, so a missing size must fail HERE with a sentence a human can act on
|
||||
# rather than as a bare KeyError three frames down.
|
||||
for field, why in (
|
||||
("export", "the absolute NFS export path this volume attaches"),
|
||||
("size", "the PV capacity (advisory, but the API requires the field)"),
|
||||
):
|
||||
if not vol.get(field):
|
||||
raise ValueError(
|
||||
f"{s['name']}: storage entry {vol['name']!r} has class 'bulk' but no "
|
||||
f"'{field}' — {why}. A bulk volume attaches storage that already exists "
|
||||
f"outside the cluster, so it cannot be rendered without it."
|
||||
)
|
||||
nfs = {"server": BULK_NFS_SERVER, "path": vol["export"]}
|
||||
if vol.get("readOnly"):
|
||||
nfs["readOnly"] = True
|
||||
spec = {
|
||||
"capacity": {"storage": vol["size"]},
|
||||
"accessModes": [_bulk_access_mode(vol)],
|
||||
# Retain: deleting the PVC must NEVER be able to reach the data. There is nothing to
|
||||
# "reclaim" here in any case -- the volume was not provisioned by the cluster.
|
||||
"persistentVolumeReclaimPolicy": "Retain",
|
||||
"storageClassName": "",
|
||||
"mountOptions": list(BULK_NFS_MOUNT_OPTIONS),
|
||||
"nfs": nfs,
|
||||
"claimRef": {"namespace": s["name"], "name": name},
|
||||
}
|
||||
out.append(
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "PersistentVolume",
|
||||
"metadata": {
|
||||
"name": name,
|
||||
"labels": {"app.kubernetes.io/name": s["name"]},
|
||||
},
|
||||
"spec": spec,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def r_pvcs(s: dict) -> list:
|
||||
"""One PVC per storage entry (R-STATE-01/02/04).
|
||||
|
||||
longhorn -> 3-replica replicated tier, dynamically provisioned, ReadWriteOnce.
|
||||
bulk -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName.
|
||||
longhorn -> 3-replica replicated tier; bulk -> external media tier (not 3x replicated).
|
||||
"""
|
||||
out = []
|
||||
for vol in s["storage"]:
|
||||
cls = vol.get("class", "longhorn")
|
||||
if cls == "bulk":
|
||||
# Mirror image of the PV: the same explicitly-empty class (so nothing dynamically
|
||||
# provisions), the same access mode, and volumeName completing the two-sided binding.
|
||||
out.append(
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "PersistentVolumeClaim",
|
||||
"metadata": {
|
||||
"name": f"{s['name']}-{vol['name']}",
|
||||
"namespace": s["name"],
|
||||
"labels": {"app.kubernetes.io/name": s["name"]},
|
||||
},
|
||||
"spec": {
|
||||
"accessModes": [_bulk_access_mode(vol)],
|
||||
"storageClassName": "",
|
||||
"volumeName": f"{s['name']}-{vol['name']}",
|
||||
"resources": {"requests": {"storage": vol["size"]}},
|
||||
},
|
||||
}
|
||||
)
|
||||
continue
|
||||
storage_class = "longhorn" if cls == "longhorn" else "bulk"
|
||||
spec = {
|
||||
"accessModes": ["ReadWriteOnce"],
|
||||
"storageClassName": "longhorn",
|
||||
"storageClassName": storage_class,
|
||||
}
|
||||
if vol.get("size"):
|
||||
spec["resources"] = {"requests": {"storage": vol["size"]}}
|
||||
@@ -413,17 +257,7 @@ def r_deployment(s: dict) -> dict:
|
||||
# it verbatim for apps that need a fixed on-disk layout (e.g. Gitea keeps all state
|
||||
# directly under /data). Omitting it preserves the /data/<name> default (back-compat).
|
||||
mount_path = vol.get("mountPath", f"/data/{vol['name']}")
|
||||
mount = {"name": vol["name"], "mountPath": mount_path}
|
||||
# subPath mounts a subtree of the volume instead of its root. Emitted only when declared,
|
||||
# so every pre-existing golden stays byte-identical.
|
||||
if vol.get("subPath"):
|
||||
mount["subPath"] = vol["subPath"]
|
||||
# readOnly on the mount as well as on the PV's nfs source. Belt and braces on purpose: the
|
||||
# PV field is what makes the kernel mount `ro`; this one is what a reviewer reading the
|
||||
# Deployment sees. An edit that drops one still leaves the other.
|
||||
if vol.get("readOnly"):
|
||||
mount["readOnly"] = True
|
||||
volume_mounts.append(mount)
|
||||
volume_mounts.append({"name": vol["name"], "mountPath": mount_path})
|
||||
volumes.append(
|
||||
{
|
||||
"name": vol["name"],
|
||||
@@ -709,82 +543,6 @@ def r_gateway_ingress_policy(s: dict):
|
||||
}
|
||||
|
||||
|
||||
def r_in_cluster_client_policies(s: dict) -> list:
|
||||
"""East-west allow: let a declared IN-CLUSTER client reach this service DIRECTLY (R-NET-02).
|
||||
|
||||
`ingressRoutes` opens the NORTH-SOUTH path only — r_gateway_ingress_policy admits the
|
||||
reserved `ingress` identity (the Gateway proxy) and nothing else — and `dependencies`
|
||||
compiles edges only BETWEEN CATALOG SERVICES (both halves are rendered from a catalog
|
||||
entry's own declaration). A PLATFORM CONTROLLER that dials this service's ClusterIP
|
||||
directly therefore had no authorable form at all, and under default-deny (R-NET-01) it is
|
||||
DROPPED.
|
||||
|
||||
That gap is not theoretical: prod's `flux-system` source-controller clones the platform's
|
||||
own repo from the in-cluster Gitea Service (clusters/prod/kustomization.yaml pins
|
||||
`url: http://gitea.gitea.svc.cluster.local:3000/...` so prod's GitOps depends on neither
|
||||
public DNS, Cloudflare, its own Gateway nor ${BASE_DOMAIN}). gitea-allow-gateway-ingress
|
||||
admits `fromEntities: [ingress]` only, so the clone was denied, the source could not
|
||||
reconcile — and the HTTPRoute that would have served a public name instead lives in a
|
||||
DOWNSTREAM tier that cannot apply while the source is down. Source down -> routes never
|
||||
rename -> source stays down. It had to be broken with a live `kubectl apply`, i.e. with a
|
||||
cluster object outside git, which one `kubectl delete` or one rebuild would take away.
|
||||
|
||||
Each entry renders ONE least-privilege ingress CiliumNetworkPolicy, shaped exactly like the
|
||||
reciprocal-dependency half (r_reverse_dependency_policies) but selecting the client by
|
||||
NAMESPACE alone, because a platform controller's pod labels are the upstream chart's to
|
||||
change and are not part of this platform's contract: `fromEndpoints` carrying ONLY the
|
||||
reserved `k8s:io.kubernetes.pod.namespace` label, `toPorts` carrying ONLY the declared
|
||||
port/protocol. Never `fromEntities: all`, and never the whole service. Additive: the
|
||||
namespace default-deny baseline keeps denying every undeclared source and port.
|
||||
|
||||
The policy NAME is `<service>-allow-<name>-ingress`, `name` defaulting to the namespace —
|
||||
it is operator-facing (`kubectl get cnp`, a Hubble drop), and letting an author pin it is
|
||||
what allows an allowance that already had to be applied live to be authored under its
|
||||
EXISTING name, so Flux adopts that object in place rather than creating a near-duplicate
|
||||
beside it and leaving the live one unmanaged.
|
||||
"""
|
||||
out = []
|
||||
for client in s["inClusterClients"]:
|
||||
proto = client.get("protocol", "tcp").upper()
|
||||
out.append(
|
||||
{
|
||||
"apiVersion": "cilium.io/v2",
|
||||
"kind": "CiliumNetworkPolicy",
|
||||
"metadata": {
|
||||
"name": f"{s['name']}-allow-{client['name']}-ingress",
|
||||
"namespace": s["name"],
|
||||
"labels": {"app.kubernetes.io/name": s["name"]},
|
||||
},
|
||||
"spec": {
|
||||
"endpointSelector": {
|
||||
"matchLabels": {"app.kubernetes.io/name": s["name"]}
|
||||
},
|
||||
"ingress": [
|
||||
{
|
||||
"fromEndpoints": [
|
||||
{
|
||||
"matchLabels": {
|
||||
"k8s:io.kubernetes.pod.namespace": client[
|
||||
"namespace"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"toPorts": [
|
||||
{
|
||||
"ports": [
|
||||
{"port": str(client["port"]), "protocol": proto}
|
||||
]
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _route_meta(s: dict, route: dict) -> dict:
|
||||
meta = {
|
||||
"name": f"{s['name']}-{route['port']}",
|
||||
@@ -1337,14 +1095,6 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
|
||||
if s["kind"] == "container":
|
||||
# Container PVCs. (For a vm, the CDI DataVolume owns/creates the disk PVC, so we
|
||||
# never render a plain PVC for the VM root disk — it would collide with CDI.)
|
||||
# The static bulk PV is rendered BEFORE the PVC that names it. Document order inside one
|
||||
# kustomize build is not an apply-ordering guarantee, but a PVC whose PV does not exist
|
||||
# yet merely stays Pending and binds on the next reconcile, so this is a readability
|
||||
# convention rather than a correctness requirement.
|
||||
for pv in r_bulk_pvs(s):
|
||||
docs.append(
|
||||
(f"PersistentVolume {pv['metadata']['name']} - static external NFS (bulk tier)", pv)
|
||||
)
|
||||
for pvc in r_pvcs(s):
|
||||
docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc))
|
||||
docs.append(("Deployment (R-PLAT-02)", r_deployment(s)))
|
||||
@@ -1377,15 +1127,6 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
|
||||
)
|
||||
)
|
||||
|
||||
for cnp in r_in_cluster_client_policies(s):
|
||||
docs.append(
|
||||
(
|
||||
"CiliumNetworkPolicy in-cluster client ingress allow (R-NET-02): "
|
||||
f"{cnp['metadata']['name']}",
|
||||
cnp,
|
||||
)
|
||||
)
|
||||
|
||||
ie = r_internet_egress(s)
|
||||
docs.append(
|
||||
(
|
||||
@@ -1580,11 +1321,6 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
|
||||
- `<svc>-db.yaml` -> wave(svc-db) (a strictly-lower wave; CNPG DB)
|
||||
- `<svc>-db-app.sops.yaml` -> wave(svc-db) (CNPG initdb owner secret)
|
||||
- `<svc>-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret)
|
||||
- `<svc>-db-backup-credentials-prod.sops.yaml` -> wave(svc-db) (the PROD half of that
|
||||
secret: the two environments carry DIFFERENT object-store credentials, selected in
|
||||
the Cluster CR by `${BACKUP_CREDENTIALS_SUFFIX}`. Both objects ship to both clusters,
|
||||
so both must be placed in the DB wave — the Secret has to exist before the Cluster
|
||||
that names it.)
|
||||
Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
|
||||
Only files that actually exist in `present_files` are placed, so the plan is robust to a
|
||||
service having no bootstrap secret or no DB. Deterministic (sorted).
|
||||
@@ -1612,9 +1348,6 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
|
||||
add(wdb, f"{db}.yaml")
|
||||
add(wdb, f"{db}-app.sops.yaml")
|
||||
add(wdb, f"{db}-backup-credentials.sops.yaml") # CNPG Barman S3 creds (R-STATE-03)
|
||||
# PROD half of the same credential (2026-09-07 per-env split). Placed only when
|
||||
# the file exists, so a DB that has not been split is unaffected.
|
||||
add(wdb, f"{db}-backup-credentials-prod.sops.yaml")
|
||||
|
||||
for unit, cfg in EXTRA_UNITS.items():
|
||||
anchor = cfg["anchor"]
|
||||
@@ -1902,101 +1635,6 @@ def _selftest() -> int:
|
||||
"a non-exposed service must not render a gateway-ingress allow (least privilege)",
|
||||
)
|
||||
|
||||
# 4c) inClusterClients -> one ADDITIVE, least-privilege ingress allow per entry that admits a
|
||||
# PLATFORM controller's NAMESPACE (not the reserved `ingress` identity, not `all`) to ONE
|
||||
# declared port. This is what keeps a cluster whose GitOps source is an in-cluster Service
|
||||
# reconcilable: fromEntities[ingress] does not cover a controller dialling the ClusterIP.
|
||||
icc_docs = render(
|
||||
{
|
||||
"name": "g",
|
||||
"source": {"image": "a:1"},
|
||||
"ingressRoutes": [{"host": "g.home.keithjackson.dev", "port": 3000}],
|
||||
"auth": {"mode": "none"},
|
||||
"inClusterClients": [
|
||||
{"name": "flux", "namespace": "flux-system", "port": 3000}
|
||||
],
|
||||
}
|
||||
)
|
||||
icc = [
|
||||
d[1]
|
||||
for d in icc_docs
|
||||
if d[1]["kind"] == "CiliumNetworkPolicy"
|
||||
and d[1]["metadata"]["name"] == "g-allow-flux-ingress"
|
||||
]
|
||||
check(
|
||||
len(icc) == 1,
|
||||
"an inClusterClients entry must render one ingress CNP named "
|
||||
"<service>-allow-<name>-ingress",
|
||||
)
|
||||
if icc:
|
||||
p = icc[0]
|
||||
check(
|
||||
p["metadata"]["namespace"] == "g"
|
||||
and p["spec"]["endpointSelector"]["matchLabels"]
|
||||
== {"app.kubernetes.io/name": "g"},
|
||||
"in-cluster client allow must live in the service's namespace and select the service",
|
||||
)
|
||||
ing = p["spec"]["ingress"]
|
||||
check(
|
||||
ing[0]["fromEndpoints"]
|
||||
== [{"matchLabels": {"k8s:io.kubernetes.pod.namespace": "flux-system"}}],
|
||||
"in-cluster client allow must match the client NAMESPACE only (reserved k8s: label)",
|
||||
)
|
||||
check(
|
||||
"fromEntities" not in ing[0],
|
||||
"in-cluster client allow must not widen to an entity (never fromEntities: all)",
|
||||
)
|
||||
icc_ports = {
|
||||
(pt["port"], pt["protocol"])
|
||||
for r in ing
|
||||
for tp in r["toPorts"]
|
||||
for pt in tp["ports"]
|
||||
}
|
||||
check(
|
||||
icc_ports == {("3000", "TCP")},
|
||||
"in-cluster client allow must be least-privilege (only the declared port/protocol)",
|
||||
)
|
||||
# ADDITIVE: default-deny ingress stays [] and the gateway allow is untouched.
|
||||
icc_dd = next(d[1] for d in icc_docs if d[1]["kind"] == "NetworkPolicy")
|
||||
check(
|
||||
icc_dd["spec"]["ingress"] == [],
|
||||
"in-cluster client allow must be ADDITIVE — default-deny ingress stays [] (R-NET-01)",
|
||||
)
|
||||
check(
|
||||
any(
|
||||
d[1]["metadata"].get("name") == "g-allow-gateway-ingress"
|
||||
for d in icc_docs
|
||||
if d[1]["kind"] == "CiliumNetworkPolicy"
|
||||
),
|
||||
"in-cluster client allow must not displace the north-south gateway allow",
|
||||
)
|
||||
# `name` is OPTIONAL and defaults to the namespace, so the common case is namespace + port.
|
||||
icc_default = render(
|
||||
{
|
||||
"name": "g",
|
||||
"source": {"image": "a:1"},
|
||||
"inClusterClients": [{"namespace": "monitoring", "port": 9090}],
|
||||
}
|
||||
)
|
||||
check(
|
||||
any(
|
||||
d[1]["metadata"].get("name") == "g-allow-monitoring-ingress"
|
||||
for d in icc_default
|
||||
if d[1]["kind"] == "CiliumNetworkPolicy"
|
||||
),
|
||||
"inClusterClients.name must default to the admitted namespace",
|
||||
)
|
||||
# A service that declares none must render NO such policy (grant-nothing-extra).
|
||||
check(
|
||||
not any(
|
||||
d[1]["kind"] == "CiliumNetworkPolicy"
|
||||
and d[1]["metadata"]["name"].endswith("-ingress")
|
||||
and "fromEndpoints" in d[1]["spec"].get("ingress", [{}])[0]
|
||||
for d in render({"name": "q", "source": {"image": "a:1"}})
|
||||
),
|
||||
"a service declaring no inClusterClients must render no in-cluster client allow",
|
||||
)
|
||||
|
||||
# 5) vm kind -> VirtualMachine + standalone DataVolume.
|
||||
vm_docs = render(
|
||||
{
|
||||
@@ -2192,7 +1830,6 @@ def _selftest() -> int:
|
||||
present = {
|
||||
"a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
|
||||
"svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
|
||||
"svc-db-backup-credentials-prod.sops.yaml",
|
||||
"gitea.yaml",
|
||||
"gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
|
||||
}
|
||||
@@ -2202,10 +1839,8 @@ def _selftest() -> int:
|
||||
flat = {f: wv for wv, fs in plan.items() for f in fs}
|
||||
check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
|
||||
check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
|
||||
and flat.get("svc-db-backup-credentials.sops.yaml") == 0
|
||||
and flat.get("svc-db-backup-credentials-prod.sops.yaml") == 0,
|
||||
"CNPG DB manifest + its owner secret + BOTH halves of its per-env Barman backup secret "
|
||||
"must be in the DB's (lower) wave")
|
||||
and flat.get("svc-db-backup-credentials.sops.yaml") == 0,
|
||||
"CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave")
|
||||
check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
|
||||
"a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
|
||||
check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")
|
||||
|
||||
+11
-31
@@ -78,34 +78,16 @@ env:
|
||||
# (util ConfigType `oidc_providers`, env-tagged SEMAPHORE_OIDC_PROVIDERS). The value is a single
|
||||
# JSON blob for the `keycloak` provider — provider_url is the homelab realm issuer (auto-discovery),
|
||||
# client_id=semaphore, redirect_url=/api/auth/oidc/keycloak/redirect, claims map to the realm's
|
||||
# token. The name is NOT a kubelet service-link var (those are SEMAPHORE_SERVICE_HOST /
|
||||
# SEMAPHORE_PORT_*), so no collision. The LOCAL SEMAPHORE_ADMIN above stays break-glass (docs/15).
|
||||
#
|
||||
# PER-ENVIRONMENT URLS (2026-09-07 fix). The blob USED to live whole inside the SOPS-encrypted
|
||||
# semaphore-bootstrap Secret (key `oidc-providers-json`) because it CONTAINS the confidential
|
||||
# client secret. That hid two environment-specific, NON-SECRET URLs — the realm issuer and the
|
||||
# redirect URI — inside ciphertext, where Flux's ${BASE_DOMAIN} postBuild substitution CANNOT
|
||||
# reach them. Both were frozen at the apex `home.keithjackson.dev`, so PROD (BASE_DOMAIN =
|
||||
# prod.home.keithjackson.dev) pointed its OIDC at DEV's Keycloak — unroutable from prod's
|
||||
# 192.168.2.0/24 (the identical root cause as the Headlamp login timeout; see
|
||||
# platform/headlamp/README.md). Dev was only ever "right" because dev IS the apex.
|
||||
#
|
||||
# THE SPLIT: the URLs are public, so the blob is now a PLAINTEXT literal here and gets
|
||||
# ${BASE_DOMAIN}-substituted per environment like every other hostname. Only the confidential
|
||||
# client secret stays SOPS-encrypted, in its own key (`oidc-client-secret`), and is spliced back
|
||||
# in by the KUBELET's dependent-env-var expansion: an env `value` may reference an EARLIER env
|
||||
# var of the same container as $(NAME), and secretKeyRef-sourced vars are eligible. So
|
||||
# SEMAPHORE_OIDC_CLIENT_SECRET MUST STAY IMMEDIATELY ABOVE — the kubelet resolves env in list
|
||||
# order, and a reference to a not-yet-defined name is left as the LITERAL text $(...). It is
|
||||
# fail-loud in the way that matters: if the Secret key is missing the Pod never starts
|
||||
# (CreateContainerConfigError), rather than silently authenticating against nothing. ---
|
||||
- name: SEMAPHORE_OIDC_CLIENT_SECRET
|
||||
# token. It CONTAINS the confidential client secret, so the whole blob is held SOPS-encrypted in the
|
||||
# semaphore-bootstrap Secret (key oidc-providers-json; same secret value SOPS-held in
|
||||
# keycloak-realm-clients) and injected here by reference — never in cleartext. The name is NOT a
|
||||
# kubelet service-link var (those are SEMAPHORE_SERVICE_HOST / SEMAPHORE_PORT_*), so no collision.
|
||||
# The LOCAL SEMAPHORE_ADMIN above stays as the break-glass path (docs/15). ---
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: semaphore-bootstrap
|
||||
key: oidc-client-secret
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
|
||||
key: oidc-providers-json
|
||||
# The semaphoreui/semaphore image runs as the non-root user UID 1001 / GID 0 (tini -> server-wrapper),
|
||||
# and on boot the wrapper writes its generated config.json into the config dir. A mounted Longhorn PVC
|
||||
# is root:root, so UID 1001 cannot write it -> the pod panics ("permission denied"). fsGroup=1001 makes
|
||||
@@ -135,12 +117,10 @@ auth:
|
||||
groups:
|
||||
- platform-admins # access scoped to an authorized group (R-IDP-06)
|
||||
# WIRED (native OIDC): the Keycloak `semaphore` confidential client (platform/keycloak/keycloak-realm-
|
||||
# import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env — a PLAINTEXT JSON blob
|
||||
# (see `env` above) whose per-environment URLs are ${BASE_DOMAIN}-substituted, with the confidential
|
||||
# client secret spliced in at runtime from the semaphore-bootstrap Secret key `oidc-client-secret`.
|
||||
# The `groups` claim is exposed for authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the
|
||||
# break-glass path so an IdP outage never locks the operator out (docs/15 "Availability caveat").
|
||||
# No secret is embedded here.
|
||||
# import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env (JSON blob held encrypted
|
||||
# in the semaphore-bootstrap Secret, oidc-providers-json). The `groups` claim is exposed for
|
||||
# authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the break-glass path so an IdP outage
|
||||
# never locks the operator out (docs/15 "Availability caveat"). No secret is embedded here.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=2.19.0 <3.0.0" # stay on the 2.x stable line
|
||||
|
||||
+5
-82
@@ -9,23 +9,6 @@ name: sonarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
env:
|
||||
- name: PUID
|
||||
value: "1000" # owner uid of every directory under /mnt/Storage/plex
|
||||
- name: PGID
|
||||
value: "3000" # dominant group there; keeps files this writes readable to the
|
||||
# EXISTING non-k3s consumers of the same export, which stay live
|
||||
- name: TZ
|
||||
value: Etc/UTC # same safe default plex uses; the arrs schedule by local time
|
||||
podSecurityContext:
|
||||
# For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
|
||||
# does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
|
||||
# what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
|
||||
# /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
|
||||
fsGroup: 3000
|
||||
exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
@@ -33,66 +16,11 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
|
||||
# /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
|
||||
#
|
||||
# *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Sonarr imports by HARDLINKING or
|
||||
# atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
|
||||
# and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
|
||||
# /mnt/Storage/plex/tv/ the library
|
||||
# /mnt/Storage/plex/downloads/complete/tv-sonarr/ where finished releases land
|
||||
# — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
|
||||
# them as two subPath volumes (subPath: tv at /media/tv, subPath: downloads elsewhere) creates
|
||||
# TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Sonarr then
|
||||
# silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
|
||||
# seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
|
||||
# is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
|
||||
# requirement, not a shortcut. (The cost is stated plainly: Sonarr can also write /media/movies,
|
||||
# which is radarr's. Narrowing that would cost hardlinks, which is a worse trade.)
|
||||
#
|
||||
# *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
|
||||
# renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
|
||||
# config volume entirely; the pod would come up with its app database shadowed by an 8 T media
|
||||
# tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
|
||||
# the same export, so when qbittorrent is eventually given this mount too, the arrs and the
|
||||
# download client will already agree on absolute paths — which is the other precondition for
|
||||
# hardlinked imports (otherwise every import needs a Remote Path Mapping).
|
||||
#
|
||||
# SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
|
||||
# is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
|
||||
# with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
|
||||
# Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
|
||||
# documents having already been bitten by. Fixing it means starting from an empty config volume, so
|
||||
# it belongs in its own change with the operator present.
|
||||
#
|
||||
# UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
|
||||
# + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
|
||||
# the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
|
||||
# write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
|
||||
# every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
|
||||
#
|
||||
# AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
|
||||
# ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
|
||||
# lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
|
||||
# that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
|
||||
# the image re-chowning /config for itself.
|
||||
- name: media
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
|
||||
mountPath: /media # => /media/tv (Root Folder) and /media/downloads (import source)
|
||||
size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
|
||||
# rounded up, so `kubectl get pv` reports something true.
|
||||
# NOTE (bulk-media PREREQUISITE): docs/13 row 2 places the media library / downloads volume on the
|
||||
# `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
|
||||
# but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
|
||||
# volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
|
||||
# therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Sonarr deployable today.
|
||||
dependencies:
|
||||
- to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
|
||||
port: 8080
|
||||
@@ -107,11 +35,6 @@ auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
groups:
|
||||
- media-users # authenticated AND in the media group (R-IDP-06)
|
||||
# See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
|
||||
# platform-admins here is a superset, not a widening (that group already carries cluster-admin
|
||||
# via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
|
||||
# immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
|
||||
- platform-admins # operators of these services must not be locked out of them
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=4.0.0 <5.0.0" # stay on the Sonarr v4 stable line
|
||||
|
||||
+3
-27
@@ -27,31 +27,9 @@ make -C ci test # unit tests for these scripts themselves (ci/tests/*.tes
|
||||
```
|
||||
|
||||
The `l0` stage has three sub-stages: ServiceDefinition schema validation, the Kyverno policy gate
|
||||
(`kyverno test`), and **`conformance/run.sh --layer l0 --include-scope whole-repo`** — the whole-repo
|
||||
L0 invariants a per-resource policy cannot express (secret hygiene across every tracked file,
|
||||
least-privilege RBAC, spec-immutability enforcement). See `conformance/tests/README.md`.
|
||||
|
||||
### `--include-scope whole-repo`: why the L0 sub-stage is not purely requirement-scoped
|
||||
|
||||
R-TEST-03 runs the conformance subset a change **touches**. That has one blind spot, and it is the
|
||||
worst-shaped kind: an invariant that quantifies over the whole tracked tree, or over subtrees the
|
||||
coarse path→requirement map keys to *nothing* (`apps/`, `infra/`, most of `platform/`), cannot be
|
||||
selected *by* the requirements a change touches — because **the change most likely to break it
|
||||
attributes no requirement at all**. Requirement-scoped selection then filters the check out of
|
||||
exactly that change.
|
||||
|
||||
This used to be handled by a hand-written "UNCONDITIONAL" stage per such check (3c subst-safety,
|
||||
3d keycloak-client-secrets, 3e bulk-tier): three near-identical rationales, three hardcoded
|
||||
`check.sh` paths, three copies of the same `rc 0/3/*` case block — and, worse, a mechanism that
|
||||
failed **silently**, because whoever added the next whole-repo invariant had to *remember* to write
|
||||
a stage. Those stages are gone. A test now **declares** `conformance.home.k3s/scope: "whole-repo"`
|
||||
and the runner unions every declaring test into the run. Adding the annotation is the only step.
|
||||
|
||||
`rc 3` is preserved exactly as those stages had it: a **SKIP** of a scope-declared check means the
|
||||
tree it audits has vanished, so the runner exits `3`, `ci/l0.sh` sets the gate's own `rc=3`, and the
|
||||
gate reports SKIPPED — **never a pass**. The mechanism is itself proven, end-to-end and falsifiably,
|
||||
by `conformance/tests/l0-static/whole-repo-invariants-wired`; `conformance/lib/common.sh` carries the
|
||||
full rationale and the incidents behind it.
|
||||
(`kyverno test`), and **`conformance/run.sh --layer l0`** — the whole-repo L0 invariants a
|
||||
per-resource policy cannot express (secret hygiene across every tracked file, least-privilege RBAC,
|
||||
spec-immutability enforcement). See `conformance/tests/README.md`.
|
||||
|
||||
## Exit-code contract
|
||||
|
||||
@@ -109,8 +87,6 @@ ci/
|
||||
lint.sh Stage: yaml/lint
|
||||
l0.sh Stage: schema validation + Kyverno L0 gate + L0 static conformance
|
||||
l1.sh Stage: L1 rendering checks (Chainsaw L1 via the conformance runner)
|
||||
(harness: platform/monitoring/tests/). Catches the two alerting bugs no
|
||||
other stage can see — permanently-firing and structurally INERT rules.
|
||||
conformance.sh Stage: L2/L3 in-cluster Chainsaw Job (trigger + collect)
|
||||
touched-requirements.sh compute the R-* subset a change touches (drives the R-TEST-03 filter);
|
||||
exits 3 LOUDLY rather than guessing when the diff base is unreliable
|
||||
|
||||
+1
-5
@@ -47,11 +47,7 @@ fi
|
||||
# (conformance/lib/common.sh) now unions a comma/space list of layers, so this is a single
|
||||
# invocation with one aggregated report (no per-layer looping needed). Artifacts are then
|
||||
# COLLECTED from the fixed location below.
|
||||
# --include-scope: union the touched subset with every L2/L3 test DECLARING
|
||||
# `conformance.home.k3s/scope: whole-repo` (none today — wired once, here, so a future
|
||||
# whole-repo behavioural invariant needs only the annotation; see conformance/lib/common.sh).
|
||||
: "${WHOLE_REPO_SCOPE:=whole-repo}"
|
||||
"$CONFORMANCE_CMD" --layer l2,l3 --include-scope "$WHOLE_REPO_SCOPE" "${req_args[@]}"
|
||||
"$CONFORMANCE_CMD" --layer l2,l3 "${req_args[@]}"
|
||||
|
||||
CONF_ART="$(dirname "$CONFORMANCE_CMD")/_artifacts"
|
||||
if [ -f "${CONF_ART}/report.junit.xml" ]; then
|
||||
|
||||
@@ -10,18 +10,6 @@
|
||||
# Stage 3b L0 : conformance/run.sh --layer l0 — the WHOLE-REPO L0 invariants that
|
||||
# a per-resource policy cannot express (secret hygiene across every
|
||||
# tracked file, least-privilege RBAC, spec-immutability enforcement).
|
||||
# Selection = the TOUCHED requirement subset (R-TEST-03) UNION every L0
|
||||
# test DECLARING `conformance.home.k3s/scope: whole-repo`. That union
|
||||
# replaces what used to be a hand-written "UNCONDITIONAL" stage per
|
||||
# whole-repo check (3c subst-safety / 3d keycloak-client-secrets / 3e
|
||||
# bulk-tier): three near-identical 12-line rationales, three hardcoded
|
||||
# check.sh paths and three copies of the same rc 0/3/* case block, which
|
||||
# did not scale and — worse — failed SILENTLY, because whoever added the
|
||||
# next whole-repo invariant had to REMEMBER to hand-write a stage. Now the
|
||||
# test declares its own scope and the runner enforces it. The reasoning and
|
||||
# the incidents those stages recorded were moved INTO the tests they
|
||||
# describe (see each chainsaw-test.yaml's `scope` annotation) and the
|
||||
# general mechanism is documented in conformance/lib/common.sh.
|
||||
#
|
||||
# Integration seams are env-overridable (see ci/lib.sh + README). When an owning
|
||||
# subtree is still a placeholder, this reports SKIPPED (exit 3) for that sub-stage
|
||||
@@ -95,45 +83,58 @@ fi
|
||||
# binding, the spec-immutability gate's own behaviour) live in the conformance
|
||||
# harness's tests/l0-static subtree and run with no cluster. This stage executes them
|
||||
# through the same single-command runner CI and a laptop both use (R-TEST-02).
|
||||
#
|
||||
# THE UNION, AND WHY IT IS NOT OPTIONAL (R-TEST-03's blind spot)
|
||||
# --------------------------------------------------------------
|
||||
# 3b scopes the conformance run to the TOUCHED requirements. That is right for an invariant
|
||||
# whose inputs live under the subtree its requirements map to. It is WRONG for one that
|
||||
# quantifies over the WHOLE tracked tree, or over subtrees (apps/, infra/, most of platform/)
|
||||
# that ci/touched-requirements.sh's coarse path map keys to NOTHING: for those, the change
|
||||
# most likely to break the invariant attributes no requirement at all, so requirement-scoped
|
||||
# selection filters the check out of EXACTLY that change. `--include-scope whole-repo` unions
|
||||
# in every L0 test that DECLARES that scope, so those run on every gate invocation.
|
||||
#
|
||||
# rc 3 IS NOT A PASS. The runner returns 3 when a scope-declared check SKIPPED (its subject
|
||||
# subtree is absent, or it has no check script): UNWIRED, not green, exactly as the deleted
|
||||
# per-check stages had it ("check SKIPPED (clusters/ absent) — not a pass"). It is mapped to
|
||||
# the gate's own rc 3 below, never to rc 0.
|
||||
: "${WHOLE_REPO_SCOPE:=whole-repo}"
|
||||
log "Stage: L0 static conformance checks (${CONFORMANCE_CMD} --layer l0 --include-scope ${WHOLE_REPO_SCOPE})"
|
||||
log "Stage: L0 static conformance checks (${CONFORMANCE_CMD} --layer l0)"
|
||||
if ! entrypoint_ready "$CONFORMANCE_CMD"; then
|
||||
skip "conformance runner not present at '${CONFORMANCE_CMD}' — UNWIRED, not green"
|
||||
summary "- l0-static: **SKIPPED** (conformance runner missing)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi
|
||||
else
|
||||
load_touched_requirements
|
||||
l0_static_rc=0
|
||||
"$CONFORMANCE_CMD" --layer l0 --include-scope "$WHOLE_REPO_SCOPE" \
|
||||
${REQ_ARGS[@]+"${REQ_ARGS[@]}"} || l0_static_rc=$?
|
||||
case "$l0_static_rc" in
|
||||
0) ok "L0 static conformance checks passed"
|
||||
summary "- l0-static: **passed**" ;;
|
||||
3) skip "an L0 check declaring scope='${WHOLE_REPO_SCOPE}' SKIPPED — UNWIRED, not a pass"
|
||||
summary "- l0-static: **SKIPPED** (a whole-repo check is UNWIRED — not a pass)"
|
||||
if "$CONFORMANCE_CMD" --layer l0 ${REQ_ARGS[@]+"${REQ_ARGS[@]}"}; then
|
||||
ok "L0 static conformance checks passed"
|
||||
summary "- l0-static: **passed**"
|
||||
else
|
||||
err "L0 static conformance checks failed"
|
||||
summary "- l0-static: **FAILED**"
|
||||
rc=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- Stage 3c: Flux post-build substitution safety (UNCONDITIONAL) ----------
|
||||
# R-ENV-02's promote-once mechanism is `postBuild.substituteFrom cluster-vars`, and it
|
||||
# FAILS OPEN: an undefined ${TOKEN} becomes the empty string and the Kustomization still
|
||||
# reports Ready. That emptied Cilium's k8sServiceHost and broke the CNI on 2/3 dev nodes
|
||||
# (DECISIONS-LOG 2026-09-01, "substitution race that fails OPEN"). The check asserts every
|
||||
# ${...} in every substituted path is DEFINED in cluster-vars or deliberately ESCAPED
|
||||
# `$${...}` — i.e. that enabling the StrictPostBuildSubstitutions feature gate cannot fail
|
||||
# a legitimate manifest, and that no expansion silently empties today.
|
||||
#
|
||||
# WHY THIS IS ITS OWN STAGE and not left to Stage 3b: 3b scopes the conformance run to the
|
||||
# TOUCHED requirements (R-TEST-03), and ci/touched-requirements.sh's coarse path map keys
|
||||
# only clusters/* to R-ENV-02 — an edit to a SUBSTITUTED manifest under apps/ or platform/
|
||||
# attributes no requirement, so the requirement-scoped selection would filter this check
|
||||
# out of exactly the change that introduces the bug. The invariant is whole-repo (same
|
||||
# reasoning as spec-immutability's own gate stage), so it runs on EVERY gate invocation.
|
||||
# When 3b's touched set does include R-ENV-02/R-GITOPS-01 the check runs twice; it is a
|
||||
# sub-second offline scan with no side effects, so the duplication is not worth avoiding.
|
||||
SUBST_CHECK="${SUBST_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/flux-postbuild-substitution/check.sh}"
|
||||
log "Stage: Flux postBuild substitution safety (${SUBST_CHECK})"
|
||||
if [ ! -f "$SUBST_CHECK" ]; then
|
||||
skip "substitution check not present at '${SUBST_CHECK}' — UNWIRED, not green"
|
||||
summary "- subst-safety: **SKIPPED** (check missing)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi
|
||||
else
|
||||
subst_rc=0
|
||||
bash "$SUBST_CHECK" || subst_rc=$?
|
||||
case "$subst_rc" in
|
||||
0) ok "Flux postBuild substitution safety passed"
|
||||
summary "- subst-safety: **passed**" ;;
|
||||
3) skip "substitution check SKIPPED (clusters/ absent) — not a pass"
|
||||
summary "- subst-safety: **SKIPPED** (nothing to audit)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi ;;
|
||||
*) err "L0 static conformance checks failed"
|
||||
summary "- l0-static: **FAILED**"
|
||||
rc=1 ;;
|
||||
*) err "Flux postBuild substitution safety FAILED"
|
||||
summary "- subst-safety: **FAILED**"; rc=1 ;;
|
||||
esac
|
||||
# Per-check step-summary lines, so the gate output still NAMES what ran — the generated
|
||||
# replacement for the one hardcoded `summary` line each deleted stage carried.
|
||||
emit_offline_summary
|
||||
fi
|
||||
|
||||
case "$rc" in
|
||||
|
||||
@@ -29,16 +29,7 @@ req_args=(${REQ_ARGS[@]+"${REQ_ARGS[@]}"})
|
||||
# [--requirements R-A,R-B]; it ALWAYS writes artifacts to a FIXED location
|
||||
# (conformance/_artifacts/) and does NOT accept --junit/--coverage. Invoke by layer +
|
||||
# touched subset, then COLLECT its artifacts into the CI artifact dirs.
|
||||
# --include-scope: union the touched subset with every L1 test DECLARING
|
||||
# `conformance.home.k3s/scope: whole-repo`. No L1 test declares it today, so this is a no-op
|
||||
# now — it is wired HERE, once, so the NEXT whole-repo invariant (at any layer) is picked up by
|
||||
# declaring the annotation, instead of needing someone to remember to hand-write a stage. That
|
||||
# "someone must remember" is the failure mode this whole change removes; see
|
||||
# conformance/lib/common.sh. rc 3 = a scope-declared check SKIPPED = UNWIRED, never a pass.
|
||||
: "${WHOLE_REPO_SCOPE:=whole-repo}"
|
||||
l1_rc=0
|
||||
"$CONFORMANCE_CMD" --layer l1 --include-scope "$WHOLE_REPO_SCOPE" \
|
||||
${req_args[@]+"${req_args[@]}"} || l1_rc=$?
|
||||
"$CONFORMANCE_CMD" --layer l1 ${req_args[@]+"${req_args[@]}"}
|
||||
|
||||
CONF_ART="$(dirname "$CONFORMANCE_CMD")/_artifacts"
|
||||
if [ -f "${CONF_ART}/report.junit.xml" ]; then
|
||||
@@ -48,16 +39,5 @@ if [ -d "${CONF_ART}/coverage" ]; then
|
||||
cp -r "${CONF_ART}/coverage/." "${COVERAGE_DIR}/"
|
||||
fi
|
||||
|
||||
case "$l1_rc" in
|
||||
0) ok "L1 rendering checks passed"
|
||||
summary "- l1-render: **passed**"
|
||||
emit_offline_summary ;;
|
||||
3) skip "an L1 check declaring scope='${WHOLE_REPO_SCOPE}' SKIPPED — UNWIRED, not a pass"
|
||||
summary "- l1-render: **SKIPPED** (a whole-repo check is UNWIRED — not a pass)"
|
||||
emit_offline_summary
|
||||
exit 3 ;;
|
||||
*) err "L1 rendering checks FAILED (rc=${l1_rc})"
|
||||
summary "- l1-render: **FAILED**"
|
||||
emit_offline_summary
|
||||
exit "$l1_rc" ;;
|
||||
esac
|
||||
ok "L1 rendering checks passed"
|
||||
summary "- l1-render: **passed**"
|
||||
@@ -86,25 +86,6 @@ cluster_available() {
|
||||
|
||||
mkartifacts() { mkdir -p "$JUNIT_DIR" "$COVERAGE_DIR"; }
|
||||
|
||||
# --- Offline (L0/L1) per-check step summary -----------------------------------
|
||||
# conformance/lib/run_l1_offline.py writes one Markdown line per check it executed to
|
||||
# conformance/_artifacts/offline-summary.md. Splice those into $GITHUB_STEP_SUMMARY so the
|
||||
# gate's output still NAMES every check that ran and its verdict.
|
||||
#
|
||||
# WHY THIS EXISTS: ci/l0.sh used to carry a hand-written stage per whole-repo check, each with
|
||||
# its own `summary "- subst-safety: **passed**"` line. Those stages are gone (the tests declare
|
||||
# `conformance.home.k3s/scope: whole-repo` instead and the runner unions them in), so the
|
||||
# naming is generated for EVERY offline check rather than for the three that had a stage.
|
||||
: "${OFFLINE_SUMMARY_FILE:=${CONFORMANCE_DIR}/_artifacts/offline-summary.md}"
|
||||
emit_offline_summary() {
|
||||
local f="${1:-$OFFLINE_SUMMARY_FILE}" line
|
||||
[ -f "$f" ] || return 0
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] && summary " $line"
|
||||
done < "$f"
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Touched-requirement subset (R-TEST-03 input) ----------------------------
|
||||
# Sets: TOUCHED_RC (0 ok / 3 not determinable), TOUCHED_REQS[] and REQ_ARGS[]
|
||||
# (ready to splice into a conformance/run.sh invocation).
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# =============================================================================
|
||||
# ci/render-keycloak-client-urls.py
|
||||
#
|
||||
# Render platform/keycloak/client-urls.yaml FROM platform/keycloak/keycloak-realm-import.yaml.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# ---------------
|
||||
# The Keycloak realm import is a ONE-SHOT that fails OPEN: once it rests at
|
||||
# Done=True / Started=False it never re-runs, so a redirect URI changed in git is
|
||||
# never applied to the realm. The client-secret half of that failure class is
|
||||
# already healed by a level-triggered CronJob (platform/keycloak/client-secret-
|
||||
# reconciler.yaml). This renderer supplies the DESIRED STATE that the same CronJob
|
||||
# needs to heal the URL half.
|
||||
#
|
||||
# THE CONSTRAINT THAT SHAPES IT: the reconciler pod runs with
|
||||
# automountServiceAccountToken: false. It has NO Kubernetes API access at all, by
|
||||
# design (R-SEC-02) — so it cannot read the KeycloakRealmImport CR to learn what
|
||||
# the realm is supposed to look like. Its inputs must be MOUNTED. Hence a
|
||||
# ConfigMap.
|
||||
#
|
||||
# A hand-authored ConfigMap would be a SECOND SOURCE OF TRUTH for every public
|
||||
# hostname in the realm, free to disagree with the realm import — and "two
|
||||
# declarations that can disagree" is the same shape as the bug this whole effort
|
||||
# is closing. So the ConfigMap is GENERATED from the realm import and the
|
||||
# generation is GATED: conformance/tests/l0-static/keycloak-client-secret-
|
||||
# reconciler/check.sh runs this script with --check on every merge, and the merge
|
||||
# fails if the committed file is not what this renderer produces. That is the same
|
||||
# render==golden idiom catalog/render.py + ci/render-golden.sh already use for
|
||||
# apps/.
|
||||
#
|
||||
# STRINGS ARE COPIED VERBATIM, NEVER RE-DERIVED. If a URL in the realm import is
|
||||
# written as a Flux substitution token (https://grafana.${BASE_DOMAIN}/...), the
|
||||
# token is carried through into the ConfigMap CHARACTER FOR CHARACTER. Both files
|
||||
# are rendered by the SAME Flux Kustomization (./platform-config, whose
|
||||
# postBuild.substituteFrom reads cluster-vars), so both are substituted in the
|
||||
# same pass with the same values and cannot disagree about what BASE_DOMAIN is.
|
||||
# This renderer never parses, splits, or reconstructs a hostname, and never reads
|
||||
# cluster-vars — so there is no second place for the domain to be decided.
|
||||
#
|
||||
# USAGE
|
||||
# python3 ci/render-keycloak-client-urls.py # print the rendered file
|
||||
# python3 ci/render-keycloak-client-urls.py --write # write it (regenerate)
|
||||
# python3 ci/render-keycloak-client-urls.py --check # exit 1 if it has drifted
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("FATAL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
|
||||
OUT_FILE = "platform/keycloak/client-urls.yaml"
|
||||
DATA_KEY = "client-urls.json"
|
||||
CONFIGMAP_NAME = "keycloak-client-urls"
|
||||
NAMESPACE = "keycloak"
|
||||
|
||||
# The client fields this renderer exports, and therefore the ONLY fields the
|
||||
# reconciler may write. Every one of them is an environment-derived URL: it changes
|
||||
# when BASE_DOMAIN changes, and a stale value breaks OIDC login (a wrong
|
||||
# redirectUri) or silently widens trust (a stale wildcard left behind).
|
||||
#
|
||||
# DELIBERATELY ABSENT: `secret` (healed by the secret pass from the SOPS Secret,
|
||||
# and it must never appear in a ConfigMap), and every non-URL field —
|
||||
# protocolMappers, defaultClientScopes, attributes, flags. Those are not
|
||||
# environment-derived, so nothing re-renders them on a domain flip; keeping them
|
||||
# out keeps the reconciler's write surface as narrow as the problem.
|
||||
LIST_FIELDS = ("redirectUris", "webOrigins")
|
||||
STR_FIELDS = ("rootUrl", "baseUrl", "adminUrl")
|
||||
|
||||
|
||||
def render(root: Path) -> str:
|
||||
"""Return the full text of client-urls.yaml as it should be committed."""
|
||||
docs = list(yaml.safe_load_all((root / IMPORT_FILE).read_text()))
|
||||
imp = next((d for d in docs if d and d.get("kind") == "KeycloakRealmImport"), None)
|
||||
if imp is None:
|
||||
raise SystemExit("FATAL: no KeycloakRealmImport document in %s" % IMPORT_FILE)
|
||||
clients = (imp.get("spec") or {}).get("realm", {}).get("clients") or []
|
||||
if not clients:
|
||||
# Fail closed. Emitting an empty desired state would make the reconciler
|
||||
# believe every client is unmanaged and silently stop reconciling URLs.
|
||||
raise SystemExit("FATAL: %s declares zero clients — refusing to render an "
|
||||
"empty desired state" % IMPORT_FILE)
|
||||
|
||||
want: dict[str, dict] = {}
|
||||
for c in clients:
|
||||
cid = c.get("clientId")
|
||||
if not isinstance(cid, str) or not cid:
|
||||
raise SystemExit("FATAL: %s has a client with no clientId" % IMPORT_FILE)
|
||||
entry: dict[str, object] = {}
|
||||
for f in LIST_FIELDS:
|
||||
if f not in c:
|
||||
continue # not declared => NOT MANAGED (never forced empty)
|
||||
v = c[f]
|
||||
if not isinstance(v, list) or not all(isinstance(x, str) for x in v):
|
||||
raise SystemExit("FATAL: %s client %r field %s is not a list of strings"
|
||||
% (IMPORT_FILE, cid, f))
|
||||
entry[f] = list(v) # verbatim, order as authored
|
||||
for f in STR_FIELDS:
|
||||
if f not in c:
|
||||
continue
|
||||
v = c[f]
|
||||
if not isinstance(v, str):
|
||||
raise SystemExit("FATAL: %s client %r field %s is not a string"
|
||||
% (IMPORT_FILE, cid, f))
|
||||
entry[f] = v
|
||||
if entry:
|
||||
want[cid] = entry
|
||||
if not want:
|
||||
raise SystemExit("FATAL: no client in %s declares any of %s — refusing to render "
|
||||
"an empty desired state"
|
||||
% (IMPORT_FILE, ", ".join(LIST_FIELDS + STR_FIELDS)))
|
||||
|
||||
body = json.dumps(want, indent=2, sort_keys=True, ensure_ascii=False)
|
||||
indented = "\n".join((" " + ln) if ln else "" for ln in body.splitlines())
|
||||
return HEADER + indented + "\n"
|
||||
|
||||
|
||||
HEADER = """---
|
||||
# =============================================================================
|
||||
# GENERATED FILE — DO NOT EDIT BY HAND.
|
||||
#
|
||||
# source: platform/keycloak/keycloak-realm-import.yaml (spec.realm.clients)
|
||||
# regenerate: python3 ci/render-keycloak-client-urls.py --write
|
||||
# gated by: conformance/tests/l0-static/keycloak-client-secret-reconciler
|
||||
# (runs this renderer with --check on every merge)
|
||||
#
|
||||
# WHAT THIS IS: the DESIRED STATE for every environment-derived client URL in the
|
||||
# `homelab` realm — redirectUris, webOrigins, rootUrl, baseUrl, adminUrl. The
|
||||
# client-secret reconciler CronJob mounts it and makes the live realm match it.
|
||||
#
|
||||
# WHY IT IS A SEPARATE OBJECT: the realm import is a ONE-SHOT that never re-runs
|
||||
# once Done=True, so a redirect URI edited in git never reaches the realm. Prod's
|
||||
# stored redirect URIs were measured still pointing at the pre-migration hostname
|
||||
# while git declared the new one, and every OIDC login would have failed with a
|
||||
# redirect_uri mismatch. The reconciler is the level-triggered loop that closes
|
||||
# that; this file is what it reconciles TOWARDS.
|
||||
#
|
||||
# WHY IT IS GENERATED AND NOT AUTHORED: a hand-written copy of every public
|
||||
# hostname would be a second source of truth, free to disagree with the realm
|
||||
# import. It is rendered from the import and the render is gated, so the two
|
||||
# cannot drift. URL strings are copied VERBATIM — if the source uses a
|
||||
# ${BASE_DOMAIN} token it appears here unchanged, and both files are substituted
|
||||
# by the SAME Flux Kustomization (./platform-config), so the domain is decided in
|
||||
# exactly one place.
|
||||
#
|
||||
# A field that is NOT declared on a client in the realm import is simply ABSENT
|
||||
# here, and the reconciler leaves it alone. Absence means UNMANAGED, never
|
||||
# "force to empty".
|
||||
# =============================================================================
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
labels:
|
||||
app.kubernetes.io/name: %s
|
||||
app.kubernetes.io/part-of: keycloak
|
||||
data:
|
||||
%s: |
|
||||
""" % (CONFIGMAP_NAME, NAMESPACE, CONFIGMAP_NAME, DATA_KEY)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
g = ap.add_mutually_exclusive_group()
|
||||
g.add_argument("--write", action="store_true", help="write %s" % OUT_FILE)
|
||||
g.add_argument("--check", action="store_true",
|
||||
help="exit 1 if the committed file has drifted from the source")
|
||||
args = ap.parse_args()
|
||||
|
||||
want = render(ROOT)
|
||||
target = ROOT / OUT_FILE
|
||||
|
||||
if args.write:
|
||||
target.write_text(want, encoding="utf-8")
|
||||
print("wrote %s" % OUT_FILE)
|
||||
return 0
|
||||
|
||||
if args.check:
|
||||
if not target.is_file():
|
||||
print("CHECK FAIL: %s does not exist. Run: python3 "
|
||||
"ci/render-keycloak-client-urls.py --write" % OUT_FILE, file=sys.stderr)
|
||||
return 1
|
||||
have = target.read_text(encoding="utf-8")
|
||||
if have == want:
|
||||
print("client-urls.yaml is in sync with %s" % IMPORT_FILE)
|
||||
return 0
|
||||
print("CHECK FAIL: %s has DRIFTED from %s — the reconciler would converge the "
|
||||
"realm on a desired state the realm import does not declare.\n"
|
||||
"Regenerate with: python3 ci/render-keycloak-client-urls.py --write\n"
|
||||
% (OUT_FILE, IMPORT_FILE), file=sys.stderr)
|
||||
for line in difflib.unified_diff(have.splitlines(True), want.splitlines(True),
|
||||
fromfile="committed " + OUT_FILE,
|
||||
tofile="rendered from " + IMPORT_FILE):
|
||||
sys.stderr.write(line)
|
||||
return 1
|
||||
|
||||
sys.stdout.write(want)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,164 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# ci/tests/touched-requirements-test-attribution.test.sh — regression tests for DETECTION
|
||||
# RULE 3 in ci/touched-requirements.sh: a change anywhere under `conformance/tests/<test>/`
|
||||
# attributes THAT TEST'S OWN `conformance.home.k3s/R-*` labels.
|
||||
#
|
||||
# THE GAP THESE LOCK DOWN (2026-09-12). The coarse path map keys `conformance/*` to
|
||||
# R-TEST-01..04 — the requirements about the HARNESS — and says nothing about what a given
|
||||
# test PROVES. So weakening `conformance/tests/.../check.sh` attributed R-TEST-01..04 and
|
||||
# nothing else: the merge gate ran the harness's self-tests and NOT the weakened test's own
|
||||
# requirement subset. Weakening (or deleting) a test is the one change that most needs the
|
||||
# requirement it certifies to be re-verified, and it was the change least likely to select it.
|
||||
# The content scan cannot close this: it reads only ADDED/REMOVED lines, so it sees a change
|
||||
# to the LABEL BLOCK but not one to the test BODY — exactly the case that matters.
|
||||
#
|
||||
# Also covers the two path-map entries added in the same change (infra/* -> R-ENV-01, and
|
||||
# R-TEST-06 on .gitea/*|ci/*), which is how the two L0 checks that are DELIBERATELY not
|
||||
# whole-repo-scoped get selected by the paths that can break them.
|
||||
#
|
||||
# Same harness as the sibling suites: the real script, pointed at a THROWAWAY git repo via
|
||||
# the REPO_ROOT override. No network, no cluster, the working repo is never touched.
|
||||
#
|
||||
# Run: ci/tests/touched-requirements-test-attribution.test.sh (or `make -C ci test`)
|
||||
set -euo pipefail
|
||||
|
||||
CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SCRIPT="${CI_DIR}/touched-requirements.sh"
|
||||
|
||||
PASS=0 FAIL=0
|
||||
ok() { printf ' ok %s\n' "$*"; PASS=$((PASS+1)); }
|
||||
bad() { printf ' FAIL %s\n' "$*" >&2; FAIL=$((FAIL+1)); }
|
||||
|
||||
assert_present() {
|
||||
if grep -qxF "$1" <<<"$2"; then ok "$3"
|
||||
else bad "$3 (missing '$1' from: $(tr '\n' ' ' <<<"$2"))"; fi
|
||||
}
|
||||
assert_absent() {
|
||||
if grep -qxF "$1" <<<"$2"; then bad "$3 (found unexpected '$1' in: $(tr '\n' ' ' <<<"$2"))"
|
||||
else ok "$3"; fi
|
||||
}
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
git -C "$TMP" init -q -b main
|
||||
git -C "$TMP" config user.email [email protected]
|
||||
git -C "$TMP" config user.name t
|
||||
git -C "$TMP" config commit.gpgsign false
|
||||
|
||||
# --- Baseline: two conformance tests, each declaring its own requirement labels ----------
|
||||
# TEST A: a flat l0-static test. TEST B: a test with a NESTED fixture dir, to prove the
|
||||
# walk-up from an arbitrary changed file to the enclosing test dir.
|
||||
mkdir -p "$TMP/conformance/tests/l0-static/alpha" \
|
||||
"$TMP/conformance/tests/l2-behavioral/beta/fixtures" \
|
||||
"$TMP/conformance/lib" "$TMP/infra" "$TMP/ci"
|
||||
|
||||
cat > "$TMP/conformance/tests/l0-static/alpha/chainsaw-test.yaml" <<'YAML'
|
||||
# HEADER PROSE. This test used to also cover conformance.home.k3s/R-STALE-09: "true",
|
||||
# which is a REFERENCE in a comment, not a declaration, and must NOT be attributed.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: alpha
|
||||
labels:
|
||||
conformance.home.k3s/R-ALPHA-01: "true"
|
||||
conformance.home.k3s/R-ALPHA-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/layer: "L0"
|
||||
spec: {}
|
||||
YAML
|
||||
# The check BODY deliberately contains NO R-ID text, so anything the scan finds for a
|
||||
# body-only edit came from rule 3 and not from the content scan.
|
||||
cat > "$TMP/conformance/tests/l0-static/alpha/check.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
echo "assert something"
|
||||
SH
|
||||
|
||||
cat > "$TMP/conformance/tests/l2-behavioral/beta/chainsaw-test.yaml" <<'YAML'
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: beta
|
||||
labels:
|
||||
conformance.home.k3s/R-BETA-07: "true"
|
||||
spec: {}
|
||||
YAML
|
||||
echo "seed: 1" > "$TMP/conformance/tests/l2-behavioral/beta/fixtures/data.yaml"
|
||||
|
||||
echo "harness helper" > "$TMP/conformance/lib/helper.sh"
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm baseline
|
||||
|
||||
# ========================================================================================
|
||||
echo "== case 1: a BODY-ONLY edit to a test attributes that test's own labels"
|
||||
git -C "$TMP" checkout -q -b body-edit
|
||||
cat > "$TMP/conformance/tests/l0-static/alpha/check.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
echo "assert something weaker"
|
||||
SH
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm weaken
|
||||
OUT="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set:\n%s\n---\n' "$OUT"
|
||||
assert_present "R-ALPHA-01" "$OUT" "body-only test edit attributes the test's own R-ALPHA-01"
|
||||
assert_present "R-ALPHA-02" "$OUT" "body-only test edit attributes the test's own R-ALPHA-02"
|
||||
assert_present "R-TEST-03" "$OUT" "the conformance/* path map still applies (R-TEST-03)"
|
||||
assert_absent "R-STALE-09" "$OUT" "an R-ID cited only in the test's HEADER COMMENT is NOT attributed"
|
||||
assert_absent "R-BETA-07" "$OUT" "an UNRELATED test's labels are NOT attributed"
|
||||
|
||||
# ========================================================================================
|
||||
echo "== case 2: an edit to a NESTED file walks up to the enclosing test"
|
||||
git -C "$TMP" checkout -q -b nested-edit main
|
||||
echo "seed: 2" > "$TMP/conformance/tests/l2-behavioral/beta/fixtures/data.yaml"
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm fixture
|
||||
OUT="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set:\n%s\n---\n' "$OUT"
|
||||
assert_present "R-BETA-07" "$OUT" "nested fixture edit attributes the enclosing test's R-BETA-07"
|
||||
assert_absent "R-ALPHA-01" "$OUT" "nested fixture edit does NOT attribute a sibling test's labels"
|
||||
|
||||
# ========================================================================================
|
||||
echo "== case 3: DELETING a test still attributes what it used to certify"
|
||||
# Removing the only verifier of a requirement must not be the cheapest way to make the gate
|
||||
# stop checking it — so the labels are read from the BASE when HEAD no longer has them.
|
||||
git -C "$TMP" checkout -q -b delete-test main
|
||||
git -C "$TMP" rm -rq "$TMP/conformance/tests/l0-static/alpha"
|
||||
git -C "$TMP" commit -qm "delete the alpha test"
|
||||
OUT="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set:\n%s\n---\n' "$OUT"
|
||||
assert_present "R-ALPHA-01" "$OUT" "deleting a test STILL attributes its R-ALPHA-01 (read from base)"
|
||||
assert_present "R-ALPHA-02" "$OUT" "deleting a test STILL attributes its R-ALPHA-02 (read from base)"
|
||||
|
||||
# ========================================================================================
|
||||
echo "== case 4: a change OUTSIDE conformance/tests/ gains no test labels"
|
||||
git -C "$TMP" checkout -q -b lib-edit main
|
||||
echo "harness helper v2" > "$TMP/conformance/lib/helper.sh"
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm lib
|
||||
OUT="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set:\n%s\n---\n' "$OUT"
|
||||
assert_present "R-TEST-02" "$OUT" "a conformance/lib edit still attributes the harness requirements"
|
||||
assert_absent "R-ALPHA-01" "$OUT" "a conformance/lib edit attributes NO individual test's labels"
|
||||
assert_absent "R-BETA-07" "$OUT" "a conformance/lib edit attributes NO individual test's labels (B)"
|
||||
|
||||
# ========================================================================================
|
||||
echo "== case 5: path-map entries for the two DELIBERATELY path-scoped L0 checks"
|
||||
# infra/* -> R-ENV-01 (dev-nodes-provisioned-as-code reads nothing but infra/).
|
||||
git -C "$TMP" checkout -q -b infra-edit main
|
||||
printf 'resource "x" "y" {}\n' > "$TMP/infra/main.tf"
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm infra
|
||||
OUT="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set:\n%s\n---\n' "$OUT"
|
||||
assert_present "R-ENV-01" "$OUT" "an infra/ edit attributes R-ENV-01 (was UNMAPPED: attributed nothing)"
|
||||
|
||||
# ci/* -> ... R-TEST-06 (spec-immutability-enforced reads nothing but ci/ + the spec repo).
|
||||
git -C "$TMP" checkout -q -b ci-edit main
|
||||
printf 'echo hello\n' > "$TMP/ci/thing.sh"
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm ci
|
||||
OUT="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set:\n%s\n---\n' "$OUT"
|
||||
assert_present "R-TEST-06" "$OUT" "a ci/ edit attributes R-TEST-06 (the spec-immutability test's own ID)"
|
||||
assert_present "R-GITOPS-01" "$OUT" "a ci/ edit still attributes R-GITOPS-01"
|
||||
|
||||
echo
|
||||
if [ "$FAIL" -eq 0 ]; then
|
||||
echo "touched-requirements test-attribution tests: PASS (${PASS} assertions)"; exit 0
|
||||
fi
|
||||
echo "touched-requirements test-attribution tests: FAIL (${FAIL} failed, ${PASS} passed)" >&2
|
||||
exit 1
|
||||
@@ -93,49 +93,6 @@ printf 'touched set (removal):\n%s\n---\n' "$OUT2"
|
||||
assert_absent "R-DELDOC-01" "$OUT2" "REMOVED whole-line comment R-ID is NOT attributed"
|
||||
assert_present "R-DELREAL-01" "$OUT2" "REMOVED real-line R-ID IS attributed"
|
||||
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# PATH MAP — bootstrap/* (added 2026-09-13)
|
||||
#
|
||||
# WHY THIS IS LOCKED DOWN. path_requirements() keys `bootstrap/*` to R-STATE-03 +
|
||||
# R-GITOPS-01, and until 2026-09-13 the comment justifying it described
|
||||
# `bootstrap/standup.sh` as a file that carries the placeholder guards. That file has never
|
||||
# been on `main` (it landed only on unmerged prod-standup branches), so the comment was
|
||||
# stale and the entry looked like dead weight someone would tidy away. It is not dead: it
|
||||
# is what SELECTS conformance/tests/l0-static/placeholder-guard-coupling for the change
|
||||
# that eventually lands a placeholder guard under bootstrap/ — and that test discovers its
|
||||
# guard by content rather than by path, so it activates on exactly that change. Removing
|
||||
# the entry would let the change introducing a guard be gated by nothing, which is the
|
||||
# 2026-09-05 class. These assertions make the entry's deletion a test failure rather than a
|
||||
# silent loss of selection.
|
||||
#
|
||||
# NOTE the R-ID content scan must not be what produces these: `notes/` is outside the path
|
||||
# map precisely so the other cases isolate the scan, and here the file content cites no
|
||||
# R-ID at all, so any R-ID in the output came from the PATH.
|
||||
git -C "$TMP" checkout -q -b feature3 main
|
||||
mkdir -p "$TMP/bootstrap/flux"
|
||||
cat > "$TMP/bootstrap/flux/bootstrap.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
# a bootstrap-path change that cites no requirement id of its own
|
||||
echo "flux bootstrap"
|
||||
SH
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm "touch bootstrap/"
|
||||
OUT3="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set (bootstrap/ path map):\n%s\n---\n' "$OUT3"
|
||||
assert_present "R-STATE-03" "$OUT3" "bootstrap/* attributes R-STATE-03 (selects placeholder-guard-coupling)"
|
||||
assert_present "R-GITOPS-01" "$OUT3" "bootstrap/* attributes R-GITOPS-01 (it IS the GitOps bootstrap path)"
|
||||
|
||||
# And the neighbouring entry the same test leans on for its other half: clusters/* must
|
||||
# attribute R-ENV-02, which is what selects placeholder-guard-coupling's HALF A (the
|
||||
# authored-values sweep over clusters/<env> effective cluster-vars).
|
||||
git -C "$TMP" checkout -q -b feature4 main
|
||||
mkdir -p "$TMP/clusters/base"
|
||||
printf 'kind: ConfigMap\nmetadata:\n name: cluster-vars\n' > "$TMP/clusters/base/cluster-vars.yaml"
|
||||
git -C "$TMP" add -A && git -C "$TMP" commit -qm "touch clusters/"
|
||||
OUT4="$(REPO_ROOT="$TMP" BASE_REF=main "$SCRIPT" 2>/dev/null)"
|
||||
printf 'touched set (clusters/ path map):\n%s\n---\n' "$OUT4"
|
||||
assert_present "R-ENV-02" "$OUT4" "clusters/* attributes R-ENV-02 (selects placeholder-guard-coupling HALF A)"
|
||||
assert_present "R-GITOPS-01" "$OUT4" "clusters/* attributes R-GITOPS-01"
|
||||
|
||||
echo
|
||||
if [ "$FAIL" -eq 0 ]; then echo "touched-requirements tests: PASS ($PASS assertions)"; exit 0;
|
||||
else echo "touched-requirements tests: FAIL ($FAIL failed, $PASS passed)"; exit 1; fi
|
||||
+10
-122
@@ -21,12 +21,8 @@
|
||||
# the R-IDs that policies/ServiceDefinitions self-declare (docs/14 + ADR-0011).
|
||||
# Untouched lines and prose are NOT scanned; see changed_diff() for why.
|
||||
# 2. A coarse path->requirement fallback for changes that don't self-declare IDs.
|
||||
# 3. For a change anywhere under `conformance/tests/<test>/`: THAT TEST'S OWN requirement
|
||||
# labels, read from its chainsaw-test.yaml. See test_requirements_for_path().
|
||||
# The coarse map (2) NEVER overrides an explicit tag; it only widens coverage for
|
||||
# untagged changes so nothing silently escapes the gate. (3) is the same principle applied
|
||||
# to the tests themselves: editing or weakening a test must be gated by the requirements
|
||||
# that test CERTIFIES, not merely by R-TEST-0x.
|
||||
# untagged changes so nothing silently escapes the gate.
|
||||
set -euo pipefail
|
||||
CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
@@ -184,128 +180,23 @@ path_requirements() {
|
||||
platform/externaldns/*) echo "R-DNS-02" ;;
|
||||
platform/velero/*) echo "R-STATE-03" ;;
|
||||
platform/monitoring/*) echo "R-OBS-01 R-OBS-02" ;;
|
||||
# platform/cnpg was UNMAPPED until 2026-09-06. It is the PostgreSQL DR path for keycloak-db /
|
||||
# nextcloud-db / semaphore-db (barmanObjectStore WAL archiving + ScheduledBackup), so a change
|
||||
# there is an R-IDP-05 / R-STATE-03 change; it now also carries the platform-health alert rules
|
||||
# for that path (R-OBS-04). Without this line an edit under platform/cnpg/ contributed ZERO
|
||||
# requirements unless it happened to retype an R-ID on a non-comment line, and escaped the gate.
|
||||
platform/cnpg/*) echo "R-IDP-05 R-STATE-03 R-OBS-04" ;;
|
||||
# vpn-egress was UNMAPPED until 2026-09-06. platform/vpn-egress/egress-steering.yaml IS the
|
||||
# R-NET-04 datapath (the rule that selects between the two egress paths) and the R-NET-05/06/07
|
||||
# steering policy — it even self-declares those IDs in its `homelab.keithjackson.dev/requirements`
|
||||
# annotation. But the content scan only reads ADDED/REMOVED lines, so any edit that does not
|
||||
# happen to retype an R-ID (a selector change, an excludedCIDRs change, a comment) contributed
|
||||
# ZERO requirements and escaped the gate entirely. Mapped explicitly; the annotation still wins
|
||||
# when a diff does carry the tags.
|
||||
platform/vpn-egress/*) echo "R-NET-04 R-NET-05 R-NET-06 R-NET-07" ;;
|
||||
# Keycloak is the whole SSO surface: the realm import (declarative identity config),
|
||||
# the SOPS client secrets, and the reconciler that heals them. Added 2026-09-06 after
|
||||
# the client-secret incident, in which editing platform/keycloak/ attributed NO
|
||||
# requirement, so the L2 keycloak tests were filtered out of the merge gate's touched
|
||||
# subset — the change that could break every OIDC login was gated by nothing.
|
||||
# R-IDP-05 is in scope because platform/keycloak/ also carries the CNPG DB Cluster and
|
||||
# its off-cluster backup credential, not only the realm.
|
||||
platform/keycloak/*) echo "R-IDP-01 R-IDP-02 R-IDP-03 R-IDP-04 R-IDP-05 R-IDP-06" ;;
|
||||
platform/oauth2-proxy/*) echo "R-IDP-02 R-IDP-06" ;;
|
||||
platform/image-automation/*) echo "R-UPD-01 R-UPD-02" ;;
|
||||
conformance/*) echo "R-TEST-01 R-TEST-02 R-TEST-03 R-TEST-04" ;;
|
||||
# infra/ was UNMAPPED until 2026-09-12 — the OpenTofu + Ansible that provision the dev
|
||||
# nodes. conformance/tests/l0-static/dev-nodes-provisioned-as-code reads NOTHING but
|
||||
# infra/, yet an infra/ edit attributed ZERO requirements, so the one check that asserts
|
||||
# the nodes are provisioned as code was filtered out of every change that could break it.
|
||||
# That test's invariant IS path-scoped (it is deliberately NOT whole-repo-scoped; see its
|
||||
# chainsaw-test.yaml), and the correct fix for a path-scoped invariant is a path-map entry.
|
||||
infra/*) echo "R-ENV-01" ;;
|
||||
clusters/*) echo "R-GITOPS-01 R-ENV-02" ;;
|
||||
# bootstrap/ was UNMAPPED until 2026-09-06. The entry STAYS, but its original rationale
|
||||
# was written against a file that is not here, and that is corrected on 2026-09-13:
|
||||
# it claimed "bootstrap/standup.sh carries the placeholder guards that decide whether
|
||||
# `velero` may be enabled against a real backup endpoint". bootstrap/standup.sh HAS NEVER
|
||||
# BEEN ON `main` — it was not deleted (no deletion commit exists anywhere), it landed only
|
||||
# on unmerged prod-standup branches. `git ls-tree main bootstrap/` lists README.md, flux/,
|
||||
# gitea/, sops/. So R-STATE-03 here is ANTICIPATORY, not descriptive.
|
||||
#
|
||||
# WHY IT IS STILL RIGHT. conformance/tests/l0-static/placeholder-guard-coupling DISCOVERS
|
||||
# its guard (any tracked *.sh defining `<name>_is_placeholder()`) instead of hardcoding a
|
||||
# path, so the moment a standup guard lands under bootstrap/ that test's coupling half
|
||||
# activates — and this entry is what selects the test for the change that lands it.
|
||||
# Deleting the entry would mean the change introducing a placeholder guard is gated by
|
||||
# nothing, which is the 2026-09-05 class (a prefix glob that stopped matching when the
|
||||
# value it guards changed, in a different file) left to reintroduce itself. R-GITOPS-01 is
|
||||
# descriptive today: bootstrap/flux/, gitea/ and sops/ ARE the GitOps bootstrap path.
|
||||
# Locked down by ci/tests/touched-requirements.test.sh's path-map section.
|
||||
# bootstrap/ was UNMAPPED until 2026-09-06, and the omission was load-bearing:
|
||||
# bootstrap/standup.sh carries the placeholder guards that decide whether `velero` may be
|
||||
# enabled against a real backup endpoint. A change touching standup.sh ALONGSIDE any
|
||||
# already-mapped path scoped the L0 run to that other path's requirements and silently
|
||||
# excluded conformance/tests/l0-static/placeholder-guard-coupling — the check that exists
|
||||
# precisely to catch a guard/value mismatch. The 2026-09-05 guard bug (a prefix glob that
|
||||
# stopped matching when the value it guards changed, in a different file) is exactly the
|
||||
# class that omission would hide.
|
||||
bootstrap/*) echo "R-STATE-03 R-GITOPS-01" ;;
|
||||
# R-TEST-06 added 2026-09-12. The comment in changed_diff()'s rule 3 above records why it
|
||||
# was ABSENT: R-TEST-06 used to be enforced by the spec-immutability STAGE and carried by no
|
||||
# Chainsaw test, so attributing it made the coverage `--require` check false-block. It is now
|
||||
# carried by conformance/tests/l0-static/spec-immutability-enforced — whose entire input
|
||||
# surface is ci/ — so a ci/ change must attribute it, or the test that proves the
|
||||
# spec-immutability mechanism is still load-bearing runs on nothing that can break it.
|
||||
.gitea/*|ci/*) echo "R-GITOPS-01 R-TEST-02 R-TEST-03 R-TEST-06" ;;
|
||||
.gitea/*|ci/*) echo "R-GITOPS-01 R-TEST-02 R-TEST-03" ;;
|
||||
*) : ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------------
|
||||
# A CHANGED TEST ATTRIBUTES THE REQUIREMENTS IT CERTIFIES (detection rule 3).
|
||||
#
|
||||
# THE GAP THIS CLOSES (2026-09-12). The path map keys `conformance/*` to
|
||||
# R-TEST-01..04 — the requirements about the HARNESS. It says nothing about what a given
|
||||
# test PROVES. So editing `conformance/tests/l2-behavioral/tenant-isolation/check.sh` to
|
||||
# weaken or delete an assertion attributed R-TEST-01..04 and nothing else: the merge gate
|
||||
# then ran the harness's self-tests and NOT the weakened test's own requirement subset.
|
||||
# Weakening a test is the one change that most needs the requirement it certifies to be
|
||||
# re-verified, and it was the change least likely to select it.
|
||||
#
|
||||
# The `conformance.home.k3s/R-*: "true"` labels on a test ARE the authoritative declaration of
|
||||
# what it verifies (lib/coverage.py reads the same labels); this reads them directly. Note the
|
||||
# content scan in main() only sees ADDED/REMOVED lines, so it catches a change to the LABEL
|
||||
# BLOCK but not a change to the test BODY — which is exactly the case that matters here.
|
||||
#
|
||||
# WHY IT READS HEAD *OR* THE BASE: a change that DELETES a test, or deletes its
|
||||
# chainsaw-test.yaml, must still attribute what that test used to certify — otherwise removing
|
||||
# the only verifier of a requirement is the cheapest way to make the gate stop checking it.
|
||||
# HEAD first (the post-change truth), base as the fallback (the deleted case). Fail-closed:
|
||||
# this function can only ADD requirements, never remove one.
|
||||
: "${CONF_TEST_LABEL_PREFIX:=conformance.home.k3s}" # == CONF_LABEL_PREFIX, conformance/lib/common.sh
|
||||
|
||||
file_text_head_or_base() {
|
||||
local path="$1"
|
||||
git -C "$REPO_ROOT" show "HEAD:${path}" 2>/dev/null && return 0
|
||||
git -C "$REPO_ROOT" show "${BASE}:${path}" 2>/dev/null && return 0
|
||||
return 0
|
||||
}
|
||||
|
||||
# test_requirements_for_path <changed-path> -> space-separated R-IDs (possibly empty).
|
||||
# Walks UP from the changed file to the nearest ancestor directory carrying a
|
||||
# chainsaw-test.yaml, so it works for any file in a test dir (check.sh, mutations.py,
|
||||
# fixtures, the chainsaw-test.yaml itself) and for nested test layouts.
|
||||
test_requirements_for_path() {
|
||||
local f="$1" dir yaml text
|
||||
case "$f" in conformance/tests/*/*) ;; *) return 0 ;; esac
|
||||
dir="${f%/*}"
|
||||
while :; do
|
||||
case "$dir" in
|
||||
conformance/tests|conformance|.|"") return 0 ;;
|
||||
*/*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
yaml="${dir}/chainsaw-test.yaml"
|
||||
text="$(file_text_head_or_base "$yaml")"
|
||||
if [ -n "$text" ]; then
|
||||
# Whole-line `#` comments are dropped first — same "reference, not declaration" rule the
|
||||
# content scan applies (changed_diff() rule 3): a label shown inside a header example is
|
||||
# documentation. Only a real YAML mapping line counts.
|
||||
printf '%s\n' "$text" \
|
||||
| grep -vE '^[[:space:]]*#' \
|
||||
| sed -nE "s#^[[:space:]]*${CONF_TEST_LABEL_PREFIX}/(R-[A-Z0-9]+-[0-9]+):[[:space:]]*[\"']?true.*#\1#p" \
|
||||
| tr '\n' ' '
|
||||
return 0
|
||||
fi
|
||||
dir="${dir%/*}"
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
local files reqs="" rc=0 base_sha head_sha local_main merge_base
|
||||
|
||||
@@ -339,9 +230,6 @@ main() {
|
||||
while IFS= read -r f; do
|
||||
[ -n "$f" ] || continue
|
||||
reqs+=" $(path_requirements "$f")"
|
||||
# Detection rule 3: a change under conformance/tests/<test>/ attributes THAT TEST'S own
|
||||
# requirement labels, so weakening a test is gated by what the test certifies.
|
||||
reqs+=" $(test_requirements_for_path "$f")"
|
||||
done <<< "$files"
|
||||
|
||||
# Self-declared R-IDs on the ADDED/REMOVED lines of the change (see changed_diff).
|
||||
|
||||
@@ -21,15 +21,6 @@ YAMLLINT_VERSION="1.35.1"
|
||||
KUBECTL_VERSION="1.36.4"
|
||||
YQ_VERSION="4.44.6"
|
||||
|
||||
# Alert-rule unit tests (promtool test rules) — the L0 check
|
||||
# conformance/tests/l0-static/alert-rules-unit-tested runs every
|
||||
# platform/**/*.promtool-test.yaml through it. MUST stay in lockstep with
|
||||
# infra/devcontainer/versions.env (which bakes it into the image) AND with the Prometheus
|
||||
# version running on prod, because promtool IS the rule engine: verified 2026-09-12 as
|
||||
# v3.14.0-distroless via `kubectl get prometheus -n monitoring -o jsonpath='{.items[*].spec.version}'`.
|
||||
# Shipped inside the prometheus release tarball, not as a separate release.
|
||||
PROMTOOL_VERSION="3.14.0"
|
||||
|
||||
# ServiceDefinition JSON-Schema validation (schemas/validate.sh). Pip-installed;
|
||||
# pinned here so CI and a laptop validate against the exact same validator.
|
||||
CHECK_JSONSCHEMA_VERSION="0.29.4"
|
||||
@@ -34,9 +34,6 @@ spec:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption for the `cloud` INI credentials — velero-cloud.sops.yaml (dev's scoped MinIO key)
|
||||
# and velero-cloud-prod.sops.yaml (prod's Cloudflare R2 token) — is layered per environment by the
|
||||
# overlay (clusters/dev and clusters/prod patch THIS Kustomization by name), identical to how
|
||||
# headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. Both Secrets are applied on both
|
||||
# clusters; ${BACKUP_CREDENTIALS_SUFFIX} (substituted from cluster-vars above) decides which one the
|
||||
# velero Deployment actually mounts. See clusters/dev/kustomization.yaml.
|
||||
# SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per
|
||||
# environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how
|
||||
# headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml.
|
||||
+50
-139
@@ -96,22 +96,9 @@ data:
|
||||
#
|
||||
# Substitution coverage (verify before adding a knob here): every path that consumes these declares
|
||||
# postBuild.substituteFrom cluster-vars — platform (K3S_API_HOST), platform-config
|
||||
# (CILIUM_LB_POOL_*, and keycloak-db's S3_BACKUP_ENDPOINT + CNPG_BACKUP_PATH: the Cluster lives at
|
||||
# platform/keycloak/postgres-cluster.yaml but is rendered by platform-config, which is the ONLY
|
||||
# tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db:
|
||||
# S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH), velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION,
|
||||
# VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an
|
||||
# UNSUBSTITUTED path would ship the literal string into the cluster.
|
||||
#
|
||||
# ADDING A KNOB IS SAFE UNDER StrictPostBuildSubstitutions ONLY BECAUSE OF THE ROOT EDGE. An
|
||||
# UNDEFINED variable is a HARD Kustomization failure, so a manifest that references a knob the
|
||||
# live ConfigMap does not yet carry breaks its whole tier. `clusters/base/platform.yaml` gained
|
||||
# `dependsOn: [flux-system]` for exactly this: dependsOn is REVISION-gated, so no substituting
|
||||
# tier can reconcile revision N until the ROOT Kustomization has APPLIED revision N — i.e. until
|
||||
# the new cluster-vars key EXISTS. Every substituting consumer chains transitively through
|
||||
# `platform`, so one edge covers the graph, and a new knob plus its first reference may ship in
|
||||
# ONE commit. (A hand-run `kubectl apply -k clusters/<env>` bypasses that ordering entirely: apply
|
||||
# the overlay BEFORE the tier that references the knob, or the tier fails until it retries.)
|
||||
# (CILIUM_LB_POOL_*, S3_BACKUP_ENDPOINT), crafty (CRAFTY_L4_VIP), apps-wave-0 (S3_BACKUP_ENDPOINT),
|
||||
# velero (S3_BACKUP_ENDPOINT), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an UNSUBSTITUTED path would
|
||||
# ship the literal string into the cluster.
|
||||
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
|
||||
#
|
||||
@@ -160,45 +147,29 @@ data:
|
||||
UNIFI_L4_VIP: "192.168.1.54" # UniFi controller VM: inform/STUN/adoption (apps/vms/unifi)
|
||||
CRAFTY_L4_VIP: "192.168.1.55" # Crafty: Minecraft Java (TCP) + Bedrock (UDP) game ports
|
||||
|
||||
# --- Bulk (external NFS) storage host (docs/04 bulk tier) ---------------------------------------
|
||||
# THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media
|
||||
# library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive
|
||||
# (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's
|
||||
# `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is
|
||||
# declared in the ServiceDefinition, because the path is identical in every environment and only
|
||||
# the ADDRESS is not.
|
||||
# Off-cluster S3 backup target (R-STATE-03, R-IDP-05). Consumed by BOTH backup paths: every CNPG
|
||||
# Cluster's barmanObjectStore.endpointURL (keycloak-db, nextcloud-db, semaphore-db) and Velero's
|
||||
# backupStorageLocation s3Url. Full URL including scheme (and port, if not the scheme default).
|
||||
#
|
||||
# WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense.
|
||||
# The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN.
|
||||
# dev nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one
|
||||
# NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in
|
||||
# /etc/exports), so this path is proven, not theoretical. Base default = the dev value,
|
||||
# per the convention of the LAN block above.
|
||||
# prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today;
|
||||
# 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening
|
||||
# the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give
|
||||
# DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route).
|
||||
# So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the
|
||||
# blast radius of shipping it unset, which is NOT the usual "fails to apply".
|
||||
BULK_NFS_SERVER: "192.168.1.9"
|
||||
# OPERATOR DECISION 2026-09-05: **Cloudflare R2**, replacing the on-Proxmox MinIO — "for the time
|
||||
# being until a better on-prem backup solution is created". Two reasons beyond cost:
|
||||
# 1. The k3s prod cluster is on the SOHO network (pfSense's WAN) and CANNOT reach the MinIO inside
|
||||
# pfSense's LAN — measured, from a pod: connection timed out. R2 needs no firewall change.
|
||||
# 2. That MinIO runs on the SAME Proxmox host as dev's VMs, so dev's "off-cluster" backup shared a
|
||||
# physical failure domain with dev. One host loss took the cluster AND its backups. R2 is
|
||||
# genuinely off-site.
|
||||
# Measured to justify the cost: dev's whole estate is 4.0 GiB Velero (6,264 objects, 7d retention) +
|
||||
# 211 MiB CNPG (1,755 objects) — pennies per month, and R2 egress is free, so restore REHEARSALS
|
||||
# stay free and R-STATE-03 has no cost disincentive to stay proven.
|
||||
#
|
||||
# The MinIO copy is NOT deleted — it stays as a historical/local copy (3-2-1). Restoring from it
|
||||
# means pointing this knob back at http://192.168.1.9:9000 temporarily.
|
||||
S3_BACKUP_ENDPOINT: "https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com"
|
||||
|
||||
# Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
|
||||
# Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
|
||||
# nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port.
|
||||
# Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR —
|
||||
# and, measured 2026-09-05 from a pod INSIDE prod, it CANNOT reach it anyway (prod is on the SOHO
|
||||
# network 192.168.2.0/24, which is pfSense's WAN, while this MinIO sits inside pfSense's LAN:
|
||||
# `curl http://192.168.1.9:9000/minio/health/live` timed out after 15s). Prod uses Cloudflare R2.
|
||||
S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"
|
||||
|
||||
# S3 region. Every S3 client REQUIRES one; MinIO ignores whatever it is; Cloudflare R2 demands
|
||||
# exactly `auto` (its `head-bucket` reports the physical hint `WNAM`, which is NOT a usable region
|
||||
# string). This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-
|
||||
# accident for MinIO and wrong for anything else, which is why it is a knob now. The base default
|
||||
# is dev's LIVE value, deliberately: dev's Velero has been signing requests with `us-east-1`
|
||||
# against that MinIO since bootstrap, and this repo's rule is that base == the known-good dev
|
||||
# value. Prod overrides to `auto`.
|
||||
S3_BACKUP_REGION: "us-east-1"
|
||||
# S3 region. Required by every S3 client, IGNORED by MinIO, and must be exactly `auto` for R2.
|
||||
# This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-accident for
|
||||
# MinIO and wrong for R2, which is why it is a knob now.
|
||||
S3_BACKUP_REGION: "auto"
|
||||
|
||||
# --- storage (ADR-0003) ---
|
||||
LONGHORN_DEFAULT_REPLICA_COUNT: "3" # default StorageClass replica count (3 on 3 nodes)
|
||||
@@ -252,98 +223,38 @@ data:
|
||||
VELERO_BACKUP_SCHEDULE: "0 3 * * *" # cron; overlays may differ
|
||||
VELERO_RETENTION: "168h" # dev short | prod long
|
||||
|
||||
# ==========================================================================================
|
||||
# OBJECT-STORE TENANCY — the three knobs that keep two clusters and two backup tools from
|
||||
# writing into each other's data. THESE ARE SAFETY KNOBS, not preferences.
|
||||
# ==========================================================================================
|
||||
# Nothing about the object layout was per-environment before this: Velero's bucket was the
|
||||
# hardcoded literal `velero` with no prefix, and every CNPG Cluster used the hardcoded
|
||||
# `s3://cnpg` with barman's `serverName` defaulting to the CLUSTER NAME — and BOTH environments
|
||||
# have a keycloak-db, a nextcloud-db and a semaphore-db. Point two clusters at one bucket under
|
||||
# that layout and:
|
||||
# (a) each cluster's Velero ADOPTS the other's backups and TTL-DELETES them on retention; and
|
||||
# (b) two different Postgres servers stream WAL into ONE Barman prefix, which does not merely
|
||||
# mix files — it CORRUPTS THE CHAIN and destroys PITR for both. That is the worse half:
|
||||
# dev's DR restore is PROVEN (R-STATE-03) and this would silently un-prove it.
|
||||
# BACKUP TENANCY — the two knobs that keep two clusters sharing ONE MinIO from destroying each
|
||||
# other's backups. Added 2026-09-05 when the operator ruled prod uses the SAME endpoint as dev.
|
||||
#
|
||||
# WHY THERE IS NO SINGLE `S3_BACKUP_BUCKET`. The two consumers take differently-SHAPED values:
|
||||
# the Velero chart wants `bucket` and `prefix` as separate fields, while CNPG's
|
||||
# barmanObjectStore.destinationPath is ONE full s3:// URL whose first path segment IS the bucket.
|
||||
# CNPG's bucket therefore already lives inside CNPG_BACKUP_PATH; adding a second knob that had to
|
||||
# hold the same bucket name a second time would be a rotation waiting to drift — the exact class
|
||||
# of duplication this repo has been bitten by. Three knobs, each shaped for its own consumer, and
|
||||
# no two of them can contradict each other.
|
||||
# Sharing an endpoint is fine. Sharing a PATH is not, and nothing here was per-environment before:
|
||||
#
|
||||
# BASE = DEV'S LIVE LAYOUT, BYTE-FOR-BYTE, AND MUST STAY THAT WAY. Dev's existing backups are at
|
||||
# `s3://velero/` (bucket root, no prefix) and `s3://cnpg/<clusterName>/` on that MinIO. Giving dev
|
||||
# a prefix, or renaming its buckets, would ORPHAN the very backups its proven restore depends on —
|
||||
# the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default` and its LONGHORN_DATA_PATH stays
|
||||
# /var/lib/longhorn. Verified: with an EMPTY prefix the Velero chart omits `objectStorage.prefix`
|
||||
# from the rendered BackupStorageLocation entirely (helm template, chart 12.1.0), so dev's live
|
||||
# BSL is unchanged by this parameterization rather than merely equivalent to it.
|
||||
# VELERO — bucket `velero`, no prefix, and BOTH clusters run schedules named `daily` and
|
||||
# `daily-resources-only`. Each cluster's Velero SYNCS backups it finds in the BSL into
|
||||
# itself as Backup CRs and GARBAGE-COLLECTS them when their TTL expires. Prod would
|
||||
# therefore adopt dev's backups and delete them on schedule.
|
||||
# CNPG — destinationPath `s3://cnpg` with CNPG appending serverName, which defaults to the
|
||||
# CLUSTER NAME. Both environments have `keycloak-db`, `nextcloud-db`, `semaphore-db`,
|
||||
# so prod's Postgres would archive WAL into the exact prefix dev's does. Two servers
|
||||
# interleaving WAL into one Barman stream CORRUPTS THE CHAIN and destroys PITR — and
|
||||
# dev's DR restore is PROVEN (R-STATE-03). This is the worst of the two.
|
||||
#
|
||||
# PROD, which has no history to preserve, gets ONE bucket with TWO disjoint top-level prefixes:
|
||||
# Both environments are now NAMESPACED SYMMETRICALLY, which was not possible an hour ago and is
|
||||
# worth explaining, because the earlier reasoning was the opposite:
|
||||
#
|
||||
# s3://k3s-prod-backup/velero/… Velero (kopia repo + backup metadata)
|
||||
# s3://k3s-prod-backup/barman/<clusterName>/… CNPG WAL + base backups
|
||||
# While the target was the MinIO, dev's base value HAD to be the bucket root — its existing
|
||||
# backups lived there and giving dev a prefix would have ORPHANED the very backups its proven
|
||||
# restore depends on (the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default`).
|
||||
#
|
||||
# The operator created exactly one R2 bucket, so velero-vs-barman separation has to come from the
|
||||
# prefix rather than the bucket. The BUCKET ROOT IS DELIBERATELY LEFT EMPTY: if a prefix ever
|
||||
# renders wrong, its objects land somewhere obviously bogus instead of silently colliding with a
|
||||
# real tenant's data. Dev↔prod isolation is doubly independent — different endpoint AND different
|
||||
# bucket — so neither cluster can reach the other's objects even with a mis-substituted prefix.
|
||||
VELERO_BACKUP_BUCKET: "velero" # dev: the `velero` MinIO bucket | prod: k3s-prod-backup
|
||||
VELERO_BACKUP_PREFIX: "" # dev: bucket root (live layout) | prod: "velero"
|
||||
CNPG_BACKUP_PATH: "s3://cnpg" # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# WHICH CREDENTIAL SECRET THE BACKUP CONSUMERS READ (added 2026-09-07 — the per-env credential split)
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# The three knobs above say WHERE the backups go. This one says WHICH KEY signs the requests, and
|
||||
# it exists because the endpoint moved per-environment while the credential did not.
|
||||
# Moving to R2 starts BOTH clusters on an EMPTY bucket, so there is no history to orphan and no
|
||||
# reason to keep the asymmetry. `dev` and `prod` is clearer, greppable, and leaves the bucket root
|
||||
# free — so an unset/misrendered prefix lands somewhere obviously wrong instead of silently
|
||||
# colliding with a real environment's backups.
|
||||
#
|
||||
# THE BUG IT FIXES. The four backup-credential Secrets — velero-cloud.sops.yaml and the three
|
||||
# <db>-backup-credentials.sops.yaml — are SHARED BASE manifests: platform/ and apps/ are rendered
|
||||
# by the same Flux tiers on both clusters, and cluster-vars substitution is the ONLY thing that
|
||||
# differs (R-ENV-02). So writing prod's R2 token into those four files did not GIVE prod a
|
||||
# credential, it REPLACED DEV'S: dev's Velero and all three dev CNPG Clusters would have started
|
||||
# signing MinIO requests with an R2 key id. For Velero that is a broken BSL; for CNPG it is worse,
|
||||
# because `archive_mode=on` means Postgres REFUSES TO RECYCLE a WAL segment it could not archive,
|
||||
# so a 403 loop is a monotonically filling pgdata PVC ending in a PANIC — the same failure prod
|
||||
# spent its whole bring-up in with a placeholder endpoint. One shared Secret also means rotating
|
||||
# either environment's key silently breaks the other; that is the structural reason for the split,
|
||||
# independent of dev's eventual decommission.
|
||||
#
|
||||
# THE MECHANISM. Each credential is now TWO Secret objects, both applied to BOTH clusters (they are
|
||||
# base resources; nothing here forks a manifest per environment). The consumer names one:
|
||||
#
|
||||
# velero: credentials.existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
# CNPG: s3Credentials.*.name: <db>-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
#
|
||||
# ONE knob, not four, ON PURPOSE. Four independent secret-name knobs could be updated three-
|
||||
# quarters of the way, leaving one consumer pointed at the OTHER environment's key with nothing
|
||||
# rendering wrong — and StrictPostBuildSubstitutions cannot catch that, because it only fails on an
|
||||
# UNDEFINED variable, never on a defined-but-stale one. With a single suffix the four move together
|
||||
# or not at all. The cost is that the rendered Secret name is not greppable from the consumer; the
|
||||
# names are spelled out in full in each kustomization.yaml beside the resources.
|
||||
#
|
||||
# THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING. It renders the UNSUFFIXED names,
|
||||
# which are the objects that already exist and that the live dev cluster is running. A default
|
||||
# naming a Secret that does not exist would be worse than the shared-Secret state it replaces:
|
||||
# Velero's pod would sit in ContainerCreating forever on a missing `existingSecret` volume, and a
|
||||
# CNPG Cluster whose s3Credentials Secret is absent stops archiving WAL — i.e. the disk-full clock
|
||||
# again — so an unresolvable default breaks the referencing cluster HARDER than a wrong key does.
|
||||
# Empty is also the value dev must carry, so base and dev agree, exactly like VELERO_BACKUP_PREFIX.
|
||||
#
|
||||
# An EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions: strict mode fails on a variable that is
|
||||
# not DEFINED, not on one defined as "". VELERO_BACKUP_PREFIX above has shipped empty since the
|
||||
# gate was enabled and proves it. Both overlays state this knob explicitly — dev "" and prod
|
||||
# "-prod" — so neither inherits it silently.
|
||||
#
|
||||
# ADDING A NEW BACKUP CREDENTIAL: create BOTH halves (`x-credentials.sops.yaml` and
|
||||
# `x-credentials-prod.sops.yaml`), list both in the kustomization, and reference
|
||||
# `x-credentials${BACKUP_CREDENTIALS_SUFFIX}`. Never a single shared Secret again.
|
||||
BACKUP_CREDENTIALS_SUFFIX: "" # dev: "" (MinIO creds) | prod: "-prod" (R2 creds)
|
||||
# *** THESE TWO VALUES AND S3_BACKUP_ENDPOINT MUST CHANGE TOGETHER. *** Setting a `dev` prefix while
|
||||
# the endpoint is still the MinIO would orphan dev's existing MinIO backups for no benefit. They are
|
||||
# one change; do not split the commit.
|
||||
VELERO_BACKUP_PREFIX: "dev" # dev: "dev" | prod: "prod"
|
||||
CNPG_BACKUP_PATH: "s3://cnpg/dev" # dev: s3://cnpg/dev | prod: s3://cnpg/prod
|
||||
|
||||
# --- placement ---
|
||||
PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os" # base no-op selector; prod may pin roles
|
||||
|
||||
@@ -50,13 +50,6 @@ resources:
|
||||
# reverse-proxy sidecar (Option B). POST-platform tier (dependsOn platform-config, wait:false); same
|
||||
# non-gating leaf-UI rationale as headlamp/homepage/oauth2-proxy (a UI must never gate the platform).
|
||||
- crafty.yaml # Flux Kustomization -> ./platform/crafty (dependsOn platform-config)
|
||||
# Alert runbooks — POST-platform tier (dependsOn platform-config, wait:false). Serves the 34
|
||||
# runbooks the alert rules' `runbook_url` annotations point at, at <apex>/runbooks/<name>; before
|
||||
# this tier existed that path was a 404 and every alert handed the operator a dead link. Same
|
||||
# non-gating leaf rationale as headlamp/homepage/crafty. Carries NO postBuild substitution on
|
||||
# purpose — it renders documentation containing `${...}` shell examples that envsubst would empty;
|
||||
# see clusters/base/runbooks.yaml.
|
||||
- runbooks.yaml # Flux Kustomization -> ./runbooks (dependsOn platform-config)
|
||||
# Velero backup controller — POST-platform tier (dependsOn platform-config, wait:false). A backup
|
||||
# controller must NEVER gate the platform chain; same non-gating rationale as headlamp/oauth2-proxy.
|
||||
# Carries velero-cloud.sops.yaml, so clusters/dev adds a SOPS decryption patch for it (by name).
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
---
|
||||
# Flux Kustomization: alert runbooks — reconciles ./runbooks.
|
||||
#
|
||||
# WHAT THIS SERVES. Every alert rule in this repo carries
|
||||
# `runbook_url: "https://${BASE_DOMAIN}/runbooks/<name>"` — 42 annotations over 34 distinct names.
|
||||
# Until this tier existed that path returned 404 on the live cluster, so every alert that reached a
|
||||
# human handed them a dead link at the moment they needed help. This Kustomization renders the
|
||||
# markdown under ./runbooks into a ConfigMap and a two-replica nginx that serves it at that exact
|
||||
# prefix. See runbooks/README.md and runbooks/serving/httproute.yaml.
|
||||
#
|
||||
# POST-PLATFORM TIER (dependsOn platform-config), NON-GATING (wait:false) — the same placement and
|
||||
# the same reasoning as homepage/headlamp/oauth2-proxy. It needs the wildcard TLS cert, the shared
|
||||
# Gateway (including the `https-apex` listener) and split-horizon DNS, all of which arrive with
|
||||
# `platform` + `platform-config`; and a leaf web server must never be able to stall the platform
|
||||
# chain, which is the whole lesson of DECISIONS-LOG's "Headlamp gating stall". wait:false guarantees
|
||||
# its health can never gate anything even if something is later mistakenly pointed dependsOn it.
|
||||
#
|
||||
# NO postBuild.substituteFrom — THE ONE DELIBERATE DIVERGENCE FROM EVERY OTHER TIER, AND THE MOST
|
||||
# IMPORTANT LINE IN THIS FILE.
|
||||
# This tier renders the runbook PROSE into a ConfigMap, and runbooks are full of shell: `$POD`,
|
||||
# `$(kubectl ...)`, and — unavoidably, since they document these very annotations — the literal text
|
||||
# `${BASE_DOMAIN}`. Flux's postBuild substitution runs envsubst over the ENTIRE built stream, and it
|
||||
# FAILS OPEN: an undefined `${X}` becomes the empty string while the Kustomization still reports
|
||||
# Ready (DECISIONS-LOG 2026-09-01 — that is how `k8sServiceHost` was emptied and Cilium broke on 2
|
||||
# of 3 dev nodes). Enabled here it would silently mutilate the documentation: every braced expansion
|
||||
# in every command example blanked, in a file nobody re-reads after writing. Worse, the L0 check
|
||||
# `conformance/tests/l0-static/flux-postbuild-substitution` scans only files reachable through
|
||||
# kustomization `resources:`, not `configMapGenerator` inputs, so the damage would be invisible to
|
||||
# the gate as well.
|
||||
# The fix is to have nothing to substitute. The only knob this tier would have wanted is the apex
|
||||
# hostname, and runbooks/serving/httproute.yaml avoids needing it by declaring NO `hostnames:` and
|
||||
# inheriting the `https-apex` listener's own hostname instead. So this tier is
|
||||
# environment-independent by construction and R-ENV-02 ("authored once, promoted dev -> prod") holds
|
||||
# without substitution rather than because of it.
|
||||
# IF A FUTURE KNOB EVER GENUINELY NEEDS SUBSTITUTION HERE: do not add substituteFrom to this
|
||||
# Kustomization. Split the serving manifests into their own substituted tier and leave the content
|
||||
# ConfigMap in an unsubstituted one, or the runbooks lose their shell examples.
|
||||
#
|
||||
# NO SOPS decryption stanza: this tier carries no encrypted Secret (markdown plus five plain
|
||||
# manifests), so — like homepage — neither clusters/dev nor clusters/prod patches it.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: runbooks
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: platform-config # wildcard TLS + Gateway (incl. the https-apex listener) first
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./runbooks
|
||||
@@ -106,11 +106,8 @@ patches:
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml).
|
||||
# It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped
|
||||
# MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption
|
||||
# stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by
|
||||
# BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is
|
||||
# out-of-band (name only).
|
||||
# It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS
|
||||
# decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
@@ -225,67 +222,20 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CRAFTY_L4_VIP
|
||||
value: "192.168.1.55"
|
||||
# Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes
|
||||
# (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes).
|
||||
# Restated verbatim rather than inherited, per this overlay's "supply every knob" convention.
|
||||
# Dev is therefore the environment in which the bulk manifests can actually be VALIDATED
|
||||
# end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=`
|
||||
# change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses.
|
||||
- op: replace
|
||||
path: /data/BULK_NFS_SERVER
|
||||
value: "192.168.1.9"
|
||||
# Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
|
||||
# barmanObjectStore and Velero's backupStorageLocation write to.
|
||||
# OFF-CLUSTER BACKUP TARGET — no op here any more, deliberately.
|
||||
#
|
||||
# DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07).
|
||||
# Prod could not use it at all — it cannot even route to it — but dev's move is a separate,
|
||||
# operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write
|
||||
# a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window
|
||||
# would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo
|
||||
# (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_ENDPOINT
|
||||
value: "http://192.168.1.9:9000"
|
||||
# DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same
|
||||
# reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base
|
||||
# defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones
|
||||
# its proven restore depends on — without any manifest looking wrong.
|
||||
# This used to pin the on-Proxmox MinIO (http://192.168.1.9:9000). As of 2026-09-05 the operator
|
||||
# moved backups to Cloudflare R2 "for the time being until a better on-prem backup solution is
|
||||
# created", and R2 is a single environment-INDEPENDENT endpoint: both clusters write to the same
|
||||
# buckets and are isolated by VELERO_BACKUP_PREFIX / CNPG_BACKUP_PATH (dev vs prod), not by URL.
|
||||
# Under R-ENV-02 a value that does not differ per environment belongs in base and must NOT be
|
||||
# restated here — two copies of one endpoint is just a rotation waiting to drift.
|
||||
#
|
||||
# `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO
|
||||
# ignores the value, but changing it is a needless risk against a live, working backup path.
|
||||
# An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already
|
||||
# are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified
|
||||
# against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_REGION
|
||||
value: "us-east-1"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_BUCKET
|
||||
value: "velero"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_PREFIX
|
||||
value: ""
|
||||
- op: replace
|
||||
path: /data/CNPG_BACKUP_PATH
|
||||
value: "s3://cnpg"
|
||||
# WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets
|
||||
# (velero-cloud-credentials, <db>-backup-credentials) — the scoped MinIO keys dev's Velero and
|
||||
# its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod
|
||||
# renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead.
|
||||
#
|
||||
# RESTATED EXPLICITLY even though it equals the base default, for the same reason as
|
||||
# EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED
|
||||
# base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev
|
||||
# signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on
|
||||
# turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR
|
||||
# restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential.
|
||||
#
|
||||
# DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator,
|
||||
# 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING
|
||||
# backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs.
|
||||
- op: replace
|
||||
path: /data/BACKUP_CREDENTIALS_SUFFIX
|
||||
value: ""
|
||||
# *** MERGING THIS RESETS DEV'S PITR WINDOW. *** Dev's R2 prefix starts EMPTY, so Velero writes a
|
||||
# fresh kopia repository and CNPG takes new base backups; the 30-day Barman window rebuilds from
|
||||
# zero. The existing MinIO copy is NOT deleted and stays restorable — point S3_BACKUP_ENDPOINT
|
||||
# back at http://192.168.1.9:9000 (and VELERO_BACKUP_PREFIX back to "") to reach it. Do not merge
|
||||
# this while dev's proven restore is the only recovery path you are relying on.
|
||||
# dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas.
|
||||
- op: replace
|
||||
path: /data/LONGHORN_DEFAULT_REPLICA_COUNT
|
||||
|
||||
@@ -185,56 +185,9 @@ patches:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# CUTOVER DONE 2026-09-06 (operator direction: "move the repo to prod"). Prod now reconciles
|
||||
# from PROD'S OWN in-cluster Gitea. The workstation bootstrap container
|
||||
# (http://192.168.2.132:3000) is retired as prod's source — it was a podman container on the
|
||||
# operator's workstation holding a DHCP lease on WIFI, i.e. production's source of truth was a
|
||||
# laptop. It is kept only as a recovery path (bootstrap/gitea/bootstrap-gitea.sh).
|
||||
#
|
||||
# THIS LINE IS SELF-REFERENTIAL AND THAT IS THE WHOLE DIFFICULTY. The root Kustomization
|
||||
# renders ./clusters/prod, which contains this patch — so a live `kubectl patch` of
|
||||
# GitRepository.spec.url is REVERTED on the next reconcile unless the authored value moves
|
||||
# too. The cutover therefore had to be: (1) push the repo to prod's Gitea, (2) fast-forward
|
||||
# its `main` to the exact commit prod was already running so the source swap changed nothing
|
||||
# else, (3) live-patch the url to open a window, (4) land THIS commit through that window so
|
||||
# the authored value agrees. Dev hit the identical trap on 2026-09-05
|
||||
# (infra/PROD-PROVISIONING.md §10a "D0/D1 as executed").
|
||||
#
|
||||
# Verified before the switch: a throwaway pod INSIDE prod resolved this host through
|
||||
# in-cluster CoreDNS to the Gateway VIP and cloned over the public LE cert with no CA
|
||||
# supplied. The workstation being able to reach it proves nothing about pod egress.
|
||||
# MOVED WITH THE APEX FLIP (2026-09-13) — and it had to move in the SAME fetch.
|
||||
# This URL is a hardcoded literal, NOT templated on ${BASE_DOMAIN}, so the flip did not
|
||||
# carry it. But the flip DOES rename prod's own HTTPRoute hostnames, so:
|
||||
# - external-dns (policy=sync) deletes gitea.prod.home.keithjackson.dev, and
|
||||
# - platform/coredns/coredns-custom.yaml answers the whole ${BASE_DOMAIN} zone with
|
||||
# ${GATEWAY_VIP}, so in-cluster resolution moves from prod.home. to home.
|
||||
# Leaving this on the old name would therefore have severed prod from its OWN GitOps
|
||||
# source moments after the flip reconciled — self-reference, the same class of trap as
|
||||
# patching this url live and having the root Kustomization revert it. The fix has to land
|
||||
# THROUGH THE OLD SOURCE, which is why it is in the same push as the flip rather than
|
||||
# after it.
|
||||
# FOLLOW-UP worth doing: point this at the in-cluster Service
|
||||
# (gitea-http.gitea.svc.cluster.local) so prod's GitOps stops depending on public DNS and
|
||||
# on its own Gateway at all. Deferred: it needs the TLS/credential path rethought.
|
||||
# IN-CLUSTER SERVICE, NOT A PUBLIC HOSTNAME (2026-09-13). This is the durable fix for a
|
||||
# deadlock the apex flip walked straight into, and it is worth stating precisely.
|
||||
#
|
||||
# The url used to be a public name. The flip renamed prod's HTTPRoute hostnames, so the
|
||||
# name this GitRepository pointed at stopped serving — but the HTTPRoute that WOULD have
|
||||
# served the new name lives in a DOWNSTREAM tier, which cannot reconcile while the source
|
||||
# is broken. Source down -> routes never rename -> source stays down. Observed live:
|
||||
# `srcReady=False`, "unable to list remote for https://gitea.home.keithjackson.dev".
|
||||
#
|
||||
# Pointing at the Service breaks the cycle permanently: prod's GitOps no longer depends on
|
||||
# public DNS, on Cloudflare, on its own Gateway, or on ${BASE_DOMAIN} at all. A future
|
||||
# domain change cannot sever prod from its own source again.
|
||||
#
|
||||
# Plain http is deliberate and safe here: it is cluster-internal traffic to a ClusterIP,
|
||||
# TLS terminates at the Gateway for external clients only, the GitRepository has no
|
||||
# secretRef (the repo is cloned anonymously), and every secret in the tree is SOPS-encrypted
|
||||
# at rest regardless of transport.
|
||||
url: http://gitea.gitea.svc.cluster.local:3000/Home-Network/home-network-k3s-impl.git
|
||||
# TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block
|
||||
# above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final.
|
||||
url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
- target:
|
||||
@@ -272,30 +225,20 @@ patches:
|
||||
# `<svc>.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they
|
||||
# would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# THE APEX INVERSION IS DELIBERATELY NOT ON THIS BRANCH. Moving prod to the bare apex (and dev
|
||||
# to dev.home.) is a LIVE MIGRATION of the dev cluster, not a variable change — see
|
||||
# infra/PROD-PROVISIONING.md §10a. It is kept as a single commit on branch
|
||||
# `migration/dns-apex-inversion` (9137bbd) and is applied as step D3 of that runbook, with the
|
||||
# operator present. Everything else on this branch is independent of it, which is the whole
|
||||
# reason it was split out.
|
||||
#
|
||||
# [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml.
|
||||
# The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is
|
||||
# set Ansible-side from that variable and cannot read this one. If they disagree, kubectl SSO
|
||||
# trusts a different Keycloak than the one prod actually runs.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
# APEX AS OF 2026-09-13. Prod now serves home.keithjackson.dev directly; dev has been
|
||||
# decommissioned and released the name. Prerequisites completed and VERIFIED before this
|
||||
# flip, in this order:
|
||||
# 1. dev's 18 A records released by its own external-dns (policy=sync, sources deleted)
|
||||
# 2. dev's 18 LEGACY UNPREFIXED TXT registry entries deleted from Cloudflare by hand —
|
||||
# external-dns v0.21.0 removed only the `a-` prefixed form and then reported "All
|
||||
# records are already up to date", so it would NEVER have cleared them. They carried
|
||||
# `owner=default`, and prod runs `owner=prod`, so prod could not have claimed these
|
||||
# names while they existed. Confirmed gone at the AUTHORITY (betty.ns.cloudflare.com);
|
||||
# 1.1.1.1 served stale cache for minutes afterwards and must not be trusted here.
|
||||
# 3. dev's `*.home.keithjackson.dev` + apex Certificate deleted, so it stops renewing.
|
||||
# It was issued ~18d ago, so prod's request is #1 this week against LE's duplicate
|
||||
# limit of 5 per identical name set per week.
|
||||
# NOTE the zone has wildcards `*.keithjackson.dev` AND `*.home.keithjackson.dev` pointing
|
||||
# at the public IP 136.51.115.252 (legacy non-k3s prod). Dev's specific records were
|
||||
# SHADOWING that wildcard; prod's new specific records shadow it again. The legacy stack
|
||||
# lives on `*.keithjackson.dev` and is untouched by any of this.
|
||||
value: "home.keithjackson.dev"
|
||||
value: "prod.home.keithjackson.dev"
|
||||
# SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's
|
||||
# OWN subtree, so prod cannot create, update or delete any record outside it no matter what the
|
||||
# TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for
|
||||
@@ -441,145 +384,37 @@ patches:
|
||||
path: /data/CRAFTY_L4_VIP
|
||||
value: "192.168.2.22"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. ***
|
||||
# OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05).
|
||||
#
|
||||
# THERE IS DELIBERATELY NO `S3_BACKUP_ENDPOINT` OP HERE, and that is a reversal of the rule
|
||||
# stated at the top of this overlay ("an OMITTED op means prod silently inherits a dev
|
||||
# address"). That rule exists because base defaults are dev's LAN addresses. R2 is not one: it
|
||||
# is a single shared, environment-independent endpoint, so under R-ENV-02 it belongs in base and
|
||||
# overriding it with an identical value would only create two places to change on a rotation —
|
||||
# the exact drift this repo has been bitten by before. Isolation does NOT come from the
|
||||
# endpoint; it comes from the two path knobs below.
|
||||
#
|
||||
# WHY R2 AND NOT THE MinIO the operator first named: prod is on the SOHO network
|
||||
# 192.168.2.0/24, which is pfSense's WAN, while the MinIO sits inside pfSense's LAN
|
||||
# 192.168.1.0/24. Measured from a pod INSIDE prod (the workstation proves nothing about pod
|
||||
# egress): `curl http://192.168.1.9:9000/minio/health/live` → connection timed out after 15s.
|
||||
# Reaching it would need a WAN→LAN rule that does not exist. R2 needs no firewall change, and
|
||||
# unlike that MinIO — which runs on the same Proxmox host as dev's VMs — it is genuinely
|
||||
# off-site. Verify prod can reach the new endpoint before resuming velero:
|
||||
# bootstrap/standup.sh ... --s3-endpoint <the R2 endpoint> s3check
|
||||
#
|
||||
# THE PATH KNOBS ARE WHAT MAKE ONE SHARED BUCKET SAFE. Nothing about the paths was
|
||||
# per-environment before 2026-09-05: Velero used one unprefixed bucket, and every CNPG Cluster
|
||||
# used `s3://cnpg` with serverName defaulting to the CLUSTER NAME — and both environments have
|
||||
# keycloak-db / nextcloud-db / semaphore-db. Without these knobs prod would (a) adopt and
|
||||
# TTL-delete dev's Velero backups and (b) interleave two Postgres servers' WAL into ONE Barman
|
||||
# stream, corrupting the chain and destroying PITR for both. See clusters/base/cluster-vars.yaml.
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr,
|
||||
# radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud).
|
||||
#
|
||||
# THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface,
|
||||
# vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN.
|
||||
# 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102;
|
||||
# 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate
|
||||
# outcomes give DIFFERENT values here:
|
||||
# port-forward 192.168.2.102:2049 -> 192.168.1.9:2049 => "192.168.2.102"
|
||||
# a WAN->LAN route to 192.168.1.0/24 => "192.168.1.9"
|
||||
# Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess
|
||||
# ever became reachable — mounts something unintended. So it stays a placeholder.
|
||||
#
|
||||
# WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and
|
||||
# NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4
|
||||
# over v3 was in part this decision's cost: v3 would have needed four ports and three of them
|
||||
# on ephemeral numbers.
|
||||
#
|
||||
# WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes.
|
||||
# Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward:
|
||||
# the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN
|
||||
# address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via
|
||||
# 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node
|
||||
# addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md.
|
||||
#
|
||||
# *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. ***
|
||||
# A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this
|
||||
# placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The
|
||||
# failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server
|
||||
# TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in
|
||||
# ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at
|
||||
# apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles
|
||||
# a revision containing the bulk volumes; the dev overlay renders a working value, which is
|
||||
# where these manifests are meant to be proven first.
|
||||
- op: replace
|
||||
path: /data/BULK_NFS_SERVER
|
||||
# 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox
|
||||
# host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot
|
||||
# reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and
|
||||
# keeps the base value 192.168.1.9.
|
||||
#
|
||||
# PROVEN from k3s-prod-1 on 2026-09-12, not inferred:
|
||||
# - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp
|
||||
# - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export
|
||||
# carries no_root_squash, so a root write would have passed and proved nothing, while
|
||||
# LSIO images default to PUID 911 and would have gone Ready then failed every import
|
||||
# with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer.
|
||||
# - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get
|
||||
# real atomic moves instead of silently degrading to copy-then-delete on EXDEV.
|
||||
# pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer),
|
||||
# so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP.
|
||||
value: "192.168.2.102"
|
||||
# ==========================================================================================
|
||||
# OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
|
||||
# ==========================================================================================
|
||||
# Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the network and is NOT an
|
||||
# option for prod, for two independent reasons:
|
||||
# 1. UNREACHABLE. Prod is on the SOHO network 192.168.2.0/24 — pfSense's WAN — while that
|
||||
# MinIO is inside pfSense's LAN. Measured from a pod INSIDE prod (the workstation proves
|
||||
# nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` → connection
|
||||
# timed out after 15s. Reaching it would need a WAN→LAN rule that does not exist.
|
||||
# 2. NOT OFF-CLUSTER DR. It runs on the same Proxmox host as dev's VMs, so it already shares
|
||||
# a physical failure domain with dev. R2 is genuinely off-site, and its free egress means
|
||||
# restore REHEARSALS cost nothing — R-STATE-03 has no financial disincentive to stay proven.
|
||||
#
|
||||
# THIS UNBLOCKS AN ACTIVE INCIDENT, not just a roadmap item. All three prod CNPG Clusters ship
|
||||
# `archive_mode=on` with `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT`, so barman-cloud-wal-archive
|
||||
# exits 4 on every segment and Postgres — correctly — REFUSES TO RECYCLE WAL. Verified live
|
||||
# 2026-09-07: keycloak-db / nextcloud-db / semaphore-db all report
|
||||
# ContinuousArchiving=False/ContinuousArchivingFailing, and keycloak-db's 5 GiB PVC is at
|
||||
# 579 MiB used / 4360 MiB free and climbing monotonically. A disk-full PANIC there takes the
|
||||
# cluster's SSO database down, and with it kubectl OIDC, Grafana, Headlamp, Gitea and
|
||||
# forward-auth. Fixing the endpoint lets Postgres drain the whole backlog.
|
||||
#
|
||||
# ENDPOINT IS ACCOUNT-SCOPED, so it stays a prod-overlay value even though it is not a LAN
|
||||
# address: this is the operator's Cloudflare account id, confirmed from inside a prod pod
|
||||
# (`head-bucket` on k3s-prod-backup → BucketRegion WNAM; PUT and DELETE both succeed).
|
||||
# It is NOT put in base, because base is dev's live MinIO and dev is deliberately STAYING
|
||||
# there — moving dev to R2 would reset its PITR window and orphan the backups its PROVEN
|
||||
# restore depends on. Two environments, two targets, isolated by URL *and* by bucket.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_ENDPOINT
|
||||
value: "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com"
|
||||
# R2 requires the region string to be exactly `auto`. (`head-bucket` answers
|
||||
# `{"BucketRegion":"WNAM"}` — that is a PHYSICAL LOCATION HINT, not a signing region; using it
|
||||
# here produces AuthorizationHeaderMalformed.) Dev keeps the base `us-east-1` its live MinIO
|
||||
# has always been signed with.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_REGION
|
||||
value: "auto"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# OBJECT TENANCY INSIDE THE ONE BUCKET. The operator created a SINGLE R2 bucket, so velero and
|
||||
# barman cannot be separated by bucket the way they are on dev's MinIO (`velero` and `cnpg`).
|
||||
# They are separated by disjoint TOP-LEVEL PREFIXES instead:
|
||||
#
|
||||
# s3://k3s-prod-backup/velero/… Velero: kopia repo + backup metadata
|
||||
# s3://k3s-prod-backup/barman/<clusterName>/… CNPG: WAL + base backups
|
||||
#
|
||||
# The bucket ROOT is left empty on purpose — a misrendered prefix then lands somewhere
|
||||
# obviously wrong instead of silently colliding with a live tenant. Velero's retention TTL
|
||||
# deletes objects under its own prefix only, so it can never reap a WAL segment; and barman's
|
||||
# per-cluster serverName subdirectory keeps the three Postgres servers' chains apart, which
|
||||
# matters because interleaving two servers' WAL into one prefix corrupts the chain outright.
|
||||
# Dev cannot be reached from any of these: different endpoint AND different bucket names.
|
||||
# ------------------------------------------------------------------------------------------
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_BUCKET
|
||||
value: "k3s-prod-backup"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_PREFIX
|
||||
value: "velero"
|
||||
value: "prod"
|
||||
# Barman objects land at s3://cnpg/prod/<clusterName>/; dev's at s3://cnpg/dev/<clusterName>/.
|
||||
- op: replace
|
||||
path: /data/CNPG_BACKUP_PATH
|
||||
value: "s3://k3s-prod-backup/barman"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# WHICH CREDENTIAL SIGNS THE REQUESTS TO THAT BUCKET (per-env credential split, 2026-09-07).
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The knobs above moved the ENDPOINT and the OBJECT LAYOUT per environment. The credential did
|
||||
# not move with them, and it could not: the four backup-credential Secrets are SHARED BASE
|
||||
# manifests, applied to both clusters by the same Flux tiers. Writing the R2 token into them
|
||||
# gave prod a working key by TAKING DEV'S AWAY — dev would have signed MinIO requests with an
|
||||
# R2 key id, and under archive_mode=on that is a filling pgdata PVC, not a missing backup.
|
||||
#
|
||||
# So each credential is now two objects, and this suffix selects prod's:
|
||||
# velero-cloud-credentials-prod the R2 token in the AWS INI `cloud` shape
|
||||
# <db>-backup-credentials-prod the same token as ACCESS_KEY_ID / ACCESS_SECRET_KEY
|
||||
# for keycloak-db, nextcloud-db and semaphore-db. Both halves ship to both clusters (they are
|
||||
# base resources); ONLY this value decides which one each consumer reads. It is a SUFFIX rather
|
||||
# than four secret-name knobs deliberately — four could be updated three-quarters of the way,
|
||||
# leaving one consumer on dev's MinIO key with nothing rendering wrong, and strict-mode
|
||||
# substitution cannot catch a defined-but-stale variable.
|
||||
#
|
||||
# THE R2 TOKEN IS THE ONE PROVEN FROM INSIDE PROD (chore/r2-credentials 0a6a862): head-bucket on
|
||||
# k3s-prod-backup returns BucketRegion WNAM, and PUT and DELETE both succeed. Rotating it now
|
||||
# touches ONLY the four `-prod` files; dev's MinIO keys are untouched objects. That isolation is
|
||||
# the structural reason for the split, independent of dev's eventual decommission.
|
||||
- op: replace
|
||||
path: /data/BACKUP_CREDENTIALS_SUFFIX
|
||||
value: "-prod"
|
||||
value: "s3://cnpg/prod"
|
||||
# TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist.
|
||||
@@ -23,9 +23,6 @@ behavioral / L3 fault-injection** (this harness).
|
||||
./run.sh --strict-coverage # hygiene gate: fail on tag/annotation drift
|
||||
./run.sh --require-all # phase-exit gate: every requirement must be covered
|
||||
./run.sh --record # run AND capture the evidence (see "Proof of pass")
|
||||
./run.sh --layer l0 --include-scope whole-repo # union: the tagged subset PLUS every test
|
||||
# declaring `conformance.home.k3s/scope: whole-repo`
|
||||
./run.sh --layer l0 --print-selection # print the dirs the selection resolves to; run nothing
|
||||
```
|
||||
|
||||
**Gating is scoped, not "all 51 always"** (R-TEST-03). Three distinct checks:
|
||||
@@ -35,29 +32,6 @@ behavioral / L3 fault-injection** (this harness).
|
||||
- `--requirements R-…` — **merge gate**: selects the tagged tests to run **and** asserts
|
||||
those touched requirements are covered (a listed safety-critical one must have an L3 test).
|
||||
- `--require-all` — **phase-exit gate**: every registry requirement must be covered.
|
||||
- `--include-scope whole-repo` — **the escape hatch scoping needs**, because scoping has one
|
||||
blind spot and it is the worst-shaped kind. Some L0 invariants quantify over the WHOLE
|
||||
tracked tree (no plaintext secrets anywhere; no RBAC wildcard anywhere), or over subtrees
|
||||
the coarse path→requirement map keys to **nothing** (`apps/`, `infra/`, most of
|
||||
`platform/`). For those, *the change most likely to break the invariant attributes no
|
||||
requirement at all*, so requirement-scoped selection filters the check out of exactly that
|
||||
change. Such a test **declares** its scope:
|
||||
|
||||
```yaml
|
||||
metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
```
|
||||
|
||||
and `--include-scope whole-repo` unions every declaring test into the selection. The CI
|
||||
stages (`ci/l0.sh`, `ci/l1.sh`, `ci/conformance.sh`) all pass it, so **adding the
|
||||
annotation is the only step** — there is nothing to remember and nothing to hand-write.
|
||||
Two consequences, both deliberate: a declaring test always runs, and a **SKIP of a
|
||||
declaring test is UNWIRED, not green** (the runner exits 3, which `ci/l0.sh` turns into the
|
||||
gate's own rc 3 — never a pass). `tests/l0-static/whole-repo-invariants-wired` proves the
|
||||
mechanism end-to-end and falsifiably, so it cannot rot silently. This replaced a
|
||||
hand-written "UNCONDITIONAL" stage per check in `ci/l0.sh`; see `lib/common.sh` for the
|
||||
full rationale and the incidents behind it.
|
||||
- Documented out-of-scope gaps (e.g. `R-ENV-02` dev→prod promote and `R-GITOPS-01`
|
||||
state-in-git, owned by WS-INFRA/WS-GITOPS and keyed wholesale by the coarse path map)
|
||||
are exempted from the completeness gates via `--allow-gap`, wired in `run.sh` as
|
||||
|
||||
@@ -47,62 +47,6 @@ conf_dirs_for_requirements() {
|
||||
done < <(conf_all_test_files) | sort -u
|
||||
}
|
||||
|
||||
# --- DECLARED SCOPE: whole-repo invariants -------------------------------------------
|
||||
# A test may also declare the SCOPE of the invariant it asserts:
|
||||
#
|
||||
# metadata:
|
||||
# annotations:
|
||||
# conformance.home.k3s/scope: "whole-repo"
|
||||
#
|
||||
# WHY THIS EXISTS — the requirement-scoped-selection blind spot (R-TEST-03)
|
||||
# ------------------------------------------------------------------------
|
||||
# The merge gate runs the conformance subset tagged with the requirements a change
|
||||
# TOUCHES (ci/touched-requirements.sh -> `--requirements`). That is right for an invariant
|
||||
# whose input surface is the subtree its requirements map to. It is WRONG — silently, and in
|
||||
# the worst direction — for an invariant that quantifies over the WHOLE tracked tree, or over
|
||||
# subtrees (apps/, platform/<most>, infra/) that the coarse path->requirement map keys to
|
||||
# NOTHING. For those, the change most likely to break the invariant attributes no requirement
|
||||
# at all, so the requirement-scoped selection filters the check out of EXACTLY that change.
|
||||
# Three real incidents came out of that blind spot; each is recorded in the header of the
|
||||
# test that now guards it (flux-postbuild-substitution, keycloak-client-secret-reconciler,
|
||||
# bulk-tier-distinct-from-replicated).
|
||||
#
|
||||
# The repo's previous answer was a hand-written "UNCONDITIONAL" stage per such test in
|
||||
# ci/l0.sh — which does not scale and fails silently in the worst way: whoever adds the NEXT
|
||||
# whole-repo invariant must REMEMBER to hand-write a stage, and if they forget, the check runs
|
||||
# only when some unrelated requirement happens to be touched. The scope annotation replaces
|
||||
# that with a DECLARATION the runner enforces: `run.sh --include-scope whole-repo` unions
|
||||
# every test carrying it into the selection, so adding the annotation is the only step.
|
||||
#
|
||||
# TWO things follow from the annotation, and both are deliberate:
|
||||
# 1. the test is ALWAYS selected (union with the requirement-scoped subset), and
|
||||
# 2. a SKIP of that test is UNWIRED, NOT GREEN (lib/run_l1_offline.py --unwired-scope).
|
||||
# A whole-repo invariant that reports "nothing to audit" is reporting that the subtree
|
||||
# it audits has VANISHED, which is not a pass. This is the rc-3 semantics the bespoke
|
||||
# stages encoded ("check SKIPPED (clusters/ absent) — not a pass"), preserved verbatim.
|
||||
export CONF_SCOPE_ANNOTATION="${CONF_LABEL_PREFIX}/scope"
|
||||
|
||||
# Directories of tests DECLARING the given scope (e.g. "whole-repo"). The annotation is
|
||||
# matched on a real YAML mapping line only: a `#` comment cannot start with the key, so a
|
||||
# scope named in a test's header prose is documentation, not a declaration (the same
|
||||
# "reference, not declaration" rule ci/touched-requirements.sh applies to R-IDs).
|
||||
conf_dirs_for_scope() {
|
||||
local scope="$1" f
|
||||
[ -n "$scope" ] || return 0
|
||||
while IFS= read -r f; do
|
||||
[ -n "$f" ] || continue
|
||||
if grep -Eq "^[[:space:]]*${CONF_SCOPE_ANNOTATION}:[[:space:]]*[\"']?${scope}[\"']?[[:space:]]*$" "$f"; then
|
||||
dirname "$f"
|
||||
fi
|
||||
done < <(conf_all_test_files) | sort -u
|
||||
}
|
||||
|
||||
# Union of conf_dirs_for_scope over a comma/space separated list of scopes.
|
||||
conf_dirs_for_scopes() {
|
||||
local scopes="$1" s
|
||||
for s in ${scopes//,/ }; do conf_dirs_for_scope "$s"; done | sort -u
|
||||
}
|
||||
|
||||
# Directories for a given layer (subtree): l0 | l1 | l2 | l3 | meta | all.
|
||||
# Accepts a comma/space separated LIST of layers (e.g. "l2,l3") and returns the UNION of
|
||||
# their test dirs, deduplicated. A single layer keeps its prior behavior (backward compatible);
|
||||
|
||||
@@ -467,14 +467,7 @@ def git_sha(repo: Path) -> str:
|
||||
capture_output=True, text=True, check=True).stdout.strip()
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
return "unknown"
|
||||
# EXCLUDE conformance/runs/. It is TRACKED, so `--record` writing ledger.jsonl dirtied the
|
||||
# tree and every LATER run in the same session was stamped `-dirty` — recording one test
|
||||
# poisoned the next, and check.sh refuses to honour a dirty proven row. That cost a second
|
||||
# live fault-injection run on prod to work around. The docstring above already states the
|
||||
# intent for the untracked runs/<id>/ half of this same artifact; the ledger is the tracked
|
||||
# half. Mirrors status.py's IMPL_EXCLUDE_PREFIXES, which lists this path for the same reason.
|
||||
dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--",
|
||||
".", ":(exclude)conformance/runs/"],
|
||||
dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--"],
|
||||
capture_output=True).returncode != 0
|
||||
return f"{sha}-dirty" if dirty else sha
|
||||
|
||||
|
||||
@@ -19,30 +19,6 @@ Exit 0 iff every selected offline check passes.
|
||||
|
||||
The per-dir check script is whichever of `render-check.sh` / `check.sh` the directory
|
||||
carries; a dir with neither is reported SKIPPED (never a pass).
|
||||
|
||||
DECLARED SCOPE, AND WHY A SKIP IS NOT ALWAYS HARMLESS (--unwired-scope)
|
||||
----------------------------------------------------------------------
|
||||
Exit 0 here means "every selected check passed"; a check that exits 3 is SKIPPED and, by
|
||||
default, does not fail the run (a scaffold with no check script, an audit whose subject
|
||||
subtree does not exist yet). That default is WRONG for a WHOLE-REPO invariant. Those are the
|
||||
checks ci/l0.sh used to run as hand-written "UNCONDITIONAL" stages, and each of those stages
|
||||
mapped rc 3 to the GATE's rc 3 with the words "not a pass":
|
||||
|
||||
3) skip "substitution check SKIPPED (clusters/ absent) — not a pass"
|
||||
summary "- subst-safety: **SKIPPED** (nothing to audit)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi
|
||||
|
||||
A whole-repo check reporting "nothing to audit" is reporting that the tree it audits has
|
||||
VANISHED — which is a louder problem than the check failing, not a quieter one. So:
|
||||
|
||||
--unwired-scope SCOPE a test DECLARING `<prefix>/scope: SCOPE` whose check SKIPS (rc 3,
|
||||
or no check script at all) makes this runner exit 3 instead of 0.
|
||||
Hard failures still win (exit 1); rc 3 never masks one.
|
||||
|
||||
Exit codes: 0 all selected checks passed / 1 at least one FAILED / 3 nothing failed but a
|
||||
scope-declared check was UNWIRED. conformance/run.sh propagates 3 verbatim; ci/l0.sh turns it
|
||||
into the gate's own rc 3. This is the ONE place the "unwired, not green" rule lives now,
|
||||
instead of once per hand-written stage.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -60,24 +36,6 @@ LABEL_LINE_RE_TMPL = r"^\s*{prefix}/(R-[A-Z0-9]+-[0-9]+)\s*:\s*[\"']?true"
|
||||
CHECK_SCRIPTS = ("render-check.sh", "check.sh")
|
||||
|
||||
|
||||
def scope_of(test_yaml: Path, annotation: str) -> str:
|
||||
"""The scope a test DECLARES, or "" when it declares none.
|
||||
|
||||
Matched on a real YAML mapping line: a `#` comment cannot begin with the annotation key,
|
||||
so a scope named in the test's header prose is documentation, not a declaration — the
|
||||
same "reference, not declaration" rule ci/touched-requirements.sh applies to R-IDs.
|
||||
Mirrors conf_dirs_for_scope() in lib/common.sh; they must agree.
|
||||
"""
|
||||
if not test_yaml.exists():
|
||||
return ""
|
||||
rx = re.compile(r"^\s*" + re.escape(annotation) + r"\s*:\s*[\"']?([A-Za-z0-9_.-]+)[\"']?\s*$")
|
||||
for line in test_yaml.read_text().splitlines():
|
||||
m = rx.match(line)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def layer_of(d: Path) -> str:
|
||||
"""Layer for the JUnit properties, from the subtree the test lives in."""
|
||||
rel = str(d)
|
||||
@@ -101,38 +59,25 @@ def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--junit", type=Path, required=True)
|
||||
ap.add_argument("--label-prefix", default="conformance.home.k3s")
|
||||
ap.add_argument("--scope-annotation", default="conformance.home.k3s/scope",
|
||||
help="annotation key a test uses to DECLARE its invariant's scope")
|
||||
ap.add_argument("--unwired-scope", action="append", default=[], metavar="SCOPE",
|
||||
help="a test declaring this scope that SKIPS is UNWIRED, not green "
|
||||
"(makes this runner exit 3); repeatable")
|
||||
ap.add_argument("--summary-out", type=Path, default=None,
|
||||
help="write one Markdown line per case here, for the CI step summary")
|
||||
ap.add_argument("dirs", nargs="+", type=Path)
|
||||
args = ap.parse_args()
|
||||
|
||||
strict_scopes = set(args.unwired_scope)
|
||||
|
||||
cases = [] # (name, reqs, dir, rc, output, scope)
|
||||
cases = [] # (name, reqs, dir, rc, output)
|
||||
for d in args.dirs:
|
||||
check = next((d / s for s in CHECK_SCRIPTS if (d / s).exists()), None)
|
||||
name, reqs = collect_meta(d / "chainsaw-test.yaml", args.label_prefix)
|
||||
scope = scope_of(d / "chainsaw-test.yaml", args.scope_annotation)
|
||||
if check is None:
|
||||
cases.append((name, reqs, d, 3,
|
||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
||||
scope))
|
||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}"))
|
||||
continue
|
||||
proc = subprocess.run(
|
||||
["bash", str(check)], capture_output=True, text=True
|
||||
)
|
||||
out = (proc.stdout + proc.stderr).strip()
|
||||
cases.append((name, reqs, d, proc.returncode, out, scope))
|
||||
cases.append((name, reqs, d, proc.returncode, out))
|
||||
|
||||
failures = sum(1 for c in cases if c[3] not in (0, 3))
|
||||
skipped = sum(1 for c in cases if c[3] == 3)
|
||||
# A SKIP of a scope-declared check is UNWIRED, not green (see the module docstring).
|
||||
unwired = [c for c in cases if c[3] == 3 and c[5] in strict_scopes]
|
||||
|
||||
suite = ET.Element(
|
||||
"testsuite",
|
||||
@@ -144,8 +89,7 @@ def main() -> int:
|
||||
"skipped": str(skipped),
|
||||
},
|
||||
)
|
||||
summary_lines = []
|
||||
for name, reqs, d, rc, out, scope in cases:
|
||||
for name, reqs, d, rc, out in cases:
|
||||
tc = ET.SubElement(
|
||||
suite,
|
||||
"testcase",
|
||||
@@ -156,33 +100,17 @@ def main() -> int:
|
||||
ET.SubElement(props, "property", {"name": "requirements", "value": ",".join(reqs)})
|
||||
ET.SubElement(props, "property", {"name": "layer", "value": layer_of(d)})
|
||||
ET.SubElement(props, "property", {"name": "path", "value": str(d)})
|
||||
ET.SubElement(props, "property", {"name": "scope", "value": scope})
|
||||
if rc == 3:
|
||||
ET.SubElement(tc, "skipped", {"message": out or "no offline check script"})
|
||||
elif rc != 0:
|
||||
fail = ET.SubElement(tc, "failure", {"message": f"offline check exit {rc}"})
|
||||
fail.text = out
|
||||
status = "PASS" if rc == 0 else ("SKIP" if rc == 3 else "FAIL")
|
||||
mark = f" scope={scope}" if scope else ""
|
||||
print(f"[offline] {layer_of(d):3s} {status:4s} {name} "
|
||||
f"({','.join(reqs) or 'untagged'}){mark}",
|
||||
f"({','.join(reqs) or 'untagged'})",
|
||||
file=sys.stderr)
|
||||
if rc not in (0, 3):
|
||||
print(out, file=sys.stderr)
|
||||
# One step-summary line per case, naming what ran — the generated replacement for the
|
||||
# hand-written `summary "- subst-safety: **passed**"` line each deleted stage carried.
|
||||
tag = f" ({scope})" if scope else ""
|
||||
if rc == 0:
|
||||
verdict = "**passed**"
|
||||
elif rc == 3:
|
||||
verdict = ("**SKIPPED** (UNWIRED, not green)" if scope in strict_scopes
|
||||
else "**SKIPPED** (nothing to audit)")
|
||||
else:
|
||||
verdict = f"**FAILED** (exit {rc})"
|
||||
summary_lines.append(f"- {layer_of(d).lower()}/{name}{tag}: {verdict}")
|
||||
if rc == 3 and scope in strict_scopes:
|
||||
print(f"[offline] {name}: SKIPPED, and it declares scope={scope!r} — "
|
||||
f"UNWIRED, NOT GREEN. {out}", file=sys.stderr)
|
||||
|
||||
root = ET.Element(
|
||||
"testsuites",
|
||||
@@ -198,19 +126,9 @@ def main() -> int:
|
||||
args.junit.parent.mkdir(parents=True, exist_ok=True)
|
||||
xml = minidom.parseString(ET.tostring(root, "utf-8")).toprettyxml(indent=" ")
|
||||
args.junit.write_text(xml)
|
||||
if args.summary_out is not None:
|
||||
args.summary_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.summary_out.write_text("\n".join(summary_lines) + ("\n" if summary_lines else ""))
|
||||
|
||||
print(f"[offline] wrote JUnit ({len(cases)} case(s), {failures} failure(s), "
|
||||
f"{skipped} skip(s), {len(unwired)} unwired) -> {args.junit}", file=sys.stderr)
|
||||
if failures:
|
||||
return 1
|
||||
if unwired:
|
||||
print("[offline] UNWIRED (declared scope, SKIPPED — not a pass): "
|
||||
+ ", ".join(c[0] for c in unwired), file=sys.stderr)
|
||||
return 3
|
||||
return 0
|
||||
print(f"[offline] wrote JUnit ({len(cases)} case(s), {failures} failure(s)) -> "
|
||||
f"{args.junit}", file=sys.stderr)
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+10
-82
@@ -10,12 +10,6 @@
|
||||
# ./run.sh --requirements R-NET-01,R-NET-06 # merge-gate subset (tagged)
|
||||
# ./run.sh --layer l2 # one layer (l0|l1|l2|l3|meta|all; l0/l1 need no cluster)
|
||||
# ./run.sh --requirements R-NET-01,R-NET-02 --layer l2 # intersect: tagged AND in layer
|
||||
# ./run.sh --layer l0 --requirements R-X --include-scope whole-repo
|
||||
# # UNION: the tagged subset PLUS every test that
|
||||
# # DECLARES `conformance.home.k3s/scope: whole-repo`.
|
||||
# # A whole-repo invariant must run on EVERY gate
|
||||
# # invocation — see lib/common.sh's scope block for
|
||||
# # why, and why a SKIP of one is exit 3, not green.
|
||||
# ./run.sh --coverage-only # regenerate coverage report only (no cluster)
|
||||
# ./run.sh --status # DERIVED STATUS report only (no cluster): per
|
||||
# # requirement, is it implemented, is it actually
|
||||
@@ -24,11 +18,6 @@
|
||||
# # (runs/ledger.jsonl) actually proves it — see
|
||||
# # lib/status.py. No ledger row => not-derivable.
|
||||
# ./run.sh --lint # structural-validate test defs (no cluster)
|
||||
# ./run.sh --print-selection # print the test dirs the CURRENT selection resolves
|
||||
# # to, one per line, and exit. Runs no check. This is
|
||||
# # how a test can PROVE the selection includes it
|
||||
# # (tests/l0-static/whole-repo-invariants-wired) without
|
||||
# # executing the suite and recursing into itself.
|
||||
# ./run.sh --strict-coverage # hygiene gate: fail on tag/annotation drift
|
||||
# ./run.sh --requirements R-NET-01 # ALSO asserts those IDs are covered (merge gate)
|
||||
# ./run.sh --require-all # phase-exit: every requirement must be covered
|
||||
@@ -90,7 +79,6 @@ ORIG_ARGS=("$@")
|
||||
|
||||
LAYER="all"
|
||||
REQS=""
|
||||
INCLUDE_SCOPES="" # --include-scope: scopes unioned into the selection (see lib/common.sh)
|
||||
MODE="run" # run | coverage-only | status | lint
|
||||
STRICT_COVERAGE=0
|
||||
REQUIRE_ALL=0
|
||||
@@ -134,11 +122,9 @@ while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--layer) LAYER="$2"; shift 2 ;;
|
||||
--requirements) REQS="$(conf_normalize_reqs "$2")"; shift 2 ;;
|
||||
--include-scope) INCLUDE_SCOPES="${INCLUDE_SCOPES:+${INCLUDE_SCOPES},}$2"; shift 2 ;;
|
||||
--coverage-only) MODE="coverage-only"; shift ;;
|
||||
--status) MODE="status"; shift ;;
|
||||
--lint) MODE="lint"; shift ;;
|
||||
--print-selection) MODE="print-selection"; shift ;;
|
||||
--strict-coverage) STRICT_COVERAGE=1; shift ;;
|
||||
--require-all) REQUIRE_ALL=1; shift ;;
|
||||
--no-cluster) FORCE_NO_CLUSTER=1; shift ;;
|
||||
@@ -265,38 +251,11 @@ lint_tests() {
|
||||
}
|
||||
|
||||
# --- Select which test directories Chainsaw should run --------------------------------
|
||||
# Requirement-scoped selection, then the DECLARED-SCOPE union on top:
|
||||
#
|
||||
# (--requirements ∩ --layer) ∪ (--include-scope ∩ --layer)
|
||||
#
|
||||
# The left half is R-TEST-03's touched subset. The right half is the fix for its blind
|
||||
# spot: an invariant that quantifies over the whole tracked tree cannot be selected BY the
|
||||
# requirements a change touches, because the change most likely to break it touches none of
|
||||
# them. See lib/common.sh's scope block for the incidents. The union is fail-closed by
|
||||
# construction — it can only ADD tests to a run, never remove one.
|
||||
select_dirs() {
|
||||
{
|
||||
requirement_scoped_dirs
|
||||
# The scope union is still LAYER-bounded: `--layer l0` must not pull a whole-repo-scoped
|
||||
# L2 test into an offline run. With no --layer (all), nothing is bounded away.
|
||||
if [ -n "${INCLUDE_SCOPES}" ]; then
|
||||
if [ "${LAYER}" != all ]; then
|
||||
comm -12 \
|
||||
<(conf_dirs_for_scopes "${INCLUDE_SCOPES}") \
|
||||
<(conf_dirs_for_layer "${LAYER}")
|
||||
else
|
||||
conf_dirs_for_scopes "${INCLUDE_SCOPES}"
|
||||
fi
|
||||
fi
|
||||
} | sort -u
|
||||
}
|
||||
|
||||
# The pre-existing requirement/layer selection, unchanged:
|
||||
# --requirements AND --layer together => intersection (tests carrying the tag
|
||||
# AND living in that layer subtree). Either alone keeps its prior meaning, so
|
||||
# this is backward compatible (LAYER defaults to "all"). Both helpers emit
|
||||
# sorted, de-duplicated dirs, so `comm -12` yields the intersection.
|
||||
requirement_scoped_dirs() {
|
||||
# --requirements AND --layer together => intersection (tests carrying the tag
|
||||
# AND living in that layer subtree). Either alone keeps its prior meaning, so
|
||||
# this is backward compatible (LAYER defaults to "all"). Both helpers emit
|
||||
# sorted, de-duplicated dirs, so `comm -12` yields the intersection.
|
||||
if [ -n "${REQS}" ] && [ "${LAYER}" != all ]; then
|
||||
comm -12 \
|
||||
<(conf_dirs_for_requirements "${REQS}") \
|
||||
@@ -437,12 +396,6 @@ layers_are_offline_only() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Markdown one-liner per executed offline check, for the CI step summary. Written next to
|
||||
# the JUnit; ci/l0.sh + ci/l1.sh splice it into $GITHUB_STEP_SUMMARY so the gate output still
|
||||
# NAMES every check that ran (that is what the deleted per-check gate stages did, one hard-coded
|
||||
# `summary` line each — now generated for all of them, whole-repo-scoped ones marked as such).
|
||||
: "${OFFLINE_SUMMARY_OUT:=${ARTIFACTS}/offline-summary.md}"
|
||||
|
||||
run_offline_checks() {
|
||||
local -a dirs
|
||||
mapfile -t dirs < <(offline_selected_dirs)
|
||||
@@ -450,30 +403,14 @@ run_offline_checks() {
|
||||
log "No L0/L1 offline checks match the selection (layer=${LAYER} reqs='${REQS}')."
|
||||
return 0
|
||||
fi
|
||||
# Declared scopes are passed through so the runner knows WHICH cases must not be allowed to
|
||||
# report a SKIP as green (rc 3). See lib/common.sh's scope block and run_l1_offline.py.
|
||||
local -a scope_args=() s
|
||||
for s in ${INCLUDE_SCOPES//,/ }; do scope_args+=(--unwired-scope "$s"); done
|
||||
log "Running ${#dirs[@]} offline L0/L1 check(s) (no cluster; docs/14)."
|
||||
[ -n "${INCLUDE_SCOPES}" ] && log " unconditional (declared scope ${INCLUDE_SCOPES}): $(conf_dirs_for_scopes "${INCLUDE_SCOPES}" | xargs -n1 basename 2>/dev/null | tr '\n' ' ')"
|
||||
"${PYTHON}" "${CONF_ROOT}/lib/run_l1_offline.py" \
|
||||
--junit "${JUNIT_OUT}" --label-prefix "${CONF_LABEL_PREFIX}" \
|
||||
--scope-annotation "${CONF_SCOPE_ANNOTATION}" \
|
||||
--summary-out "${OFFLINE_SUMMARY_OUT}" \
|
||||
${scope_args[@]+"${scope_args[@]}"} "${dirs[@]}"
|
||||
--junit "${JUNIT_OUT}" --label-prefix "${CONF_LABEL_PREFIX}" "${dirs[@]}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Main
|
||||
###############################################################################
|
||||
# --print-selection is deliberately handled FIRST: it resolves the selection and exits,
|
||||
# writing nothing and running no check. A test that asserts "the gate's selection cannot
|
||||
# filter me out" needs to ask that question without executing the suite it is part of.
|
||||
if [ "${MODE}" = "print-selection" ]; then
|
||||
select_dirs
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Coverage report first (always)"
|
||||
gen_coverage
|
||||
|
||||
@@ -503,26 +440,17 @@ esac
|
||||
# L0 static + L1 rendering are cluster-independent (docs/14): run the selected offline
|
||||
# checks and treat their result as authoritative REGARDLESS of cluster availability. An
|
||||
# offline check failure is a real failure (nothing cluster-gated about it), so it always blocks.
|
||||
# rc 3 is the harness-wide SKIPPED/not-run code (ci/README.md): NOT a pass, NOT a fail. The
|
||||
# offline runner returns it when a test carrying a --unwired-scope scope SKIPPED — a whole-repo
|
||||
# invariant reporting "nothing to audit" means the subtree it audits is gone, which the gate
|
||||
# must see as UNWIRED. It is propagated verbatim so ci/l0.sh can set the gate's own rc=3,
|
||||
# exactly as the deleted per-check stages did.
|
||||
OFFLINE_RC=0
|
||||
run_offline_checks || OFFLINE_RC=$?
|
||||
if layers_are_offline_only; then
|
||||
# Offline-only run (the ci/l0.sh + ci/l1.sh merge-gate paths): the offline result IS the result.
|
||||
case "${OFFLINE_RC}" in
|
||||
0) log "Done (offline ${LAYER}). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md" ;;
|
||||
3) log "Offline ${LAYER}: a scope-declared check SKIPPED — UNWIRED, not green (rc=3). JUnit: ${JUNIT_OUT}" ;;
|
||||
*) log "Offline ${LAYER} checks FAILED (rc=${OFFLINE_RC}). JUnit: ${JUNIT_OUT}" ;;
|
||||
esac
|
||||
if [ "${OFFLINE_RC}" -eq 0 ]; then
|
||||
log "Done (offline ${LAYER}). JUnit: ${JUNIT_OUT} | Coverage: ${COVERAGE_OUT}/coverage.md"
|
||||
else
|
||||
log "Offline ${LAYER} checks FAILED (rc=${OFFLINE_RC}). JUnit: ${JUNIT_OUT}"
|
||||
fi
|
||||
exit "${OFFLINE_RC}"
|
||||
fi
|
||||
if [ "${OFFLINE_RC}" -eq 3 ]; then
|
||||
log "Offline L0/L1: a scope-declared check SKIPPED — UNWIRED, not green (rc=3)."
|
||||
exit 3
|
||||
fi
|
||||
if [ "${OFFLINE_RC}" -ne 0 ]; then
|
||||
log "Offline L0/L1 checks FAILED (rc=${OFFLINE_RC}) — blocking (not cluster-gated)."
|
||||
exit "${OFFLINE_RC}"
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
<?xml version="1.0" ?>
|
||||
<testsuites name="home-k3s-conformance" tests="1" failures="0" errors="0" skipped="0">
|
||||
<testsuite name="home-k3s-conformance L0/L1 (offline)" tests="1" failures="0" errors="0" skipped="0">
|
||||
<testcase classname="L1:R-NET-05,R-NET-06,R-NET-07" name="egress-class-render">
|
||||
<properties>
|
||||
<property name="requirements" value="R-NET-05,R-NET-06,R-NET-07"/>
|
||||
<property name="layer" value="L1"/>
|
||||
<property name="path" value="/work/.claude/conf-clone/conformance/tests/l1-rendering/egress-class-render"/>
|
||||
</properties>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -1,7 +0,0 @@
|
||||
[1m[run.sh][0m Coverage report first (always)
|
||||
[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s).
|
||||
[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json
|
||||
[1m[run.sh][0m Running 1 offline L0/L1 check(s) (no cluster; docs/14).
|
||||
[offline] L1 PASS egress-class-render (R-NET-05,R-NET-06,R-NET-07)
|
||||
[offline] wrote JUnit (1 case(s), 0 failure(s)) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml
|
||||
[1m[run.sh][0m Done (offline l1). JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md
|
||||
@@ -1,4 +0,0 @@
|
||||
<testsuites name="chainsaw-report" tests="1" failures="0" errors="0" time="0.000000"><testsuite name="/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch" tests="1" failures="0" errors="0" id="0" time="">
|
||||
<testcase name="vpn-killswitch" classname="" time="174.632779" />
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -1,237 +0,0 @@
|
||||
[1m[run.sh][0m Coverage report first (always)
|
||||
[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s).
|
||||
[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json
|
||||
[1m[run.sh][0m No L0/L1 offline checks match the selection (layer=l3 reqs='R-NET-06').
|
||||
[1m[run.sh][0m LIGHT pass: 1 test(s), parallel=2 (pinned v0.2.12)
|
||||
Version: 0.2.12
|
||||
Loading config (/work/.claude/conf-clone/conformance/config/chainsaw-config.yaml)...
|
||||
- Using test file: chainsaw-test
|
||||
- TestDirs [/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch]
|
||||
- SkipDelete false
|
||||
- FailFast false
|
||||
- ReportFormat 'JUNIT-TEST'
|
||||
- ReportName 'chainsaw-report-light'
|
||||
- ReportPath '/work/.claude/conf-clone/conformance/_artifacts'
|
||||
- Namespace ''
|
||||
- FullName false
|
||||
- IncludeTestRegex ''
|
||||
- ExcludeTestRegex ''
|
||||
- ApplyTimeout 1m30s
|
||||
- AssertTimeout 3m0s
|
||||
- CleanupTimeout 2m0s
|
||||
- DeleteTimeout 2m0s
|
||||
- ErrorTimeout 1m0s
|
||||
- ExecTimeout 3m0s
|
||||
- DeletionPropagationPolicy Background
|
||||
- Parallel 2
|
||||
- Template true
|
||||
- NoCluster false
|
||||
- PauseOnFailure false
|
||||
Loading tests...
|
||||
- vpn-killswitch (/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch)
|
||||
Loading values...
|
||||
Running tests...
|
||||
=== RUN chainsaw
|
||||
=== PAUSE chainsaw
|
||||
=== CONT chainsaw
|
||||
=== RUN chainsaw/vpn-killswitch
|
||||
=== PAUSE chainsaw/vpn-killswitch
|
||||
=== CONT chainsaw/vpn-killswitch
|
||||
l.go:52: | 01:44:38 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | v1/Namespace @ chainsaw-credible-bird
|
||||
l.go:52: | 01:44:38 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mTRY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:44:38 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
kubectl label namespace "$NAMESPACE" \
|
||||
pod-security.kubernetes.io/enforce=privileged \
|
||||
pod-security.kubernetes.io/warn=privileged \
|
||||
pod-security.kubernetes.io/audit=privileged \
|
||||
--overwrite
|
||||
echo "NS_PRIVILEGED_OK"
|
||||
l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
namespace/chainsaw-credible-bird labeled
|
||||
NS_PRIVILEGED_OK
|
||||
l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mAPPLY[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-credible-bird/egress-internet
|
||||
l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-credible-bird/egress-internet
|
||||
l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mAPPLY[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-credible-bird/egress-internet
|
||||
l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mAPPLY[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-credible-bird/egress-vpn
|
||||
l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-credible-bird/egress-vpn
|
||||
l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mAPPLY[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-credible-bird/egress-vpn
|
||||
l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mASSERT[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-credible-bird/egress-internet
|
||||
l.go:52: | 01:44:41 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mASSERT[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-credible-bird/egress-internet
|
||||
l.go:52: | 01:44:41 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mASSERT[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-credible-bird/egress-vpn
|
||||
l.go:52: | 01:44:41 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mASSERT[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-credible-bird/egress-vpn
|
||||
l.go:52: | 01:44:41 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
get_ip() {
|
||||
kubectl -n "$ns" exec "$1" -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
|
||||
}
|
||||
wan="" vpn=""
|
||||
for i in $(seq 1 12); do
|
||||
[ -n "$wan" ] || wan="$(get_ip egress-internet)"
|
||||
[ -n "$vpn" ] || vpn="$(get_ip egress-vpn)"
|
||||
if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi
|
||||
[ "$wan" = "$vpn" ] && vpn="" # not steered yet; re-poll
|
||||
sleep 5
|
||||
done
|
||||
echo "wan=$wan vpn=$vpn"
|
||||
[ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; }
|
||||
[ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; }
|
||||
[ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; }
|
||||
echo "TUNNEL_HEALTHY_OK"
|
||||
l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
wan=136.51.115.252 vpn=158.173.241.11
|
||||
TUNNEL_HEALTHY_OK
|
||||
l.go:52: | 01:44:43 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mTRY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mTRY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mAPPLY[0;22m | [36;1mRUN[0;22m | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
|
||||
l.go:52: | 01:44:43 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
|
||||
l.go:52: | 01:44:43 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mAPPLY[0;22m | [32;1mDONE[0;22m | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
|
||||
l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mASSERT[0;22m | [36;1mRUN[0;22m | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
|
||||
l.go:52: | 01:44:48 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mASSERT[0;22m | [32;1mDONE[0;22m | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
|
||||
l.go:52: | 01:44:48 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -u
|
||||
ns="$NAMESPACE"
|
||||
wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
|
||||
for i in $(seq 1 8); do
|
||||
out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
|
||||
curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
|
||||
if [ -n "$out" ]; then
|
||||
if [ -n "$wan" ] && [ "$out" = "$wan" ]; then
|
||||
echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1
|
||||
fi
|
||||
echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1
|
||||
fi
|
||||
echo "attempt $i: vpn-class blackholed (no response), good"
|
||||
sleep 3
|
||||
done
|
||||
echo "NO_LEAK_OK"
|
||||
l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
attempt 1: vpn-class blackholed (no response), good
|
||||
attempt 2: vpn-class blackholed (no response), good
|
||||
attempt 3: vpn-class blackholed (no response), good
|
||||
attempt 4: vpn-class blackholed (no response), good
|
||||
attempt 5: vpn-class blackholed (no response), good
|
||||
attempt 6: vpn-class blackholed (no response), good
|
||||
attempt 7: vpn-class blackholed (no response), good
|
||||
attempt 8: vpn-class blackholed (no response), good
|
||||
NO_LEAK_OK
|
||||
l.go:52: | 01:46:18 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
out="$(kubectl -n "$ns" exec egress-internet -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
|
||||
echo "internet_ip=$out"
|
||||
[ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; }
|
||||
echo "INTERNET_UNAFFECTED_OK"
|
||||
l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
internet_ip=136.51.115.252
|
||||
INTERNET_UNAFFECTED_OK
|
||||
l.go:52: | 01:46:18 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mTRY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mFINALLY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
# Distinct Job name (tunnel-restart) so it never collides with the
|
||||
# tunnel-stop Job. Left-aligned manifest so the block scalar yields
|
||||
# valid YAML at column 0 at runtime.
|
||||
cat <<'JOB' | kubectl -n "$ns" apply -f -
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: tunnel-restart
|
||||
labels: { app: vpn-killswitch-fault }
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
activeDeadlineSeconds: 90
|
||||
ttlSecondsAfterFinished: 600
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: vpn-killswitch-fault }
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
homelab.keithjackson.dev/egress-gateway: "true"
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: nsenter
|
||||
image: busybox:1.37.0
|
||||
securityContext:
|
||||
privileged: true
|
||||
command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"]
|
||||
JOB
|
||||
# Block teardown until the restart actually completed on the host — a
|
||||
# leaked-down tunnel is unacceptable. `wait` exits non-zero on failure
|
||||
# or timeout, which (set -e) fails the step and surfaces the problem.
|
||||
kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart
|
||||
echo "TUNNEL_SELF_HEAL_OK"
|
||||
l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
job.batch/tunnel-restart created
|
||||
job.batch/tunnel-restart condition met
|
||||
TUNNEL_SELF_HEAL_OK
|
||||
l.go:52: | 01:46:21 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mFINALLY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
|
||||
vpn=""
|
||||
for i in $(seq 1 24); do
|
||||
vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
|
||||
curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
|
||||
if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi
|
||||
vpn=""
|
||||
sleep 5
|
||||
done
|
||||
echo "recovered_vpn=$vpn wan=$wan"
|
||||
[ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; }
|
||||
[ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; }
|
||||
echo "TUNNEL_RECOVERED_OK"
|
||||
l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
recovered_vpn=158.173.241.11 wan=136.51.115.252
|
||||
TUNNEL_RECOVERED_OK
|
||||
l.go:52: | 01:46:25 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCLEANUP[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:46:25 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
|
||||
l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCLEANUP[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCLEANUP[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:46:25 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-credible-bird/egress-vpn
|
||||
l.go:52: | 01:46:56 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-credible-bird/egress-internet
|
||||
l.go:52: | 01:47:26 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCLEANUP[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:47:26 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw [0;22m | [36;1mCLEANUP[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:47:26 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | v1/Namespace @ chainsaw-credible-bird
|
||||
l.go:52: | 01:47:33 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw [0;22m | [36;1mCLEANUP[0;22m | [36;1mEND[0;22m |
|
||||
--- PASS: chainsaw (0.00s)
|
||||
--- PASS: chainsaw/vpn-killswitch (174.63s)
|
||||
PASS
|
||||
Tests Summary...
|
||||
- Passed tests 1
|
||||
- Failed tests 0
|
||||
- Skipped tests 0
|
||||
Done.
|
||||
[1m[run.sh][0m HEAVY pass: no heavy tests selected.
|
||||
merge_junit: merged 1 report(s) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml (tests=1 failures=0 errors=0 skipped=0)
|
||||
[1m[run.sh][0m Done. JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md
|
||||
@@ -1,4 +0,0 @@
|
||||
<testsuites name="chainsaw-report" tests="1" failures="0" errors="0" time="0.000000"><testsuite name="/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch" tests="1" failures="0" errors="0" id="0" time="">
|
||||
<testcase name="vpn-killswitch" classname="" time="173.473422" />
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -1,237 +0,0 @@
|
||||
[1m[run.sh][0m Coverage report first (always)
|
||||
[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s).
|
||||
[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json
|
||||
[1m[run.sh][0m No L0/L1 offline checks match the selection (layer=l3 reqs='R-NET-06').
|
||||
[1m[run.sh][0m LIGHT pass: 1 test(s), parallel=2 (pinned v0.2.12)
|
||||
Version: 0.2.12
|
||||
Loading config (/work/.claude/conf-clone/conformance/config/chainsaw-config.yaml)...
|
||||
- Using test file: chainsaw-test
|
||||
- TestDirs [/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch]
|
||||
- SkipDelete false
|
||||
- FailFast false
|
||||
- ReportFormat 'JUNIT-TEST'
|
||||
- ReportName 'chainsaw-report-light'
|
||||
- ReportPath '/work/.claude/conf-clone/conformance/_artifacts'
|
||||
- Namespace ''
|
||||
- FullName false
|
||||
- IncludeTestRegex ''
|
||||
- ExcludeTestRegex ''
|
||||
- ApplyTimeout 1m30s
|
||||
- AssertTimeout 3m0s
|
||||
- CleanupTimeout 2m0s
|
||||
- DeleteTimeout 2m0s
|
||||
- ErrorTimeout 1m0s
|
||||
- ExecTimeout 3m0s
|
||||
- DeletionPropagationPolicy Background
|
||||
- Parallel 2
|
||||
- Template true
|
||||
- NoCluster false
|
||||
- PauseOnFailure false
|
||||
Loading tests...
|
||||
- vpn-killswitch (/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch)
|
||||
Loading values...
|
||||
Running tests...
|
||||
=== RUN chainsaw
|
||||
=== PAUSE chainsaw
|
||||
=== CONT chainsaw
|
||||
=== RUN chainsaw/vpn-killswitch
|
||||
=== PAUSE chainsaw/vpn-killswitch
|
||||
=== CONT chainsaw/vpn-killswitch
|
||||
l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | v1/Namespace @ chainsaw-beloved-toad
|
||||
l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mTRY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
kubectl label namespace "$NAMESPACE" \
|
||||
pod-security.kubernetes.io/enforce=privileged \
|
||||
pod-security.kubernetes.io/warn=privileged \
|
||||
pod-security.kubernetes.io/audit=privileged \
|
||||
--overwrite
|
||||
echo "NS_PRIVILEGED_OK"
|
||||
l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
namespace/chainsaw-beloved-toad labeled
|
||||
NS_PRIVILEGED_OK
|
||||
l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mAPPLY[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-internet
|
||||
l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-internet
|
||||
l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mAPPLY[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-internet
|
||||
l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mAPPLY[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-vpn
|
||||
l.go:52: | 01:49:25 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-vpn
|
||||
l.go:52: | 01:49:25 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mAPPLY[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-vpn
|
||||
l.go:52: | 01:49:25 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mASSERT[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-internet
|
||||
l.go:52: | 01:49:26 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mASSERT[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-internet
|
||||
l.go:52: | 01:49:26 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mASSERT[0;22m | [36;1mRUN[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-vpn
|
||||
l.go:52: | 01:49:26 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mASSERT[0;22m | [32;1mDONE[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-vpn
|
||||
l.go:52: | 01:49:26 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
get_ip() {
|
||||
kubectl -n "$ns" exec "$1" -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
|
||||
}
|
||||
wan="" vpn=""
|
||||
for i in $(seq 1 12); do
|
||||
[ -n "$wan" ] || wan="$(get_ip egress-internet)"
|
||||
[ -n "$vpn" ] || vpn="$(get_ip egress-vpn)"
|
||||
if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi
|
||||
[ "$wan" = "$vpn" ] && vpn="" # not steered yet; re-poll
|
||||
sleep 5
|
||||
done
|
||||
echo "wan=$wan vpn=$vpn"
|
||||
[ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; }
|
||||
[ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; }
|
||||
[ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; }
|
||||
echo "TUNNEL_HEALTHY_OK"
|
||||
l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
wan=136.51.115.252 vpn=158.173.241.11
|
||||
TUNNEL_HEALTHY_OK
|
||||
l.go:52: | 01:49:27 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mTRY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mTRY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mAPPLY[0;22m | [36;1mRUN[0;22m | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
|
||||
l.go:52: | 01:49:27 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mCREATE[0;22m | [32;1mOK[0;22m | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
|
||||
l.go:52: | 01:49:27 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mAPPLY[0;22m | [32;1mDONE[0;22m | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
|
||||
l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mASSERT[0;22m | [36;1mRUN[0;22m | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
|
||||
l.go:52: | 01:49:31 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mASSERT[0;22m | [32;1mDONE[0;22m | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
|
||||
l.go:52: | 01:49:31 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -u
|
||||
ns="$NAMESPACE"
|
||||
wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
|
||||
for i in $(seq 1 8); do
|
||||
out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
|
||||
curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
|
||||
if [ -n "$out" ]; then
|
||||
if [ -n "$wan" ] && [ "$out" = "$wan" ]; then
|
||||
echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1
|
||||
fi
|
||||
echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1
|
||||
fi
|
||||
echo "attempt $i: vpn-class blackholed (no response), good"
|
||||
sleep 3
|
||||
done
|
||||
echo "NO_LEAK_OK"
|
||||
l.go:52: | 01:51:01 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
attempt 1: vpn-class blackholed (no response), good
|
||||
attempt 2: vpn-class blackholed (no response), good
|
||||
attempt 3: vpn-class blackholed (no response), good
|
||||
attempt 4: vpn-class blackholed (no response), good
|
||||
attempt 5: vpn-class blackholed (no response), good
|
||||
attempt 6: vpn-class blackholed (no response), good
|
||||
attempt 7: vpn-class blackholed (no response), good
|
||||
attempt 8: vpn-class blackholed (no response), good
|
||||
NO_LEAK_OK
|
||||
l.go:52: | 01:51:01 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:51:01 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
out="$(kubectl -n "$ns" exec egress-internet -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
|
||||
echo "internet_ip=$out"
|
||||
[ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; }
|
||||
echo "INTERNET_UNAFFECTED_OK"
|
||||
l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
internet_ip=136.51.115.252
|
||||
INTERNET_UNAFFECTED_OK
|
||||
l.go:52: | 01:51:02 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mTRY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mFINALLY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
# Distinct Job name (tunnel-restart) so it never collides with the
|
||||
# tunnel-stop Job. Left-aligned manifest so the block scalar yields
|
||||
# valid YAML at column 0 at runtime.
|
||||
cat <<'JOB' | kubectl -n "$ns" apply -f -
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: tunnel-restart
|
||||
labels: { app: vpn-killswitch-fault }
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
activeDeadlineSeconds: 90
|
||||
ttlSecondsAfterFinished: 600
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: vpn-killswitch-fault }
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
homelab.keithjackson.dev/egress-gateway: "true"
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: nsenter
|
||||
image: busybox:1.37.0
|
||||
securityContext:
|
||||
privileged: true
|
||||
command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"]
|
||||
JOB
|
||||
# Block teardown until the restart actually completed on the host — a
|
||||
# leaked-down tunnel is unacceptable. `wait` exits non-zero on failure
|
||||
# or timeout, which (set -e) fails the step and surfaces the problem.
|
||||
kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart
|
||||
echo "TUNNEL_SELF_HEAL_OK"
|
||||
l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
job.batch/tunnel-restart created
|
||||
job.batch/tunnel-restart condition met
|
||||
TUNNEL_SELF_HEAL_OK
|
||||
l.go:52: | 01:51:05 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mFINALLY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mCMD[0;22m | [36;1mRUN[0;22m |
|
||||
=== COMMAND
|
||||
/usr/bin/sh -c set -eu
|
||||
ns="$NAMESPACE"
|
||||
wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
|
||||
vpn=""
|
||||
for i in $(seq 1 24); do
|
||||
vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
|
||||
curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
|
||||
if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi
|
||||
vpn=""
|
||||
sleep 5
|
||||
done
|
||||
echo "recovered_vpn=$vpn wan=$wan"
|
||||
[ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; }
|
||||
[ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; }
|
||||
echo "TUNNEL_RECOVERED_OK"
|
||||
l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mSCRIPT[0;22m | [36;1mLOG[0;22m |
|
||||
=== STDOUT
|
||||
recovered_vpn=158.173.241.11 wan=136.51.115.252
|
||||
TUNNEL_RECOVERED_OK
|
||||
l.go:52: | 01:51:08 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [32;1mSCRIPT[0;22m | [32;1mDONE[0;22m |
|
||||
l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCLEANUP[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:51:08 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
|
||||
l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak [0;22m | [36;1mCLEANUP[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCLEANUP[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:51:08 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-vpn
|
||||
l.go:52: | 01:51:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | v1/Pod @ chainsaw-beloved-toad/egress-internet
|
||||
l.go:52: | 01:52:10 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads [0;22m | [36;1mCLEANUP[0;22m | [36;1mEND[0;22m |
|
||||
l.go:52: | 01:52:10 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw [0;22m | [36;1mCLEANUP[0;22m | [36;1mBEGIN[0;22m |
|
||||
l.go:52: | 01:52:10 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw [0;22m | [32;1mDELETE[0;22m | [32;1mOK[0;22m | v1/Namespace @ chainsaw-beloved-toad
|
||||
l.go:52: | 01:52:18 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw [0;22m | [36;1mCLEANUP[0;22m | [36;1mEND[0;22m |
|
||||
--- PASS: chainsaw (0.00s)
|
||||
--- PASS: chainsaw/vpn-killswitch (173.47s)
|
||||
PASS
|
||||
Tests Summary...
|
||||
- Passed tests 1
|
||||
- Failed tests 0
|
||||
- Skipped tests 0
|
||||
Done.
|
||||
[1m[run.sh][0m HEAVY pass: no heavy tests selected.
|
||||
merge_junit: merged 1 report(s) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml (tests=1 failures=0 errors=0 skipped=0)
|
||||
[1m[run.sh][0m Done. JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md
|
||||
@@ -1,5 +1,2 @@
|
||||
{"chainsaw_version": "v0.2.12", "env": "dev", "git_sha": "57d4df8c4f992732a49f3e2daddcace866778ab0", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NODE_LOSS_VOLUME_AVAILABLE_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-STATE-02"], "run_id": "20260901T235117Z-e9a4b66", "sentinel": "NODE_LOSS_VOLUME_AVAILABLE_OK", "test": "node-loss", "ts_utc": "2026-09-01T23:51:17Z"}
|
||||
{"chainsaw_version": "v0.2.12", "env": "dev", "git_sha": "fdc3265585f4b775c0a6ae3fb33c837b0c643655", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (OFF_CLUSTER_TARGET_OK, BACKUPS_ARE_SCHEDULED_OK, DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-STATE-03"], "run_id": "20260902T001907Z-fdc3265", "sentinel": "DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK", "test": "dr-restore", "ts_utc": "2026-09-02T00:19:07Z"}
|
||||
{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L1", "note": "L1 offline check exited 0; run_l1_offline.py renders its skips as JUnit <skipped/>, so a green JUnit is truthful without a sentinel", "proof": "proven", "requirements": ["R-NET-05", "R-NET-06", "R-NET-07"], "run_id": "20260913T014321Z-19fc8bc", "sentinel": null, "test": "egress-class-render", "ts_utc": "2026-09-13T01:43:21Z"}
|
||||
{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6-dirty", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014437Z-19fc8bc", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:44:37Z"}
|
||||
{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "9d0814ae61e6a0ba64d3789a71f1cd9c985efbf0", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014923Z-9d0814a", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:49:23Z"}
|
||||
@@ -1,95 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the alert rules are UNIT-TESTED, by a pinned engine, and the gaps are declared
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# R-OBS-04 ("the platform alerts on its own health") is implemented by PrometheusRule CRs, and
|
||||
# this repo has shipped both ways they fail SILENTLY:
|
||||
# * PERMANENTLY FIRING — the CNPG primaries alert (fixed 2026-09-12), VPNTunnelDown before it;
|
||||
# * STRUCTURALLY INERT — VeleroNoRecentBackup, whose `time() - <absent series>` yields no
|
||||
# series, so a schedule that had NEVER backed up could not raise it. Prod ran with zero
|
||||
# volume backups and nothing alerted, masked by a resources-only schedule completing
|
||||
# cleanly every night (2026-09-12).
|
||||
# An inert rule and a correct rule are indistinguishable while the platform is healthy. Only
|
||||
# `promtool test rules` separates them, by constructing the bad state on purpose, offline.
|
||||
#
|
||||
# The unit tests were then written — and RUN BY NOTHING. promtool was not in the pinned
|
||||
# toolchain, no gate stage evaluated the files, and the two harnesses that grew up independently
|
||||
# each fetched an engine off the network mid-run if promtool was missing (a GitHub release
|
||||
# download in one, a prometheus container image in the other). Coverage that looks present and
|
||||
# is enforced nowhere is the failure mode this test ends.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN the ONE rule-unit harness (platform/monitoring/tests/run.sh, which flattens the
|
||||
# COMMITTED PrometheusRule CRs into the rules file promtool wants — never a copy) is run
|
||||
# over every test file in the repo with the PINNED promtool,
|
||||
# THEN promtool is pinned in BOTH versions.env files in lockstep at the Prometheus version prod
|
||||
# actually runs, baked into the image and installed on the CI runner; the promtool in use
|
||||
# reports exactly that version; no harness acquires an engine at run time; there is
|
||||
# exactly ONE harness; every rule-unit test file PASSES; the harness is proven able to
|
||||
# FAIL by a mutated synthetic test on every run; and every alert rule with NO unit test is
|
||||
# ENUMERATED in a shrink-only register, so the gap is reported rather than hidden.
|
||||
#
|
||||
# MISSING TESTS: RATCHET, NOT SKIP, NOT BLANKET-FAIL. 48 of 55 alert rules have no unit test
|
||||
# today. Failing on all of them would make the gate red until ~48 tests exist (so it would be
|
||||
# disabled, and a disabled gate protects nothing); merely reporting is precisely how the CNPG
|
||||
# tests came to be enforced by nothing. So the gap is DECLARED in check.sh, printed with counts
|
||||
# every run, and CANNOT GROW: a new PrometheusRule, or a new alert in an already-tested file,
|
||||
# FAILS without a test, and a stale register entry FAILS too. A missing test is never an exit-3
|
||||
# skip — "SKIPPED is not green" is preserved for the one thing that is genuinely an environment
|
||||
# gap: promtool being absent.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is an offline assertion script (standard library + the harness
|
||||
# it delegates to); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this
|
||||
# Chainsaw Test wraps the SAME script.
|
||||
#
|
||||
# NOT proven (named in check.sh's header): that an alert reaches a human (Alertmanager routing
|
||||
# and SMTP are L3 / R-OBS-02), that a threshold is the right one, that a rule file is actually
|
||||
# reconciled into a cluster, or that an author's assertions are meaningful — coverage of one
|
||||
# alert is detected by its name appearing in a covering test file, and only the mutation
|
||||
# self-test proves the harness CAN fail. R-OBS-04 is tagged for the static slice only: the rules
|
||||
# the repo ships behave, under the engine prod runs, as their own unit tests say.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: alert-rules-unit-tested
|
||||
labels:
|
||||
conformance.home.k3s/R-OBS-04: "true"
|
||||
conformance.home.k3s/R-TEST-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-OBS-04, R-TEST-02"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE — this invariant quantifies over every PrometheusRule in the tree
|
||||
# (apps/wave-0..2, platform/cnpg, platform/kubevirt, platform/monitoring), and the register
|
||||
# it maintains is a property of the WHOLE repo, not of any one path.
|
||||
#
|
||||
# WHY IT MUST NOT BE REQUIREMENT-SCOPED: ci/touched-requirements.sh's coarse path map keys
|
||||
# apps/* to NOTHING and platform/monitoring/* to R-OBS-01/02 — not R-OBS-04. So adding a new
|
||||
# untested PrometheusRule under apps/, or editing one under platform/kubevirt/, attributes
|
||||
# no requirement this test carries, and a requirement-scoped selection would filter the
|
||||
# check out of exactly the change that adds the untested alert. The same blind spot that let
|
||||
# `prod-baremetal-not-dev-surface` be filtered out of the run that gated its own breakage
|
||||
# (DECISIONS-LOG 2026-09-12).
|
||||
#
|
||||
# Declaring the scope also makes an honest SKIP loud: lib/run_l1_offline.py turns a rc-3
|
||||
# skip of a scope-declaring check into the runner's rc 3, and ci/l0.sh into the gate's rc 3.
|
||||
# So a gate run on a toolchain WITHOUT promtool reports UNWIRED instead of green — which is
|
||||
# the whole reason this check exists rather than another hand-written ci/l0.sh stage.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every committed alert rule that has unit tests passes them under the pinned promtool that
|
||||
matches prod's Prometheus, exactly one harness runs them, nothing fetches a rule engine at
|
||||
run time, the harness is proven able to fail, and every alert rule without a unit test is
|
||||
enumerated in a register that can only shrink.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-every-alert-rule-unit-test-and-report-the-gaps
|
||||
description: >-
|
||||
Assert the promtool pin is in lockstep and baked, run the single rule-unit harness over
|
||||
every test file in the repo, prove it can fail via a mutated synthetic test, and
|
||||
enumerate every PrometheusRule with no unit tests against the shrink-only register.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,550 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — alert-rules-unit-tested
|
||||
# Serves: R-OBS-04 ("Platform alerts on its own health (nodes, storage, GitOps, certs,
|
||||
# VPN)." — spec docs/10-observability-and-alerting.md)
|
||||
# R-TEST-02 ("Tests are reproducible: deterministic, hermetic, single-command."
|
||||
# — spec docs/14-testing-and-conformance.md)
|
||||
#
|
||||
# WHY THIS EXISTS — AN UNTESTED ALERT IS NOT A MONITORING CONTROL
|
||||
# --------------------------------------------------------------
|
||||
# This repo has shipped BOTH halves of the alerting failure space to prod:
|
||||
# * a PERMANENTLY-FIRING `critical` (the CNPG alert fixed 2026-09-12; VPNTunnelDown before
|
||||
# it), and
|
||||
# * a STRUCTURALLY INERT rule — VeleroNoRecentBackup, whose `time() - <absent series>`
|
||||
# yields no series, so a schedule that had NEVER backed up could not raise it. Prod ran
|
||||
# with no volume backups at all and nothing alerted (2026-09-12).
|
||||
# Neither is visible from the expression, from `kubectl apply`, or from a healthy cluster: an
|
||||
# inert rule and a correct rule are indistinguishable while everything is fine. `promtool test
|
||||
# rules` is the only tool that separates them, because the BAD state can be constructed on
|
||||
# purpose. Unit tests for those rules were then WRITTEN — and evaluated by nothing: promtool
|
||||
# was not in the pinned toolchain, no gate stage ran the files, and each of the two harnesses
|
||||
# that existed acquired an engine off the network mid-run if one was missing. Coverage that
|
||||
# looks present and is never enforced is the failure mode this check ends.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN the ONE rule-unit harness (platform/monitoring/tests/run.sh, which flattens the
|
||||
# COMMITTED PrometheusRule CRs — never a copy) is asked which manifests are under test
|
||||
# and then run over every test file in the repo,
|
||||
# THEN (1) promtool is a PINNED member of the toolchain: PROMTOOL_VERSION is declared in BOTH
|
||||
# infra/devcontainer/versions.env and ci/versions.env and the two AGREE (the repo's
|
||||
# lockstep rule), the Containerfile takes it as a build ARG, installs it from the
|
||||
# pinned prometheus release and records it for provenance, dev.sh passes it, and the
|
||||
# CI runner installs it for the l0 stage;
|
||||
# (2) the promtool ACTUALLY IN USE reports exactly that pinned version — a unit test is
|
||||
# evidence only if the engine matches the one prod evaluates rules with;
|
||||
# (3) NO harness acquires a rule engine at RUN TIME — neither `curl | tar -xz` of a
|
||||
# GitHub release nor a `podman/docker run` of a prometheus image. Both existed on
|
||||
# 2026-09-12 and both are removed; this is the regression guard;
|
||||
# (4) there is exactly ONE harness. Two were written independently within an hour of
|
||||
# each other; the one nothing runs is the one that rots;
|
||||
# (5) every rule-unit test file in the repo PASSES, run through that harness (which
|
||||
# `promtool check rules` first, so a non-parsing rules file cannot make unit tests
|
||||
# pass vacuously by finding no rules at all);
|
||||
# (6) the harness is NOT VACUOUS: a synthetic PrometheusRule + companion test is run
|
||||
# through the SAME harness twice — intact (must PASS) and with one expectation
|
||||
# mutated (must FAIL) — so a harness that has silently stopped evaluating anything
|
||||
# is caught on every run, not only when someone thinks to try it;
|
||||
# AND (7) the alert rules with NO unit tests are ENUMERATED and RATCHETED: each is declared
|
||||
# below, printed with counts on every run, and the register can only SHRINK.
|
||||
#
|
||||
# WHY A MISSING TEST IS A RATCHET AND NOT AN OUTRIGHT FAIL (the decision, and its reasons)
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# 48 of the repo's 55 alert rules have no unit test today. Three options were on the table:
|
||||
# * FAIL on any missing test — correct in principle, but it makes the merge gate red for
|
||||
# every change until ~48 unit tests exist, so in practice the gate gets disabled, and a
|
||||
# disabled gate protects nothing.
|
||||
# * merely REPORT — the gap prints and nothing holds it. That is exactly how the CNPG unit
|
||||
# tests came to exist and be enforced by nothing; "it is in the output somewhere" is not
|
||||
# enforcement.
|
||||
# * RATCHET (chosen) — the gap is DECLARED IN THE REPO as a reviewable register, printed with
|
||||
# counts on every gate run, and it CANNOT GROW: a new PrometheusRule, or a new alert in a
|
||||
# file that already has tests, FAILS unless it ships a unit test. Removing an entry requires
|
||||
# writing the test, and a STALE entry (file gone, or now covered) also FAILS, so the
|
||||
# register cannot rot into fiction.
|
||||
# "SKIPPED is not green" is honoured: a missing test is NEVER an exit-3 skip here. This check
|
||||
# exits 0 only with the gap fully enumerated, and exits 1 the moment the debt grows, the
|
||||
# register drifts, or a test that exists stops passing.
|
||||
# The one exit-3 path is promtool itself being ABSENT — an environment gap, not a code property
|
||||
# (the same convention as ci/l0.sh's kyverno handling), and this test DECLARES
|
||||
# `conformance.home.k3s/scope: whole-repo`, which makes lib/run_l1_offline.py turn that skip
|
||||
# into the runner's rc 3 and ci/l0.sh into the gate's rc 3: UNWIRED, never green. promtool
|
||||
# PRESENT AT THE WRONG VERSION is a FAILURE, not a skip: that case still prints SUCCESS while
|
||||
# evaluating the rules with an engine prod does not run.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps):
|
||||
# - It does not prove an alert reaches a human. promtool evaluates rules against synthetic
|
||||
# series in-process; Alertmanager routing, SMTP delivery and inhibition are L3
|
||||
# (tests/l3-fault-injection/alerting) and R-OBS-02, and are not claimed here.
|
||||
# - It does not prove the THRESHOLDS are right, only that a rule behaves as its own unit
|
||||
# test says.
|
||||
# - It does not prove a rule file is RECONCILED into a cluster. An alert file no
|
||||
# kustomization includes is a separate "looks applied and isn't" failure, not audited here.
|
||||
# - Coverage of an individual alert is detected by its NAME appearing in a test file that
|
||||
# covers its manifest. A test file that merely mentions a name without asserting anything
|
||||
# counts as covered; only the mutation self-test (6) proves the harness CAN fail, not that
|
||||
# every author wrote a meaningful assertion. Review still owns test quality.
|
||||
# - It does not read the live cluster, so "the pin matches prod's Prometheus" is asserted
|
||||
# against the repo's pin, which was established by a live read recorded in
|
||||
# infra/devcontainer/versions.env (2026-09-12: v3.14.0-distroless).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
HARNESS="${ALERT_RULE_HARNESS:-${ROOT}/platform/monitoring/tests/run.sh}"
|
||||
|
||||
# Honest SKIP (never green) if the thing under test is not present at all. Because this test
|
||||
# declares scope: whole-repo, the runner turns this into rc 3 for the whole L0 stage — a
|
||||
# PrometheusRule-less repo is a vanished tree, not a pass.
|
||||
if ! grep -rqls --include='*.yaml' -e '^[[:space:]]*kind:[[:space:]]*PrometheusRule[[:space:]]*$' \
|
||||
"${ROOT}/platform" "${ROOT}/apps" 2>/dev/null; then
|
||||
echo "check SKIPPED: no PrometheusRule under platform/ or apps/ — nothing to test" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# promtool resolution. PROMTOOL= overrides (the same override the harness honours); otherwise
|
||||
# PATH. Absent -> every OFFLINE assertion still runs and the check exits 3, so a toolchain gap
|
||||
# is reported as a gap and never as a pass.
|
||||
PROMTOOL_BIN="${PROMTOOL:-$(command -v promtool || true)}"
|
||||
if [ -n "${PROMTOOL_BIN}" ] && [ -x "${PROMTOOL_BIN}" ]; then
|
||||
PROMTOOL_BIN="$(cd "$(dirname "${PROMTOOL_BIN}")" && pwd)/$(basename "${PROMTOOL_BIN}")"
|
||||
else
|
||||
PROMTOOL_BIN=""
|
||||
fi
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" "${HARNESS}" "${PROMTOOL_BIN}" <<'PY'
|
||||
"""Assert the alert-rule unit tests are pinned, enforced, non-vacuous, and that the alert
|
||||
rules with no tests are enumerated and ratcheted. Standard library + the harness itself."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
HARNESS = Path(sys.argv[2])
|
||||
PROMTOOL = sys.argv[3]
|
||||
sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree
|
||||
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# THE DECLARED REGISTER (assertion 7). Shrink-only; see the header for why this is a ratchet.
|
||||
#
|
||||
# UNTESTED_FILES — PrometheusRule files no test file covers AT ALL. The twelve apps/* entries
|
||||
# are single `<app>-down` liveness rules generated from one ServiceDefinition template;
|
||||
# platform/kubevirt/alerts.yaml (4 rules) is the substantial one and is where the next tests
|
||||
# belong. A file listed here that later gains coverage, or stops carrying a PrometheusRule,
|
||||
# FAILS as stale.
|
||||
# =============================================================================
|
||||
UNTESTED_FILES = {
|
||||
"apps/wave-0/gitea.yaml",
|
||||
"apps/wave-0/jackett.yaml",
|
||||
"apps/wave-0/pihole.yaml",
|
||||
"apps/wave-0/plex.yaml",
|
||||
"apps/wave-0/qbittorrent.yaml",
|
||||
"apps/wave-0/vm-demo.yaml",
|
||||
"apps/wave-1/keycloak.yaml",
|
||||
"apps/wave-1/nextcloud.yaml",
|
||||
"apps/wave-1/radarr.yaml",
|
||||
"apps/wave-1/semaphore.yaml",
|
||||
"apps/wave-1/sonarr.yaml",
|
||||
"apps/wave-2/seerr.yaml",
|
||||
"platform/kubevirt/alerts.yaml",
|
||||
}
|
||||
|
||||
# UNTESTED_ALERTS — individual alerts inside a file that IS covered by a test file, but whose
|
||||
# own rule nothing asserts on. This is the half that stops a new alert being slipped into an
|
||||
# already-"tested" file untested: anything not listed here must appear in a covering test.
|
||||
# The monitoring block is large because backups.test.yaml (the repo's first suite) deliberately
|
||||
# covers the Velero/backup rules only; pvc-capacity-alerts.yaml is flattened into the same
|
||||
# rules file by the suite convention but has no assertions of its own yet.
|
||||
UNTESTED_ALERTS = {
|
||||
("platform/cnpg/alerts.yaml", "CNPGWALArchiveBacklog"),
|
||||
("platform/cnpg/alerts.yaml", "CNPGNoRecoverabilityPoint"),
|
||||
("platform/cnpg/alerts.yaml", "CNPGNoRecentBaseBackup"),
|
||||
("platform/cnpg/alerts.yaml", "CNPGWALVolumeNotRecycling"),
|
||||
("platform/cnpg/alerts.yaml", "CNPGMetricsCollectorErroring"),
|
||||
("platform/monitoring/alerts.yaml", "NodeExporterDown"),
|
||||
("platform/monitoring/alerts.yaml", "NodeMemoryPressureHigh"),
|
||||
("platform/monitoring/alerts.yaml", "NodeDiskFillingUp"),
|
||||
("platform/monitoring/alerts.yaml", "NodeCPUPressureHigh"),
|
||||
("platform/monitoring/alerts.yaml", "NodeCPULoadHigh"),
|
||||
("platform/monitoring/alerts.yaml", "NodeIOPressureHigh"),
|
||||
("platform/monitoring/alerts.yaml", "NodeIOPressureCritical"),
|
||||
("platform/monitoring/alerts.yaml", "LonghornVolumeDegraded"),
|
||||
("platform/monitoring/alerts.yaml", "LonghornVolumeFaulted"),
|
||||
("platform/monitoring/alerts.yaml", "LonghornVolumeCapacityHigh"),
|
||||
("platform/monitoring/alerts.yaml", "LonghornNodeStorageHigh"),
|
||||
("platform/monitoring/alerts.yaml", "FluxReconciliationFailure"),
|
||||
("platform/monitoring/alerts.yaml", "FluxResourceSuspended"),
|
||||
("platform/monitoring/alerts.yaml", "CertificateExpiringSoon"),
|
||||
("platform/monitoring/alerts.yaml", "CertificateExpiryImminent"),
|
||||
("platform/monitoring/alerts.yaml", "CertificateNotReady"),
|
||||
("platform/monitoring/alerts.yaml", "VPNTunnelDown"),
|
||||
("platform/monitoring/alerts.yaml", "HubbleDeniedFlowSpike"),
|
||||
("platform/monitoring/alerts.yaml", "KeycloakDown"),
|
||||
("platform/monitoring/alerts.yaml", "KeycloakDatabaseDown"),
|
||||
("platform/monitoring/alerts.yaml", "KeycloakClientSecretReconcilerFailing"),
|
||||
("platform/monitoring/alerts.yaml", "KeycloakClientSecretReconcilerStale"),
|
||||
("platform/monitoring/alerts.yaml", "MonitoringVolumeFillingUp"),
|
||||
("platform/monitoring/alerts.yaml", "PrometheusTargetsMissing"),
|
||||
("platform/monitoring/alerts.yaml", "ImageUpdateAutomationFailing"),
|
||||
("platform/monitoring/alerts.yaml", "ImagePolicyStalled"),
|
||||
("platform/monitoring/alerts.yaml", "UpdateHealthGateFailed"),
|
||||
("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeFillingUp"),
|
||||
("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeAlmostFull"),
|
||||
("platform/monitoring/pvc-capacity-alerts.yaml", "PersistentVolumeFillingUpPredicted"),
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------- (1) the pins, in lockstep
|
||||
def env_pin(rel: str, key: str) -> str | None:
|
||||
f = ROOT / rel
|
||||
if not f.is_file():
|
||||
return None
|
||||
m = re.search(rf'^{re.escape(key)}=["\']?([^"\'\s#]+)', f.read_text(), re.M)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
DEV_PIN = env_pin("infra/devcontainer/versions.env", "PROMTOOL_VERSION")
|
||||
CI_PIN = env_pin("ci/versions.env", "PROMTOOL_VERSION")
|
||||
check(DEV_PIN is not None,
|
||||
"infra/devcontainer/versions.env declares no PROMTOOL_VERSION — promtool is not part of the "
|
||||
"baked toolchain, so the alert unit tests are evaluated by whatever the host happens to "
|
||||
"have, or by nothing at all (which is where this repo was on 2026-09-12)")
|
||||
check(CI_PIN is not None,
|
||||
"ci/versions.env declares no PROMTOOL_VERSION — the gate's own version file does not pin the "
|
||||
"rule engine, so CI and the devcontainer can silently diverge")
|
||||
check(DEV_PIN is None or DEV_PIN == CI_PIN,
|
||||
f"PROMTOOL_VERSION is NOT in lockstep: infra/devcontainer/versions.env={DEV_PIN!r} vs "
|
||||
f"ci/versions.env={CI_PIN!r}. The repo's rule is that the conformance-tool versions in the "
|
||||
f"two files stay identical, so a laptop and CI run the same engine")
|
||||
|
||||
CF = ROOT / "infra/devcontainer/Containerfile"
|
||||
cf_text = CF.read_text() if CF.is_file() else ""
|
||||
check(re.search(r"^ARG PROMTOOL_VERSION\s*$", cf_text, re.M),
|
||||
"infra/devcontainer/Containerfile has no `ARG PROMTOOL_VERSION` — the pin is declared but "
|
||||
"the image build does not consume it")
|
||||
check("prometheus/prometheus/releases/download" in cf_text and "promtool" in cf_text,
|
||||
"infra/devcontainer/Containerfile does not install promtool from the pinned prometheus "
|
||||
"release tarball (promtool ships inside it)")
|
||||
check('echo "promtool=${PROMTOOL_VERSION}"' in cf_text,
|
||||
"infra/devcontainer/Containerfile does not record promtool in /etc/dev-toolchain-versions — "
|
||||
"that provenance line is how an operator tells a stale image from a current one")
|
||||
|
||||
DEVSH = ROOT / "infra/devcontainer/dev.sh"
|
||||
check(DEVSH.is_file() and "--build-arg PROMTOOL_VERSION=" in DEVSH.read_text(),
|
||||
"infra/devcontainer/dev.sh does not pass --build-arg PROMTOOL_VERSION — the ARG would be "
|
||||
"empty and the build would fetch a bogus URL")
|
||||
|
||||
INSTALL = ROOT / ".gitea/workflows/scripts/install-tools.sh"
|
||||
if INSTALL.is_file():
|
||||
itext = INSTALL.read_text()
|
||||
check("PROMTOOL_VERSION" in itext
|
||||
and re.search(r"^\s*l0\)[^\n]*install_promtool", itext, re.M),
|
||||
".gitea/workflows/scripts/install-tools.sh does not install promtool for the l0 stage — "
|
||||
"on the CI runner this check would exit 3 (UNWIRED) forever and the alert unit tests "
|
||||
"would be enforced nowhere, which is the exact gap it exists to close")
|
||||
|
||||
# --------------------------------------------- (3)(4) one harness, and it fetches no engine
|
||||
check(HARNESS.is_file() and os.access(HARNESS, os.X_OK),
|
||||
f"the rule-unit harness is missing or not executable at {HARNESS.relative_to(ROOT)} — "
|
||||
f"nothing would run the alert unit tests")
|
||||
|
||||
# Any script that invokes `promtool test rules` is a rule-unit harness. There must be exactly
|
||||
# one: two harnesses were written independently on 2026-09-12 (a cnpg-only script and this one),
|
||||
# and the one nothing runs is the one that rots out of sync with the rules.
|
||||
harnesses: list[str] = []
|
||||
fetchers: list[str] = []
|
||||
FETCH_RE = re.compile(
|
||||
r"(curl|wget)[^\n]*prometheus/prometheus/releases" # GitHub release download
|
||||
r"|(podman|docker)\s+run[^\n]*prometheus", # container-image pull
|
||||
re.I)
|
||||
# Two exclusions, both the repo's existing "reference, not declaration" rule (see
|
||||
# ci/touched-requirements.sh changed_diff() rules 2-3):
|
||||
# * THIS CHECK'S OWN DIRECTORY. It names every forbidden pattern in order to forbid it, so
|
||||
# scanning itself would make both guards permanently red — the self-referential trap.
|
||||
# * THE PINNED TOOLCHAIN INSTALLERS. Downloading the pinned promtool is their JOB, at a
|
||||
# reviewed seam (an image build, or the runner's tool-install step), from a version in
|
||||
# versions.env. What is forbidden is a RULE-UNIT RUN acquiring its own engine: then the
|
||||
# version, the network and the machine are decided silently at test time, and a missing
|
||||
# toolchain becomes a green run instead of an exit 3.
|
||||
SELF_DIR = "conformance/tests/l0-static/alert-rules-unit-tested/"
|
||||
INSTALLERS = (".gitea/workflows/scripts/install-tools.sh", "infra/devcontainer/")
|
||||
for p in sorted(ROOT.rglob("*.sh")):
|
||||
rel = str(p.relative_to(ROOT))
|
||||
if rel.startswith((".git/", "conformance/runs/", SELF_DIR)):
|
||||
continue
|
||||
try:
|
||||
text = p.read_text()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
body = "\n".join(l for l in text.splitlines() if not l.lstrip().startswith("#"))
|
||||
if "promtool" in body and re.search(r"test\s+rules", body):
|
||||
harnesses.append(rel)
|
||||
if FETCH_RE.search(body) and not rel.startswith(INSTALLERS):
|
||||
fetchers.append(rel)
|
||||
|
||||
check(harnesses == [str(HARNESS.relative_to(ROOT))],
|
||||
f"expected exactly ONE rule-unit harness ({HARNESS.relative_to(ROOT)}) but found "
|
||||
f"{harnesses}. A second harness means two behaviours and two things to keep in step; the "
|
||||
f"one the gate does not run is the one that rots. Fold it in instead")
|
||||
check(not fetchers,
|
||||
f"{fetchers} acquire a rule ENGINE at run time (a prometheus release download or a "
|
||||
f"prometheus container image). promtool must come from the pinned toolchain "
|
||||
f"(PROMTOOL_VERSION): a run-time fetch decides the version, the network and the machine "
|
||||
f"silently at test time, makes the tests non-hermetic (R-TEST-02), and quietly turns 'the "
|
||||
f"toolchain has no promtool' into a green run — which is how it stayed missing")
|
||||
|
||||
# ------------------------------------------------------------------ discover the alert files
|
||||
# Alert files: tracked YAML carrying a PrometheusRule. conformance/ is excluded deliberately —
|
||||
# the PrometheusRules under tests/l1-rendering/*/golden/ are EXPECTED-RENDER fixtures for the
|
||||
# render checks, not rules any cluster evaluates. Everything a cluster actually gets lives under
|
||||
# apps/, platform/, platform-config/ or clusters/.
|
||||
def tracked_yaml() -> list[Path]:
|
||||
out = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z", "*.yaml", "*.yml"],
|
||||
capture_output=True, text=True)
|
||||
if out.returncode == 0 and out.stdout:
|
||||
return [ROOT / p for p in out.stdout.split("\0") if p]
|
||||
# No usable index (the gate container mounts a worktree whose .git points outside the
|
||||
# mount). Fall back to a filesystem walk rather than auditing nothing.
|
||||
return sorted(p for p in ROOT.rglob("*.y*ml") if ".git/" not in str(p))
|
||||
|
||||
|
||||
KIND_RE = re.compile(r"^\s*kind:\s*PrometheusRule\s*$", re.M)
|
||||
ALERT_RE = re.compile(r"^\s*-\s*alert:\s*(\S+)", re.M)
|
||||
|
||||
alert_files: dict[str, list[str]] = {}
|
||||
for p in tracked_yaml():
|
||||
rel = str(p.relative_to(ROOT))
|
||||
if rel.startswith("conformance/") or not p.is_file():
|
||||
continue
|
||||
try:
|
||||
text = p.read_text()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
if KIND_RE.search(text):
|
||||
alert_files[rel] = ALERT_RE.findall(text)
|
||||
|
||||
check(len(alert_files) > 0,
|
||||
"no PrometheusRule found outside conformance/ — discovery or the repo layout changed, and "
|
||||
"this check would silently audit nothing")
|
||||
|
||||
# ------------------- ask the HARNESS which manifests are under test (never re-derive it here)
|
||||
# Two implementations of "which rules does this test cover" would eventually disagree, and the
|
||||
# disagreement shows up as a file the gate believes is tested and the harness never touches.
|
||||
listing = subprocess.run(["bash", str(HARNESS), "--list"], capture_output=True, text=True,
|
||||
cwd=str(ROOT))
|
||||
check(listing.returncode == 0,
|
||||
f"`{HARNESS.relative_to(ROOT)} --list` failed (rc={listing.returncode}): "
|
||||
f"{(listing.stderr or listing.stdout).strip()[-600:]}")
|
||||
covering: dict[str, list[str]] = defaultdict(list) # manifest -> [test files]
|
||||
for line in listing.stdout.splitlines():
|
||||
if "\t" in line:
|
||||
tfile, manifest = line.split("\t", 1)
|
||||
covering[manifest.strip()].append(tfile.strip())
|
||||
|
||||
for manifest in sorted(covering):
|
||||
check(manifest in alert_files,
|
||||
f"the harness reports {manifest} as under test, but it is not one of the "
|
||||
f"PrometheusRule files found in the repo — the mapping and the tree disagree")
|
||||
|
||||
# --------------------------------------------------------- (2) the engine is the pinned one
|
||||
version_line = ""
|
||||
if PROMTOOL:
|
||||
vp = subprocess.run([PROMTOOL, "--version"], capture_output=True, text=True)
|
||||
out = (vp.stdout + vp.stderr).strip()
|
||||
version_line = out.splitlines()[0] if out else ""
|
||||
m = re.search(r"version\s+(\d+\.\d+\.\d+)", version_line)
|
||||
found = m.group(1) if m else None
|
||||
check(found is not None,
|
||||
f"could not parse a version from `promtool --version`: {version_line!r}")
|
||||
check(found is None or DEV_PIN is None or found == DEV_PIN,
|
||||
f"promtool reports {found}, but the toolchain pins PROMTOOL_VERSION={DEV_PIN} (the "
|
||||
f"Prometheus version prod runs). promtool IS the rule engine: a unit test green under "
|
||||
f"one version and red under another is worse than no test. Run the gate in the pinned "
|
||||
f"devcontainer (rebuild it: ./infra/devcontainer/dev.sh build), or set PROMTOOL= to a "
|
||||
f"v{DEV_PIN} binary. A FAILURE, not a skip, because the wrong engine still prints "
|
||||
f"SUCCESS")
|
||||
|
||||
# ------------------------------------------------- (5) every rule-unit test file must pass
|
||||
if PROMTOOL:
|
||||
env = dict(os.environ, PROMTOOL=PROMTOOL)
|
||||
run = subprocess.run(["bash", str(HARNESS)], capture_output=True, text=True,
|
||||
cwd=str(ROOT), env=env)
|
||||
tailed = "\n ".join((run.stdout + run.stderr).strip().splitlines()[-40:])
|
||||
check(run.returncode == 0,
|
||||
f"`{HARNESS.relative_to(ROOT)}` FAILED (rc={run.returncode}) — the committed alert "
|
||||
f"rules do not behave as their own unit tests say:\n {tailed}")
|
||||
|
||||
# ------------------------------------------- (6) the harness is proven non-vacuous
|
||||
# A synthetic PrometheusRule + companion test through the SAME harness, outside the repo so
|
||||
# discovery never picks it up. Intact must PASS; a mutated expectation must FAIL. Without
|
||||
# this, a harness that stopped evaluating anything (an empty extraction, a rule file written
|
||||
# under the wrong name) would report green forever — which is the disease being treated.
|
||||
work = tempfile.mkdtemp(prefix="alert-rules-selftest.")
|
||||
try:
|
||||
(Path(work) / "synth.yaml").write_text("""apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: selftest
|
||||
spec:
|
||||
groups:
|
||||
- name: selftest
|
||||
rules:
|
||||
- alert: SelfTestCanary
|
||||
expr: selftest_metric > 0
|
||||
for: 5m
|
||||
annotations:
|
||||
runbook_url: https://${BASE_DOMAIN}/runbooks/selftest
|
||||
""")
|
||||
test_yaml = """rule_files:
|
||||
- generated.rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
- interval: 1m
|
||||
name: canary fires only after `for` elapses
|
||||
input_series:
|
||||
- series: 'selftest_metric'
|
||||
values: '1x10'
|
||||
promql_expr_test:
|
||||
- expr: 'ALERTS{alertname="SelfTestCanary",alertstate="firing"}'
|
||||
eval_time: EVAL_TIME
|
||||
exp_samples:
|
||||
- labels: 'ALERTS{alertname="SelfTestCanary",alertstate="firing"}'
|
||||
value: 1
|
||||
"""
|
||||
synth_test = Path(work) / "synth.promtool-test.yaml"
|
||||
synth_test.write_text(test_yaml.replace("EVAL_TIME", "6m"))
|
||||
good = subprocess.run(["bash", str(HARNESS), str(synth_test)],
|
||||
capture_output=True, text=True, cwd=str(ROOT), env=env)
|
||||
check(good.returncode == 0,
|
||||
"self-test: the INTACT synthetic unit test did not pass, so the harness itself is "
|
||||
"broken and every green result above is meaningless:\n "
|
||||
+ "\n ".join((good.stdout + good.stderr).strip().splitlines()[-20:]))
|
||||
# Mutation: assert firing at 2m, which `for: 5m` forbids.
|
||||
synth_test.write_text(test_yaml.replace("EVAL_TIME", "2m"))
|
||||
bad = subprocess.run(["bash", str(HARNESS), str(synth_test)],
|
||||
capture_output=True, text=True, cwd=str(ROOT), env=env)
|
||||
check(bad.returncode != 0,
|
||||
"self-test: a MUTATED expectation (firing before `for:` elapsed) still PASSED — "
|
||||
"the harness is not actually evaluating anything, so every green result above is "
|
||||
"vacuous")
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
||||
# --------------------------------------------------------- (7) the gap inventory + ratchet
|
||||
INVENTORY: list[str] = []
|
||||
GAP_FILES: list[str] = []
|
||||
alerts_total = covered_total = 0
|
||||
|
||||
for rel in sorted(alert_files):
|
||||
names = alert_files[rel]
|
||||
alerts_total += len(names)
|
||||
check(len(names) > 0,
|
||||
f"{rel} declares a PrometheusRule but no `alert:` rule was parsed out of it — the "
|
||||
f"parser or the file shape changed, and any unit test over it would pass vacuously")
|
||||
tests = covering.get(rel, [])
|
||||
if not tests:
|
||||
GAP_FILES.append(rel)
|
||||
INVENTORY.append(f" {rel:<46} alerts={len(names):<3} tested=0/{len(names):<3} NO TEST FILE")
|
||||
check(rel in UNTESTED_FILES,
|
||||
f"{rel} carries {len(names)} alert rule(s) and NO test file covers it. A new "
|
||||
f"PrometheusRule must ship promtool unit tests: this repo has shipped both a "
|
||||
f"permanently-firing critical and a structurally inert alert, and neither was "
|
||||
f"visible without them. Add `{rel[:-5]}.promtool-test.yaml` (companion convention) "
|
||||
f"or a <component>/tests/*.test.yaml suite. If this is deliberate debt it must be "
|
||||
f"DECLARED in UNTESTED_FILES in this check — the register is what keeps the gap "
|
||||
f"visible instead of silent")
|
||||
continue
|
||||
|
||||
check(rel not in UNTESTED_FILES,
|
||||
f"{rel} is listed in UNTESTED_FILES but is now covered by {tests} — the register is "
|
||||
f"shrink-only: delete the line so the next missing test is caught")
|
||||
text = "".join((ROOT / t).read_text() for t in tests if (ROOT / t).is_file())
|
||||
covered = [n for n in names if re.search(rf"\b{re.escape(n)}\b", text)]
|
||||
covered_total += len(covered)
|
||||
for n in names:
|
||||
if n in covered:
|
||||
check((rel, n) not in UNTESTED_ALERTS,
|
||||
f"{rel}: alert {n} is listed in UNTESTED_ALERTS but IS asserted on now — "
|
||||
f"remove the stale entry (the register is shrink-only)")
|
||||
else:
|
||||
check((rel, n) in UNTESTED_ALERTS,
|
||||
f"{rel}: alert {n} has NO unit test in {tests}. This file is already covered, "
|
||||
f"so a new alert in it must be tested too — otherwise a 'tested' alert file "
|
||||
f"quietly grows untested rules")
|
||||
INVENTORY.append(f" {rel:<46} alerts={len(names):<3} tested={len(covered)}/{len(names):<3} "
|
||||
f"{','.join(tests)}")
|
||||
|
||||
# stale register entries
|
||||
for rel in sorted(UNTESTED_FILES - set(alert_files)):
|
||||
check(False, f"UNTESTED_FILES lists {rel}, which carries no PrometheusRule in this repo any "
|
||||
f"more — a stale exemption hides the next real gap; delete the line")
|
||||
for rel in sorted({f for f, _ in UNTESTED_ALERTS} - set(alert_files)):
|
||||
check(False, f"UNTESTED_ALERTS lists alerts in {rel}, which carries no PrometheusRule in this "
|
||||
f"repo any more — delete those lines")
|
||||
|
||||
# --------------------------------------------------------------------------------- report
|
||||
print("Alert-rule unit-test inventory (PrometheusRule -> covering test file(s)):")
|
||||
for line in INVENTORY:
|
||||
print(line)
|
||||
print(f"\nharness: {HARNESS.relative_to(ROOT)} promtool: {version_line or 'NOT FOUND'}")
|
||||
print(f"pinned: PROMTOOL_VERSION={DEV_PIN} (devcontainer) / {CI_PIN} (ci) — prod runs Prometheus "
|
||||
f"v{DEV_PIN}")
|
||||
print(f"alert files: {len(alert_files)} covered by a test file: "
|
||||
f"{len(alert_files) - len(GAP_FILES)} with NO test file: {len(GAP_FILES)}")
|
||||
print(f"alert rules: {alerts_total} unit-tested: {covered_total} "
|
||||
f"NOT unit-tested: {alerts_total - covered_total}")
|
||||
|
||||
if GAP_FILES:
|
||||
print("\nDECLARED GAPS — alert files nothing tests at all (shrink-only register; each is a")
|
||||
print("rule nothing has ever evaluated, NOT a rule known to work):")
|
||||
for g in GAP_FILES:
|
||||
print(f" - {g} ({len(alert_files[g])} alert(s))")
|
||||
if UNTESTED_ALERTS:
|
||||
print("\nDECLARED GAPS — individual alerts inside a covered file that nothing asserts on:")
|
||||
for f, n in sorted(UNTESTED_ALERTS):
|
||||
print(f" - {f} {n}")
|
||||
|
||||
print()
|
||||
if not PROMTOOL and not FAILURES:
|
||||
print("check SKIPPED (exit 3, NOT a pass): promtool is not on PATH, so NO alert rule was "
|
||||
"parsed and NO unit test was evaluated.", file=sys.stderr)
|
||||
print(f" promtool is pinned at v{DEV_PIN} in infra/devcontainer/versions.env + "
|
||||
f"ci/versions.env.", file=sys.stderr)
|
||||
print(" Run the gate in the pinned toolbox (./infra/devcontainer/dev.sh build, then "
|
||||
"make -C ci gate), or set PROMTOOL=/path/to/promtool.", file=sys.stderr)
|
||||
print(f" The {CHECKS} offline assertions above (pins, lockstep, one harness, no run-time "
|
||||
f"engine fetch, the gap register) DID run and passed.", file=sys.stderr)
|
||||
print(" This test declares scope: whole-repo, so the runner reports rc 3 — UNWIRED, never "
|
||||
"green.", file=sys.stderr)
|
||||
raise SystemExit(3)
|
||||
|
||||
if not FAILURES:
|
||||
print(f"check OK: alert-rules-unit-tested — {CHECKS} assertions "
|
||||
f"({covered_total}/{alerts_total} alert rules unit-tested, "
|
||||
f"{len(GAP_FILES)} files with no test file, all declared) (R-OBS-04, R-TEST-02)")
|
||||
raise SystemExit(0)
|
||||
print(f"CHECK FAIL (alert-rules-unit-tested): {len(FAILURES)} failed, "
|
||||
f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" FAIL {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
@@ -1,89 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — every alert's runbook_url resolves to a runbook that is actually served
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# Every platform-health alert rule in this repo carries
|
||||
# `runbook_url: "https://${BASE_DOMAIN}/runbooks/<name>"` — 42 annotations across four files, over 34
|
||||
# distinct names. For the whole of phases 4-6 there was NO runbooks/ directory and NO HTTPRoute serving
|
||||
# that path: an in-cluster probe of https://home.keithjackson.dev/runbooks/node-down returned HTTP 404.
|
||||
# So an alert reaching a human handed them a dead link at the one moment documentation is read, while the
|
||||
# annotation made the problem look solved — the same green-surface-over-absent-mechanism class as the
|
||||
# import Job that reported success while storing placeholder secrets, and the guard whose glob silently
|
||||
# stopped matching.
|
||||
#
|
||||
# The link is served now (runbooks/, clusters/base/runbooks.yaml). This test is the guard that keeps it
|
||||
# true, and it asserts BOTH halves of the claim because the regression is silent in both directions: an
|
||||
# alert with a new runbook_url and no runbook links to a 404, AND a runbook committed without its
|
||||
# configMapGenerator entry is in git, reviewed, merged — and still 404s in the cluster. "The file exists"
|
||||
# and "the URL resolves" are different claims; only the second one helps an operator.
|
||||
#
|
||||
# GIVEN every `runbook_url` annotation on every PrometheusRule in the repo,
|
||||
# WHEN each is parsed out of git and resolved against runbooks/,
|
||||
# THEN (1) each URL has EXACTLY the shape the server resolves — https://${BASE_DOMAIN}/runbooks/<name>,
|
||||
# the ${BASE_DOMAIN} token (never a literal hostname), no `.md`, no trailing slash, no query;
|
||||
# (2) every referenced <name> has a runbooks/<name>.md in git;
|
||||
# (3) each runbook clears a QUALITY FLOOR — the four operator sections, a real body, and the
|
||||
# dev-container kubectl invocation with --context k3s-prod — so a stub cannot satisfy (2) and
|
||||
# make the link look fixed while still helping nobody;
|
||||
# (4) every runbook is PROJECTED INTO THE SERVER by runbooks/kustomization.yaml's
|
||||
# configMapGenerator under the extensionless key the URL asks for;
|
||||
# (5) route, listener and server agree with the annotations (PathPrefix == the annotations' prefix,
|
||||
# a sectionName that platform/gateway/gateway.yaml actually defines, the namespace opted in to
|
||||
# the Gateway, and an nginx location for that same prefix with a text/plain default_type);
|
||||
# (6) the Flux tier is wired into clusters/base AND declares NO postBuild.substituteFrom —
|
||||
# substitution would run envsubst over the runbook prose and silently empty every `${...}`
|
||||
# shell example in it (it fails open: DECISIONS-LOG 2026-09-01, the emptied Cilium
|
||||
# k8sServiceHost), and the flux-postbuild-substitution check cannot see it because that check
|
||||
# scans files reachable through `resources:`, not configMapGenerator inputs;
|
||||
# AND (7) the check is LOAD-BEARING: this Test DECLARES
|
||||
# `conformance.home.k3s/scope: whole-repo`, so ci/l0.sh Stage 3b's
|
||||
# `--include-scope whole-repo` unions it into EVERY gate invocation rather than only the runs
|
||||
# whose touched requirements happen to include R-OBS-04; and ci/gate.sh sets the merge block
|
||||
# when the l0 stage fails. A SKIP of a scope-declaring check is UNWIRED, never green.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library only);
|
||||
# conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw Test wraps the SAME
|
||||
# script.
|
||||
#
|
||||
# NOT proven (named in check.sh's header): this does not FETCH the URL. That it returns HTTP 200 with
|
||||
# real content was proven by an in-cluster probe recorded in the commit message; a live probe needs a
|
||||
# cluster, which L0 by definition does not have. What is asserted here is every static precondition for
|
||||
# that 200. It also does not judge whether a runbook is correct or current — (3) is a floor, not a
|
||||
# review — and it does not prove Alertmanager delivers the annotation to a human (that is the L3
|
||||
# `alerting` test). R-OBS-04 is tagged for the narrow slice only: the platform's own health alerts carry
|
||||
# runbook links that resolve to real, served procedure rather than to a 404.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: alert-runbooks-resolvable
|
||||
labels:
|
||||
conformance.home.k3s/R-OBS-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-OBS-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE — this is what makes the check unconditional. Stage 3b of ci/l0.sh runs
|
||||
# `conformance/run.sh --layer l0 --include-scope whole-repo`, which UNIONS every test carrying this
|
||||
# annotation into the selection regardless of which requirements the change touched. That union is
|
||||
# load-bearing here: ci/touched-requirements.sh maps platform/monitoring/* to "R-OBS-01 R-OBS-02" —
|
||||
# NOT R-OBS-04 — and maps platform/kubevirt/* and runbooks/* to nothing at all, so the single most
|
||||
# likely breaking change (adding an alert with a runbook_url, or adding a runbook without its
|
||||
# configMapGenerator entry) attributes a requirement set that would otherwise FILTER THIS CHECK OUT
|
||||
# of exactly the change that introduces the bug. Declaring the scope replaces the hand-written
|
||||
# per-test ci/l0.sh stage that this repo used to need (and that nobody remembered to add).
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every runbook_url on every alert rule resolves to a runbook that exists in git, clears a quality
|
||||
floor, and is actually projected into the server that answers that path — so an alert can never
|
||||
hand an operator a dead link.
|
||||
spec:
|
||||
steps:
|
||||
- name: audit-every-alert-runbook-link
|
||||
description: >-
|
||||
Parse every runbook_url out of every PrometheusRule and assert the runbook exists, is real, is
|
||||
served at that exact path, and that the serving tier is wired and unsubstituted.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,410 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — alert-runbooks-resolvable
|
||||
# Serves: R-OBS-04 ("The platform alerts on its own health (nodes, storage, GitOps, certs,
|
||||
# VPN)." — spec docs/10-observability-and-alerting.md)
|
||||
#
|
||||
# WHY THIS EXISTS — A DEAD LINK IN AN ALERT IS WORSE THAN NO LINK
|
||||
# --------------------------------------------------------------
|
||||
# Every platform-health alert rule in this repo carries
|
||||
# runbook_url: "https://${BASE_DOMAIN}/runbooks/<name>"
|
||||
# 42 such annotations across four files, over 34 distinct names. For the whole of phases 4-6
|
||||
# there was NO runbooks/ directory and NO HTTPRoute serving that path: a live in-cluster probe
|
||||
# of https://home.keithjackson.dev/runbooks/node-down returned HTTP 404. So an alert reaching a
|
||||
# human at 3am handed them a link that fails, at the one moment the documentation is read. The
|
||||
# annotation LOOKED like the problem was solved, which is the specific failure this repo keeps
|
||||
# finding (an import Job that reports success while storing placeholder secrets; a guard whose
|
||||
# glob silently stopped matching) — a green-looking surface over an absent mechanism.
|
||||
#
|
||||
# The link is now served (runbooks/, clusters/base/runbooks.yaml). This check is the guard that
|
||||
# keeps it true, because the regression is SILENT IN BOTH DIRECTIONS and nothing else catches it:
|
||||
# * add an alert with a new runbook_url and forget the runbook -> that alert links to a 404,
|
||||
# and no test, lint, render or policy notices;
|
||||
# * write runbooks/<name>.md but forget to add it to the ConfigMap generator -> the file is in
|
||||
# git, reviewable, apparently done, and the URL still 404s in the cluster. "The file exists"
|
||||
# and "the URL resolves" are different claims and only the second one helps an operator.
|
||||
# Both halves are asserted below, which is why this check reaches past the markdown into the
|
||||
# kustomization, the HTTPRoute, the nginx config and the Flux tier.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN every `runbook_url` annotation on every PrometheusRule is parsed out of git,
|
||||
# THEN (1) each URL has EXACTLY the shape the server resolves —
|
||||
# https://${BASE_DOMAIN}/runbooks/<name>, `${BASE_DOMAIN}` as the token (never a literal
|
||||
# hostname: a literal is how prod was authored to serve dev's hostnames, see the
|
||||
# ServiceDefinition schema's note), no `.md` suffix, no trailing slash, no query;
|
||||
# (2) every referenced <name> has a runbooks/<name>.md in git;
|
||||
# (3) each runbook clears a QUALITY FLOOR — the four operator sections, a real body, and the
|
||||
# dev-container kubectl invocation — so a one-line stub cannot satisfy (2) and make the
|
||||
# link look fixed while still helping nobody (the failure mode this check is most likely
|
||||
# to be defeated by is a placeholder file, not a missing one);
|
||||
# (4) every runbook markdown file is actually PROJECTED INTO THE SERVER by
|
||||
# runbooks/kustomization.yaml's configMapGenerator, under the EXTENSIONLESS key the URL
|
||||
# asks for — the "exists in git but 404s in the cluster" half;
|
||||
# (5) the route and the server agree with the annotations: the HTTPRoute's PathPrefix is the
|
||||
# prefix the URLs use, it attaches to a Gateway listener that actually exists, its
|
||||
# namespace is opted in to that Gateway, and the nginx config answers that same prefix;
|
||||
# (6) the Flux tier that reconciles all of it is wired into clusters/base AND declares NO
|
||||
# postBuild.substituteFrom — substitution here would run envsubst over the runbook prose
|
||||
# and silently empty every `${...}` shell example in it (it fails open: DECISIONS-LOG
|
||||
# 2026-09-01, the emptied Cilium k8sServiceHost);
|
||||
# AND (7) this check is LOAD-BEARING: ci/l0.sh runs it on every gate run and ci/gate.sh sets the
|
||||
# merge block when the l0 stage fails.
|
||||
#
|
||||
# WHY (7) IS AN UNCONDITIONAL ci/l0.sh STAGE and not left to the requirement-scoped run: Stage 3b
|
||||
# scopes `conformance/run.sh --layer l0` to the TOUCHED requirements (R-TEST-03), and
|
||||
# ci/touched-requirements.sh maps platform/monitoring/* to "R-OBS-01 R-OBS-02" — NOT R-OBS-04 — and
|
||||
# maps platform/kubevirt/* and runbooks/* to nothing at all. So the single most likely breaking
|
||||
# change (adding an alert with a runbook_url to platform/monitoring/alerts.yaml, or adding a
|
||||
# runbook and forgetting the generator entry) attributes a requirement set that would FILTER THIS
|
||||
# CHECK OUT. Declaring `conformance.home.k3s/scope: whole-repo` makes ci/l0.sh Stage 3b union it into
|
||||
# EVERY gate invocation instead; a SKIP of a scope-declaring check is UNWIRED, never green.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — this is a STATIC check):
|
||||
# - It does not fetch the URL. That it returns HTTP 200 with real content was proven by an
|
||||
# in-cluster probe (recorded in the commit message and DECISIONS-LOG); a live probe needs a
|
||||
# cluster, which L0 by definition does not have. What is asserted here is every static
|
||||
# precondition for that 200: the file, its projection, the key name, the route, the prefix.
|
||||
# - It does not judge whether a runbook is CORRECT or current. (3) is a floor, not a review: it
|
||||
# can tell a stub from a runbook, not a good runbook from a stale one. Nothing automatable can.
|
||||
# - It does not prove Alertmanager actually delivers the annotation to a human. That is the L3
|
||||
# `alerting` test.
|
||||
# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06) and is not consulted.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
# Honest SKIP (never green) if there are no alert rules at all — nothing to audit yet.
|
||||
if ! grep -rl 'kind: PrometheusRule' "${ROOT}/platform" >/dev/null 2>&1; then
|
||||
echo "check SKIPPED: no PrometheusRule found under platform/ — no alert rules to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
"""Static audit: every alert runbook_url resolves to a runbook that is actually served.
|
||||
|
||||
Pure standard library, no PyYAML — the same constraint and the same style as the neighbouring
|
||||
L0 checks (lib/coverage.py, lib/status.py). Every parse contract is documented at its parser;
|
||||
all of them are line-constrained shapes in hand-authored manifests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree
|
||||
|
||||
FAILURES: list[str] = []
|
||||
ADVISORIES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def read(p: Path) -> str:
|
||||
try:
|
||||
return p.read_text()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return ""
|
||||
|
||||
|
||||
# =============================================================== (1) parse the annotations
|
||||
# Parse contract: `runbook_url:` is an annotation on a Prometheus alerting rule, authored one per
|
||||
# line as `runbook_url: "<url>"` (single or double quoted, or bare). Only files that declare
|
||||
# `kind: PrometheusRule` are scanned, so a runbook_url quoted inside this check or inside prose
|
||||
# elsewhere in the repo is not mistaken for an alert annotation. Commented-out lines are skipped:
|
||||
# platform/monitoring/alerts.yaml carries a `# runbook_url points at a docs base` header comment.
|
||||
SCOPE_DECL_RE = r"^\s*conformance\.home\.k3s/scope:\s*[\"\']?whole-repo"
|
||||
RUNBOOK_URL_RE = re.compile(r"""^\s*runbook_url:\s*["']?([^"'\s#]+)["']?\s*$""")
|
||||
|
||||
rule_files = sorted(
|
||||
p for p in ROOT.rglob("*.yaml")
|
||||
if ".git" not in p.parts and "kind: PrometheusRule" in read(p)
|
||||
)
|
||||
check(len(rule_files) > 0,
|
||||
"no file declaring `kind: PrometheusRule` found — the parser or the repo layout changed; "
|
||||
"this check would silently audit nothing")
|
||||
|
||||
# url -> [ "<relpath>:<lineno>", ... ]
|
||||
refs: dict[str, list[str]] = {}
|
||||
for f in rule_files:
|
||||
rel = f.relative_to(ROOT)
|
||||
for lineno, line in enumerate(read(f).splitlines(), 1):
|
||||
if line.lstrip().startswith("#"):
|
||||
continue
|
||||
m = RUNBOOK_URL_RE.match(line)
|
||||
if m:
|
||||
refs.setdefault(m.group(1), []).append(f"{rel}:{lineno}")
|
||||
|
||||
n_annotations = sum(len(v) for v in refs.values())
|
||||
check(n_annotations > 0,
|
||||
"no runbook_url annotation parsed out of any PrometheusRule — the parser contract broke; "
|
||||
"this check would pass vacuously while every alert link rots")
|
||||
|
||||
# ---- URL SHAPE. The server resolves exactly one shape. Anything else is a link that 404s even
|
||||
# though a file with a plausible name exists, so shape is a failure, not a style note.
|
||||
URL_RE = re.compile(r"^https://\$\{BASE_DOMAIN\}/(runbooks)/([a-z0-9][a-z0-9-]*)$")
|
||||
names: dict[str, list[str]] = {} # runbook name -> annotation sites
|
||||
prefixes: set[str] = set()
|
||||
for url, sites in sorted(refs.items()):
|
||||
m = URL_RE.match(url)
|
||||
check(m is not None,
|
||||
f"runbook_url '{url}' ({', '.join(sites)}) is not the shape this platform serves. "
|
||||
f"Required: https://${{BASE_DOMAIN}}/runbooks/<name> — the ${{BASE_DOMAIN}} TOKEN (never a "
|
||||
f"literal hostname: a literal is how prod got authored to serve dev's public names), a "
|
||||
f"lowercase-hyphen <name>, and NO '.md' suffix, trailing slash or query. The runbook "
|
||||
f"server maps the URL path straight onto an extensionless ConfigMap key, so any other "
|
||||
f"shape is a 404 no matter which files exist")
|
||||
if m:
|
||||
prefixes.add(m.group(1))
|
||||
names.setdefault(m.group(2), []).extend(sites)
|
||||
|
||||
check(len(prefixes) <= 1,
|
||||
f"alert annotations use more than one runbook path prefix ({sorted(prefixes)}); the single "
|
||||
f"HTTPRoute + nginx location can only serve one, so the others are dead links")
|
||||
PREFIX = (sorted(prefixes) or ["runbooks"])[0]
|
||||
|
||||
# =================================================== (2)(3) the runbooks exist and are real
|
||||
RUNBOOK_DIR = ROOT / PREFIX
|
||||
check(RUNBOOK_DIR.is_dir(),
|
||||
f"{PREFIX}/ is not a directory in this repo, so every one of the {n_annotations} "
|
||||
f"runbook_url annotations is a dead link")
|
||||
|
||||
# The quality floor. (2) alone is satisfiable by `touch runbooks/node-down.md`, which makes the
|
||||
# link resolve and still abandons the operator — a worse outcome than the 404, because the 404 at
|
||||
# least told the truth. These four sections are the operator's path through an incident: what the
|
||||
# alert means HERE, what to type first, what is probably wrong, and when to stop. `dev.sh` +
|
||||
# `--context k3s-prod` are required because kubectl does not work on the workstation at all
|
||||
# (the toolchain lives only in the pinned dev container) and because dev.sh defaults its
|
||||
# current-context to k3s-DEV — a runbook that omits the context teaches the reader to answer
|
||||
# questions about the wrong cluster during an incident on this one.
|
||||
REQUIRED_SECTIONS = (
|
||||
"## What this means here",
|
||||
"## First commands",
|
||||
"## Likely causes, most likely first",
|
||||
"## When to escalate or stop",
|
||||
)
|
||||
MIN_BYTES = 1500
|
||||
|
||||
for name, sites in sorted(names.items()):
|
||||
md = RUNBOOK_DIR / f"{name}.md"
|
||||
if not md.is_file():
|
||||
check(False,
|
||||
f"runbook '{name}' does not exist: {md.relative_to(ROOT)} is missing, but "
|
||||
f"{len(sites)} alert annotation(s) link to it ({', '.join(sites[:3])}"
|
||||
f"{', ...' if len(sites) > 3 else ''}). An alert firing now hands the operator a 404. "
|
||||
f"Write {PREFIX}/{name}.md (see {PREFIX}/README.md for the required sections)")
|
||||
continue
|
||||
body = read(md)
|
||||
check(len(body) >= MIN_BYTES,
|
||||
f"{md.relative_to(ROOT)} is {len(body)} bytes, under the {MIN_BYTES}-byte floor — a stub "
|
||||
f"that makes the link resolve while still helping nobody is worse than the 404 it "
|
||||
f"replaced, because it looks fixed")
|
||||
for sec in REQUIRED_SECTIONS:
|
||||
check(sec in body,
|
||||
f"{md.relative_to(ROOT)} has no '{sec}' section — the four operator sections are what "
|
||||
f"make this a runbook rather than a restatement of the alert name")
|
||||
check("dev.sh" in body,
|
||||
f"{md.relative_to(ROOT)} never mentions dev.sh — kubectl does not exist on the "
|
||||
f"workstation; every command in a runbook here must be runnable as "
|
||||
f"`infra/devcontainer/dev.sh run bash -c '...'` and the runbook must say so")
|
||||
check("--context k3s-prod" in body,
|
||||
f"{md.relative_to(ROOT)} never passes --context k3s-prod — dev.sh merges both clusters "
|
||||
f"and picks a current-context PER INVOCATION — a default that has already changed once "
|
||||
f"(k3s-dev until dev was decommissioned on 2026-09-13, k3s-prod now). A context-less command "
|
||||
f"silently answers about whichever cluster the default happens to be, in the middle of an "
|
||||
f"incident")
|
||||
|
||||
# ADVISORY, not a failure: a runbook nothing links to yet. Writing the runbook BEFORE the alert is
|
||||
# the right order, and the repo's precedent for "authored but unreferenced" is advisory (see the
|
||||
# dead-knob advisory in flux-postbuild-substitution/check.sh).
|
||||
served_extra = {"README"}
|
||||
if RUNBOOK_DIR.is_dir():
|
||||
for md in sorted(RUNBOOK_DIR.glob("*.md")):
|
||||
stem = md.stem
|
||||
if stem in served_extra or stem in names:
|
||||
continue
|
||||
ADVISORIES.append(f"{md.relative_to(ROOT)} is a runbook no alert links to (written ahead of "
|
||||
f"its alert, or the alert was removed — advisory, not a failure)")
|
||||
|
||||
# ================================= (4) every runbook is PROJECTED INTO THE SERVER, not just in git
|
||||
# THIS IS THE HALF A FILE-EXISTENCE CHECK WOULD MISS. The pod serves a ConfigMap, not the git tree.
|
||||
# Parse contract: runbooks/kustomization.yaml declares one configMapGenerator whose `files:` entries
|
||||
# are `- <key>=<file>.md`, one per line. The key is EXTENSIONLESS on purpose (the URL carries no
|
||||
# `.md`, and the ConfigMap volume mount turns each key into a file the nginx root resolves directly).
|
||||
KUST = RUNBOOK_DIR / "kustomization.yaml"
|
||||
check(KUST.is_file(), f"{PREFIX}/kustomization.yaml is missing — the Flux tier renders nothing and "
|
||||
f"every runbook 404s no matter what is in git")
|
||||
GEN_FILE_RE = re.compile(r"^\s*-\s*([A-Za-z0-9._-]+)=([A-Za-z0-9._/-]+\.md)\s*(?:#.*)?$")
|
||||
generated: dict[str, str] = {}
|
||||
kust_text = read(KUST)
|
||||
if kust_text:
|
||||
check("configMapGenerator:" in kust_text,
|
||||
f"{PREFIX}/kustomization.yaml has no configMapGenerator — the runbook markdown is not "
|
||||
f"rendered into the ConfigMap the server mounts")
|
||||
for line in kust_text.splitlines():
|
||||
if line.lstrip().startswith("#"):
|
||||
continue
|
||||
m = GEN_FILE_RE.match(line)
|
||||
if m:
|
||||
generated[m.group(1)] = m.group(2)
|
||||
check(len(generated) > 0,
|
||||
f"{PREFIX}/kustomization.yaml declares no `<key>=<file>.md` generator entries — the "
|
||||
f"parser contract broke, or the markdown is being added with implicit keys (which would "
|
||||
f"carry the '.md' suffix the alert URLs do NOT have, so every link would 404)")
|
||||
|
||||
for name in sorted(names):
|
||||
check(generated.get(name) == f"{name}.md",
|
||||
f"runbook '{name}' is not projected into the server: {PREFIX}/kustomization.yaml has no "
|
||||
f"configMapGenerator entry `- {name}={name}.md`. The file can be in git, reviewed and "
|
||||
f"merged and the URL still returns 404 — 'the file exists' and 'the URL resolves' are "
|
||||
f"different claims")
|
||||
|
||||
# And the reverse, so a generator entry cannot point at a file that is not there.
|
||||
for key, fname in sorted(generated.items()):
|
||||
check((RUNBOOK_DIR / fname).is_file(),
|
||||
f"{PREFIX}/kustomization.yaml generates key '{key}' from {PREFIX}/{fname}, which does not "
|
||||
f"exist — `kustomize build` fails, so the WHOLE tier stops reconciling and every runbook "
|
||||
f"goes stale at once")
|
||||
|
||||
# ============================== (5) the route, the listener and the server agree with the URLs
|
||||
ROUTE = RUNBOOK_DIR / "serving" / "httproute.yaml"
|
||||
NGINX = RUNBOOK_DIR / "serving" / "nginx-config.yaml"
|
||||
NS = RUNBOOK_DIR / "serving" / "namespace.yaml"
|
||||
GATEWAY = ROOT / "platform" / "gateway" / "gateway.yaml"
|
||||
|
||||
route_text = read(ROUTE)
|
||||
check(bool(route_text), f"{PREFIX}/serving/httproute.yaml is missing — nothing routes the "
|
||||
f"/{PREFIX} prefix to the server")
|
||||
# Parse contract: a single HTTPRoute with one PathPrefix match and one parentRefs entry naming a
|
||||
# listener via sectionName; hand-authored, one key per line.
|
||||
m_path = re.search(r"^\s*type:\s*PathPrefix\s*$\n\s*value:\s*(\S+)\s*$", route_text, re.M)
|
||||
check(m_path is not None and m_path.group(1) == f"/{PREFIX}",
|
||||
f"{PREFIX}/serving/httproute.yaml does not match PathPrefix /{PREFIX} "
|
||||
f"(found {m_path.group(1) if m_path else 'no PathPrefix match'}). The alert annotations point "
|
||||
f"at /{PREFIX}/<name>; if the route's prefix drifts from the annotations the links 404 while "
|
||||
f"both files look individually correct")
|
||||
m_sec = re.search(r"^\s*sectionName:\s*(\S+)", route_text, re.M)
|
||||
check(m_sec is not None,
|
||||
f"{PREFIX}/serving/httproute.yaml declares no parentRefs sectionName. The apex host needs the "
|
||||
f"dedicated apex listener: the wildcard `*.${{BASE_DOMAIN}}` listener does NOT match the bare "
|
||||
f"apex, so a route without sectionName binds nothing and serves nothing")
|
||||
if m_sec:
|
||||
listener = m_sec.group(1)
|
||||
gw = read(GATEWAY)
|
||||
check(re.search(rf"^\s*-\s*name:\s*{re.escape(listener)}\s*$", gw, re.M) is not None,
|
||||
f"{PREFIX}/serving/httproute.yaml attaches to Gateway listener '{listener}', which "
|
||||
f"platform/gateway/gateway.yaml does not define — the route would report NotAllowed and "
|
||||
f"never serve")
|
||||
# An HTTPRoute with no `hostnames:` inherits the listener's hostname, which is how this tier avoids
|
||||
# needing Flux substitution (see (6)). Either form is routable; a LITERAL hostname is not acceptable,
|
||||
# for the same reason the schema forbids it in a ServiceDefinition.
|
||||
m_host = re.search(r"^\s*hostnames:\s*$\n\s*-\s*(\S+)", route_text, re.M)
|
||||
check(m_host is None or "${BASE_DOMAIN}" in m_host.group(1),
|
||||
f"{PREFIX}/serving/httproute.yaml hardcodes hostname '{m_host.group(1) if m_host else ''}' — "
|
||||
f"use the ${{BASE_DOMAIN}} token, or omit `hostnames:` entirely and inherit the listener's")
|
||||
|
||||
ns_text = read(NS)
|
||||
check('gateway-access: "shared"' in ns_text or "gateway-access: shared" in ns_text,
|
||||
f"{PREFIX}/serving/namespace.yaml does not carry gateway-access=shared — the shared Gateway "
|
||||
f"selects namespaces by that label, so without it the HTTPRoute is refused attachment")
|
||||
|
||||
nginx_text = read(NGINX)
|
||||
check(f"location /{PREFIX}/" in nginx_text,
|
||||
f"{PREFIX}/serving/nginx-config.yaml has no `location /{PREFIX}/` block — the route would "
|
||||
f"deliver the request and the server would answer 404 for every runbook")
|
||||
check("default_type" in nginx_text and "text/plain" in nginx_text,
|
||||
f"{PREFIX}/serving/nginx-config.yaml does not force a text/plain default_type. The ConfigMap "
|
||||
f"keys are EXTENSIONLESS, so with the stock MIME table nginx serves them as "
|
||||
f"application/octet-stream and a browser DOWNLOADS the runbook instead of showing it")
|
||||
|
||||
# ====================================== (6) the Flux tier is wired, and is NOT substituted
|
||||
FLUX = ROOT / "clusters" / "base" / f"{PREFIX}.yaml"
|
||||
BASE_KUST = ROOT / "clusters" / "base" / "kustomization.yaml"
|
||||
flux_text = read(FLUX)
|
||||
check(bool(flux_text),
|
||||
f"clusters/base/{PREFIX}.yaml is missing — nothing reconciles {PREFIX}/ into the cluster, so "
|
||||
f"the runbooks exist in git and the URL still 404s")
|
||||
check(re.search(rf"^\s*path:\s*\./{re.escape(PREFIX)}\s*$", flux_text, re.M) is not None,
|
||||
f"clusters/base/{PREFIX}.yaml does not set `path: ./{PREFIX}`")
|
||||
check(re.search(r"^\s*wait:\s*false\s*$", flux_text, re.M) is not None,
|
||||
f"clusters/base/{PREFIX}.yaml is not wait:false — a leaf web server that can be legitimately "
|
||||
f"unhealthy must never gate the reconcile chain (DECISIONS-LOG: 'Headlamp gating stall')")
|
||||
# Strip comments before testing: this file's own comment block EXPLAINS at length why
|
||||
# substituteFrom must never be added here, and a naive substring test would fire on the warning.
|
||||
flux_code = "\n".join(l for l in flux_text.splitlines() if not l.lstrip().startswith("#"))
|
||||
check("substituteFrom" not in flux_code,
|
||||
f"clusters/base/{PREFIX}.yaml declares postBuild.substituteFrom. It MUST NOT: this tier "
|
||||
f"renders documentation into a ConfigMap, and Flux's envsubst would run over the prose and "
|
||||
f"silently EMPTY every `${{...}}` shell example in every runbook — it fails open, so the "
|
||||
f"Kustomization would still report Ready (DECISIONS-LOG 2026-09-01, the emptied Cilium "
|
||||
f"k8sServiceHost). The flux-postbuild-substitution check cannot see this: it scans files "
|
||||
f"reachable through `resources:`, not configMapGenerator inputs")
|
||||
check(re.search(rf"^\s*-\s*{re.escape(PREFIX)}\.yaml\b", read(BASE_KUST), re.M) is not None,
|
||||
f"clusters/base/kustomization.yaml does not list {PREFIX}.yaml — the Flux Kustomization is "
|
||||
f"authored and never applied, which is the 'looks done, is absent' class this repo has been "
|
||||
f"bitten by repeatedly")
|
||||
|
||||
# ============================================ (7) this check is load-bearing in the merge gate
|
||||
L0 = read(ROOT / "ci" / "l0.sh")
|
||||
GATE = read(ROOT / "ci" / "gate.sh")
|
||||
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism, in two halves.
|
||||
# It is not enough that check.sh exists: the merge gate must SELECT it on every run. The repo used to
|
||||
# get that with a hand-written "UNCONDITIONAL" stage per whole-repo check in ci/l0.sh, hardcoding the
|
||||
# path; those stages are gone, because the next whole-repo invariant only got one if somebody
|
||||
# REMEMBERED to write it. The property is now DECLARED by the test and HONOURED by the gate, so both
|
||||
# halves are asserted — neither alone is sufficient. The mechanism itself (that the union really
|
||||
# selects, and that a scope-declared SKIP exits 3 rather than passing green) is proven end-to-end and
|
||||
# falsifiably by conformance/tests/l0-static/whole-repo-invariants-wired.
|
||||
SELF_YAML = (ROOT / "conformance" / "tests" / "l0-static" / "alert-runbooks-resolvable"
|
||||
/ "chainsaw-test.yaml")
|
||||
SELF = read(SELF_YAML)
|
||||
check(re.search(SCOPE_DECL_RE, SELF, re.M) is not None,
|
||||
"alert-runbooks-resolvable/chainsaw-test.yaml does not declare "
|
||||
"conformance.home.k3s/scope: whole-repo, so the merge gate selects this check only when a "
|
||||
"change happens to touch a requirement it is labelled with. ci/touched-requirements.sh maps "
|
||||
"platform/monitoring/* to 'R-OBS-01 R-OBS-02' (NOT R-OBS-04) and maps platform/kubevirt/* and "
|
||||
"runbooks/* to nothing at all — so adding an alert with a new runbook_url, or adding a runbook "
|
||||
"without its configMapGenerator entry, would be filtered out of exactly the change that "
|
||||
"introduces the dead link")
|
||||
check("--include-scope" in L0,
|
||||
"ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is a "
|
||||
"purely requirement-scoped run again and the scope annotation above is inert decoration — this "
|
||||
"check would then run only by coincidence")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
|
||||
# ======================================================================== report
|
||||
print(f"Alert runbook inventory: {n_annotations} runbook_url annotation(s) across "
|
||||
f"{len(rule_files)} PrometheusRule file(s) -> {len(names)} distinct runbook(s), "
|
||||
f"prefix /{PREFIX}")
|
||||
missing = sorted(n for n in names if not (RUNBOOK_DIR / f"{n}.md").is_file())
|
||||
print(f" runbooks present: {len(names) - len(missing)}/{len(names)} "
|
||||
f"projected into the ConfigMap: {sum(1 for n in names if generated.get(n) == f'{n}.md')}/{len(names)}")
|
||||
if missing:
|
||||
print(f" MISSING: {', '.join(missing)}", file=sys.stderr)
|
||||
|
||||
if ADVISORIES:
|
||||
print("\nADVISORIES (not failures):")
|
||||
for a in ADVISORIES:
|
||||
print(f" - {a}")
|
||||
|
||||
print()
|
||||
if not FAILURES:
|
||||
print(f"check OK: alert-runbooks-resolvable — {CHECKS} assertions (R-OBS-04)")
|
||||
raise SystemExit(0)
|
||||
print(f"CHECK FAIL (alert-runbooks-resolvable): {len(FAILURES)} failed, "
|
||||
f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" FAIL {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
@@ -77,14 +77,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-SEC-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. This is a META-invariant over the ENTIRE conformance suite: the five
|
||||
# spec-named proofs must each still have an EXECUTABLE test at the right layer, with a
|
||||
# pass sentinel, resolvable by the documented command. Every one of its inputs lives under
|
||||
# conformance/, which ci/touched-requirements.sh keys to R-TEST-01..04 — never to R-SEC-03.
|
||||
# So downgrading conformance/tests/l3-fault-injection/dr-restore to a scaffold attributes
|
||||
# R-STATE-03/R-TEST-*, and requirement-scoped selection drops the one check that notices
|
||||
# the audit evidence just stopped being producible.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -250,11 +250,6 @@ try:
|
||||
sys.dont_write_bytecode = True
|
||||
import record_run as rr # noqa: E402
|
||||
|
||||
# status.py owns the latest-wins rule; import it rather than reimplementing it, so the two
|
||||
# cannot disagree about which rows are live evidence.
|
||||
import status as _st # noqa: E402
|
||||
rr_latest_by_test = _st.latest_by_test
|
||||
|
||||
smap = rr.sentinel_map(CONF / "tests")
|
||||
check(len(smap) >= 10,
|
||||
f"record_run.py's sentinel map contains {len(smap)} test(s) — the parser lost its "
|
||||
@@ -391,15 +386,6 @@ try:
|
||||
check(not bad,
|
||||
f"conformance/runs/ledger.jsonl has unparsable line(s) {bad} — the audit record "
|
||||
"is not machine-readable, so no proof can be derived from it")
|
||||
# Which rows are HONOURED: status.py resolves duplicates latest-wins (latest_by_test),
|
||||
# so a superseded row is not evidence of anything. The dirty-tree assertion below is
|
||||
# therefore scoped to honoured rows. Without that scoping, ONE historical dirty row reds
|
||||
# this gate permanently even after a clean re-run of the same test — and a dirty-then-
|
||||
# clean sequence is the normal shape of "noticed the tree was dirty, fixed it, re-ran".
|
||||
# The row is still kept in the ledger and its run.log stays committed: nothing is erased,
|
||||
# it simply stops being treated as a live claim. Structural assertions (provenance keys,
|
||||
# sentinel, test-still-exists) still apply to EVERY row.
|
||||
honoured_ids = {id(v) for v in rr_latest_by_test(rows).values()}
|
||||
for r in rows:
|
||||
for key in ("ts_utc", "git_sha", "test", "requirements", "junit_result", "proof",
|
||||
"run_id"):
|
||||
@@ -407,22 +393,13 @@ try:
|
||||
f"a ledger row for {r.get('test', '?')!r} lacks {key!r} — a row without "
|
||||
"provenance cannot be audited")
|
||||
if r.get("proof") == "proven":
|
||||
# L0/L1 rows are recorded proven WITHOUT a sentinel, deliberately, by
|
||||
# record_run.py's own SENTINEL_EXEMPT_LAYERS: run_l1_offline.py renders skips as
|
||||
# JUnit <skipped/>, so those layers have no pass token to attribute. This
|
||||
# assertion had NO layer exemption, so the two halves of the harness contradicted
|
||||
# each other — ANY `--record` run touching L0 or L1 wrote a row this gate then
|
||||
# rejected, which meant a requirement whose verifier set includes an L1 check
|
||||
# (R-NET-06 is one) could never be both PROVEN and L0-green. Honour the same
|
||||
# constant rather than restating the policy here, so the two cannot drift apart.
|
||||
if r.get("layer") not in rr.SENTINEL_EXEMPT_LAYERS:
|
||||
check(bool(r.get("sentinel")),
|
||||
f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — "
|
||||
"the sentinel is the only thing separating a real pass from a self-skip")
|
||||
check(bool(r.get("sentinel")),
|
||||
f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — "
|
||||
"the sentinel is the only thing separating a real pass from a self-skip")
|
||||
check(r.get("test") in smap,
|
||||
f"a ledger row claims `proven` for {r.get('test')!r}, which is not a "
|
||||
"test in the suite any more — a proof of something that no longer exists")
|
||||
check(id(r) not in honoured_ids or "-dirty" not in str(r.get("git_sha", "")),
|
||||
check("-dirty" not in str(r.get("git_sha", "")),
|
||||
f"a ledger row claims `proven` for {r.get('test')!r} from a DIRTY tree "
|
||||
f"({r.get('git_sha')}) — what ran is not what is committed")
|
||||
else:
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the bulk tier's capacity strategy is DISTINCT from the replicated tier's
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# THE BUG CLASS THIS CLOSES. The `bulk` StorageClass existed from Phase 0 as a stub:
|
||||
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME`, TODO(OQ-2).
|
||||
# Four ServiceDefinitions carried a comment promising to attach their media once OQ-2 was
|
||||
# resolved, and the ONLY thing keeping a multi-terabyte library off the 3-replica
|
||||
# `longhorn` class was those four comments. A bulk volume that lands on the replicated
|
||||
# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
|
||||
# three mini-PCs cannot hold. R-STATE-04 is precisely that the two tiers are distinct, so
|
||||
# the distinctness is asserted rather than reviewed.
|
||||
#
|
||||
# It also pins the four decisions that make the STATIC form of this tier safe, each with a
|
||||
# plausible "simplification" that breaks something silently: `Retain` (bulk data pre-exists
|
||||
# the cluster and has consumers outside it, so anything else makes `delete pvc` a data-loss
|
||||
# operation); NO dynamic provisioning (a dynamic bulk class hands out a FRESH EMPTY
|
||||
# directory that mounts and Binds perfectly while the real 8 TiB is never attached — the
|
||||
# most deceptive failure available here); `hard` not `soft` (a soft mount hands a server
|
||||
# hiccup to the app as a short read, which for a media library is silent corruption); and
|
||||
# NFSv4 (mount and lock in-protocol ⇒ ONE reachable port, tcp/2049, which is what the
|
||||
# operator must open in pfSense for prod to reach this server at all).
|
||||
#
|
||||
# AND the EXDEV trap, which is not about storage. Sonarr/Radarr import by hardlinking or
|
||||
# atomically renaming, both same-filesystem operations, and to the kernel "same filesystem"
|
||||
# means SAME MOUNT. Splitting one export into two subPath mounts looks TIGHTER and breaks
|
||||
# link() with EXDEV; the arrs then degrade to copy-then-delete, rewriting every import over
|
||||
# NFS and destroying the seeding copy. It surfaces weeks later as "imports are slow and my
|
||||
# ratio is gone". A reviewer cannot see that; this check can.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN every `class: bulk` storage entry in catalog/ is compared against the objects the
|
||||
# COMMITTED apps/ goldens carry for it (what Flux reconciles — deliberately not the
|
||||
# renderer, which would only prove it agrees with itself),
|
||||
# THEN (1) no bulk volume is on a Longhorn-provisioned class, and its class is explicitly
|
||||
# EMPTY so nothing can dynamically provision an empty stand-in;
|
||||
# (2) its capacity is declared and is at least an order of magnitude above the same
|
||||
# service's replicated volume — the requirement stated as arithmetic;
|
||||
# (3) PV and PVC are pre-bound from BOTH ends (claimRef + volumeName) and agree;
|
||||
# (4) the PV reclaim policy is Retain;
|
||||
# (5) mountOptions carry `hard` and exactly one `nfsvers=4.x`, and never `soft`;
|
||||
# (6) no workload mounts one export more than once (the EXDEV trap);
|
||||
# (7) a readOnly entry renders nfs.readOnly AND a readOnly volumeMount AND
|
||||
# ReadOnlyMany; a writable one renders neither and is ReadWriteMany; every bulk
|
||||
# access mode is a `...Many`, since one export serves several namespaces and
|
||||
# several nodes;
|
||||
# (8) the PV's nfs.server is a BARE cluster-vars token (never a literal — dev and
|
||||
# prod reach this host by different addresses), that knob has a base default, and
|
||||
# EVERY overlay supplies it;
|
||||
# AND (9) the check is LOAD-BEARING (ci/l0.sh runs it unconditionally, ci/gate.sh blocks
|
||||
# on the l0 stage) and NON-VACUOUS (an empty bulk corpus FAILS rather than
|
||||
# passing trivially, and a floor on the assertion count keeps a pass from
|
||||
# degenerating into a no-op).
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library +
|
||||
# PyYAML); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test wraps the
|
||||
# SAME script.
|
||||
#
|
||||
# NOT proven — the runtime half, which needs a cluster: that the server is reachable, that
|
||||
# the export admits these clients, that the PVC binds, that a write succeeds, or that a
|
||||
# hardlink between the library and the download path actually works. The uid analysis is
|
||||
# reasoning (docs/BULK-NFS-STORAGE.md §6) and the in-cluster probes that would settle it
|
||||
# are written down in §9 of that document, unrun. Nor can this prove prod's address is
|
||||
# CORRECT — only that it is a supplied knob; while it is a placeholder, prod's pods will
|
||||
# not mount, and catching that statically needs the placeholder guard, which is absent from
|
||||
# main (§7). Read-only on the media tree is a CLIENT-side guarantee only: the export serves
|
||||
# one reader and two writers from the same node addresses, so the server cannot tell them
|
||||
# apart (§5).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: bulk-tier-distinct-from-replicated
|
||||
labels:
|
||||
conformance.home.k3s/R-STATE-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-STATE-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE — relocated verbatim from ci/l0.sh's hand-written "Stage 3e
|
||||
# (UNCONDITIONAL)", which this annotation replaces:
|
||||
#
|
||||
# The `bulk` StorageClass sat as a REPLACE-ME stub for six roadmap phases, and the only
|
||||
# thing keeping a multi-terabyte media library off the 3-replica `longhorn` class was four
|
||||
# hand-written comments in ServiceDefinitions. A bulk volume that lands on the replicated
|
||||
# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
|
||||
# three mini-PCs cannot hold.
|
||||
#
|
||||
# WHY IT MUST NOT BE REQUIREMENT-SCOPED: ci/touched-requirements.sh's path map keys
|
||||
# NEITHER apps/* nor platform/longhorn/* to the bulk requirement. So the change most
|
||||
# likely to break this — moving a bulk volume onto the replicated tier, or narrowing one
|
||||
# export into two mounts — could attribute no requirement at all and be gated by nothing.
|
||||
# The invariant spans catalog/, apps/ and clusters/.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Bulk media storage is provably NOT on the 3-replica Longhorn tier, is sized for a
|
||||
different order of magnitude, cannot be silently replaced by a dynamically
|
||||
provisioned empty volume, is never reclaimable by the cluster, mounts hard over
|
||||
NFSv4, and is never split into two mounts of one export — so hardlinked imports keep
|
||||
working. Its server is a per-environment knob, not a literal, because the two
|
||||
clusters reach the same storage host by different addresses.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-bulk-tier-is-distinct-and-statically-bound
|
||||
description: >-
|
||||
Read every `class: bulk` storage entry from catalog/, pair it with the
|
||||
PersistentVolume, PersistentVolumeClaim and volumeMount the committed apps/ goldens
|
||||
carry for it, and assert the tier separation, the static pre-binding, the reclaim
|
||||
and mount-option decisions, the read-only coupling, the one-export-one-mount
|
||||
invariant, and that the server address is supplied by every environment overlay.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,496 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated
|
||||
# Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the
|
||||
# replicated tier." — spec docs/04-storage-and-state.md)
|
||||
#
|
||||
# WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY
|
||||
# ----------------------------------------------------------------------------------------
|
||||
# `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with
|
||||
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2).
|
||||
# Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is
|
||||
# resolved", and each of them said, correctly, that adding `class: bulk` before then would
|
||||
# leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing
|
||||
# behind it, and the only thing preventing a multi-terabyte media library from being
|
||||
# authored onto the 3-replica `longhorn` class was four hand-written comments.
|
||||
#
|
||||
# That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk
|
||||
# storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the
|
||||
# replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on
|
||||
# `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the
|
||||
# tier fills up. Distinctness is therefore a property worth asserting statically, on every
|
||||
# gate run, rather than trusting to review.
|
||||
#
|
||||
# It also pins the four decisions that make the STATIC form of this tier safe, each of
|
||||
# which has a plausible-looking "simplification" that silently breaks something:
|
||||
#
|
||||
# * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has
|
||||
# consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an
|
||||
# inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl
|
||||
# delete pvc` a data-loss operation.
|
||||
# * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The
|
||||
# pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of
|
||||
# existing data was never attached. This is the single most deceptive failure available
|
||||
# in this tier, so `storageClassName` must be explicitly empty and binding must be
|
||||
# pinned from BOTH ends (claimRef + volumeName).
|
||||
# * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or
|
||||
# EIO handed to the application. For a media library that is silent corruption. `hard`
|
||||
# hangs instead — recoverable, where truncated is not.
|
||||
# * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port
|
||||
# (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is
|
||||
# a FIREWALL fact as much as a storage one: prod reaches this server through pfSense,
|
||||
# and the number of ports the operator must open is a consequence of this field.
|
||||
#
|
||||
# AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6).
|
||||
# Sonarr and Radarr import by hardlinking or atomically renaming a finished download into
|
||||
# the library. Both are same-filesystem operations, and to the kernel "same filesystem"
|
||||
# means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`)
|
||||
# looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no
|
||||
# matter that the server knows they are one dataset. The arrs do not fail loudly on that:
|
||||
# they degrade to copy-then-delete, rewriting every imported file over NFS and destroying
|
||||
# the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is
|
||||
# gone". A reviewer cannot see it, so this check does: one export, at most one mount, per
|
||||
# workload.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN every `class: bulk` storage entry in catalog/ is compared against the manifests
|
||||
# render.py produces for it (read from the committed apps/ goldens, i.e. what Flux
|
||||
# actually reconciles — not from the renderer, which would only agree with itself),
|
||||
# THEN (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC
|
||||
# storageClassName is explicitly empty — never `longhorn`, never any class the
|
||||
# repo declares with the Longhorn provisioner — and its PV's source is an
|
||||
# EXTERNAL one (nfs), not a Longhorn volume;
|
||||
# (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the
|
||||
# bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own
|
||||
# replicated volume. This is the requirement stated as arithmetic: the two tiers
|
||||
# exist because the data has two magnitudes, so a "bulk" volume sized like a
|
||||
# config PVC means the strategy was named and not actually applied;
|
||||
# (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its
|
||||
# PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and
|
||||
# the two agree — so no other claim can win a race, and nothing can provision an
|
||||
# empty replacement;
|
||||
# (4) RECLAIM: every bulk PV is `Retain`;
|
||||
# (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an
|
||||
# `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`;
|
||||
# (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above);
|
||||
# (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH
|
||||
# `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount,
|
||||
# and its access mode is ReadOnlyMany; a writable one renders NEITHER and is
|
||||
# ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed
|
||||
# by several namespaces and several nodes, which ReadWriteOnce cannot express;
|
||||
# (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk
|
||||
# PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that
|
||||
# knob has a base default; and EVERY environment overlay supplies it — so the two
|
||||
# clusters, which reach this storage by different addresses, cannot be served by
|
||||
# one hardcoded value;
|
||||
# AND (9) LOAD-BEARING + NON-VACUOUS: this check DECLARES `conformance.home.k3s/scope:
|
||||
# whole-repo`, so ci/l0.sh runs it unconditionally via --include-scope, and
|
||||
# ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT
|
||||
# empty — a repo with zero bulk volumes fails here rather than passing trivially,
|
||||
# because a check that is satisfiable by deleting the feature proves nothing.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps)
|
||||
# - Nothing about RUNTIME. Not that the server is reachable, not that the export permits
|
||||
# these clients, not that the PVC binds, not that a write succeeds. The uid analysis in
|
||||
# docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the
|
||||
# in-cluster proof — they need a cluster and are not run here.
|
||||
# - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While
|
||||
# prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md
|
||||
# §7); catching that needs the placeholder guard, which is absent from main.
|
||||
# - Read-only for the media tree is a CLIENT-side guarantee, and this check can only
|
||||
# assert the client side. The export serves a reader and two writers from the same node
|
||||
# addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5).
|
||||
# - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It
|
||||
# catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a
|
||||
# static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit.
|
||||
# - Assertion (6) reasons over ONE workload's mounts. It cannot see two different
|
||||
# services hardlinking between each other's mounts, which is not a thing that happens.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
|
||||
if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then
|
||||
echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def bail(msg):
|
||||
"""A failure that makes the rest of the check meaningless: report and stop."""
|
||||
print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def qty_to_bytes(q):
|
||||
"""Kubernetes quantity -> bytes. Only the binary suffixes the schema permits."""
|
||||
m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip())
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)]
|
||||
|
||||
|
||||
# =====================================================================================
|
||||
# The replicated tier, named by the repo rather than by this file. Any StorageClass the
|
||||
# repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1);
|
||||
# deriving it means adding a new Longhorn class cannot create a blind spot here.
|
||||
# =====================================================================================
|
||||
REPLICATED_CLASSES: set[str] = set()
|
||||
LONGHORN_DIR = ROOT / "platform" / "longhorn"
|
||||
if LONGHORN_DIR.is_dir():
|
||||
for f in sorted(LONGHORN_DIR.rglob("*.yaml")):
|
||||
for doc in yaml.safe_load_all(f.read_text()):
|
||||
if isinstance(doc, dict) and doc.get("kind") == "StorageClass":
|
||||
if "longhorn" in str(doc.get("provisioner", "")):
|
||||
REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", ""))
|
||||
# `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in
|
||||
# the set even if platform/longhorn/ is not present in this checkout.
|
||||
REPLICATED_CLASSES.add("longhorn")
|
||||
REPLICATED_CLASSES.discard("")
|
||||
|
||||
# =====================================================================================
|
||||
# The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired
|
||||
# with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling
|
||||
# the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is
|
||||
# a separate gate stage. Asking the renderer would only prove it agrees with itself.
|
||||
# =====================================================================================
|
||||
sds: dict[str, dict] = {}
|
||||
for f in sorted((ROOT / "catalog").glob("*.yaml")):
|
||||
doc = yaml.safe_load(f.read_text())
|
||||
if isinstance(doc, dict) and doc.get("name"):
|
||||
sds[doc["name"]] = doc
|
||||
|
||||
if not sds:
|
||||
bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier")
|
||||
|
||||
# Every rendered app-tier document, indexed by (kind, name).
|
||||
rendered: dict[tuple, dict] = {}
|
||||
for f in sorted((ROOT / "apps").rglob("*.yaml")):
|
||||
if f.name == "kustomization.yaml" or ".sops." in f.name:
|
||||
continue
|
||||
try:
|
||||
docs = list(yaml.safe_load_all(f.read_text()))
|
||||
except yaml.YAMLError:
|
||||
continue
|
||||
for doc in docs:
|
||||
if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"):
|
||||
rendered[(doc["kind"], doc["metadata"]["name"])] = doc
|
||||
|
||||
# (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2).
|
||||
corpus: list[tuple[str, dict]] = []
|
||||
replicated_bytes: dict[str, list[int]] = {}
|
||||
for svc, sd in sds.items():
|
||||
for vol in sd.get("storage") or []:
|
||||
cls = vol.get("class", "longhorn")
|
||||
if cls == "bulk":
|
||||
corpus.append((svc, vol))
|
||||
else:
|
||||
b = qty_to_bytes(vol.get("size"))
|
||||
if b:
|
||||
replicated_bytes.setdefault(svc, []).append(b)
|
||||
|
||||
# ------------------------------------------------------------------- (9) NON-VACUOUS
|
||||
# A corpus of zero would make every loop below trivially satisfied. The bulk tier is
|
||||
# DELIVERED; a change that removes the last bulk volume must fail here and be argued for,
|
||||
# not sail through on an empty iteration.
|
||||
check(len(corpus) > 0,
|
||||
"NO `class: bulk` storage entry exists in catalog/ — this check would then pass "
|
||||
"vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME "
|
||||
"stub for six roadmap phases. If the tier is genuinely being retired, retire this "
|
||||
"check in the same change and say so; do not let it pass on an empty corpus")
|
||||
if not corpus:
|
||||
bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)")
|
||||
|
||||
inventory: list[str] = []
|
||||
|
||||
for svc, vol in corpus:
|
||||
vname = vol.get("name", "<unnamed>")
|
||||
label = f"{svc}/{vname}"
|
||||
obj = f"{svc}-{vname}"
|
||||
ro = bool(vol.get("readOnly"))
|
||||
|
||||
# `export` is what the schema requires of a bulk entry; without it there is nothing to
|
||||
# attach and the rest of the assertions have no subject.
|
||||
export = vol.get("export")
|
||||
check(bool(export) and str(export).startswith("/"),
|
||||
f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an "
|
||||
f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)")
|
||||
|
||||
pv = rendered.get(("PersistentVolume", obj))
|
||||
pvc = rendered.get(("PersistentVolumeClaim", obj))
|
||||
check(pv is not None,
|
||||
f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk "
|
||||
f"entry that renders no PV has NO external storage behind it — the pod would get "
|
||||
f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory")
|
||||
check(pvc is not None,
|
||||
f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens")
|
||||
if pv is None or pvc is None:
|
||||
continue
|
||||
|
||||
pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {}
|
||||
|
||||
# ------------------------------------------------------ (1) DISTINCT FROM REPLICATED
|
||||
for what, spec in (("PV", pvs), ("PVC", pvcs)):
|
||||
sc = spec.get("storageClassName")
|
||||
check(sc not in REPLICATED_CLASSES,
|
||||
f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares "
|
||||
f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier "
|
||||
f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not "
|
||||
f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)")
|
||||
check(sc == "",
|
||||
f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY "
|
||||
f"string. Empty is what disables dynamic provisioning AND the default-class "
|
||||
f"annotation. Any named class risks a provisioner handing out a fresh EMPTY "
|
||||
f"directory that mounts and binds cleanly while the real data is never attached")
|
||||
check("nfs" in pvs,
|
||||
f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole "
|
||||
f"premise is that durability lives OUTSIDE the cluster (docs/04)")
|
||||
|
||||
# ------------------------------------------------------------ (4) RECLAIM POLICY
|
||||
check(pvs.get("persistentVolumeReclaimPolicy") == "Retain",
|
||||
f"{label}: bulk PV reclaim policy is "
|
||||
f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists "
|
||||
f"the cluster and has consumers outside it; anything but Retain makes `kubectl "
|
||||
f"delete pvc` a data-loss operation")
|
||||
|
||||
# ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS
|
||||
check(pvcs.get("volumeName") == obj,
|
||||
f"{label}: bulk PVC does not name its PV via volumeName (got "
|
||||
f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be "
|
||||
f"satisfied by some OTHER volume of matching size and mode")
|
||||
cref = pvs.get("claimRef") or {}
|
||||
check(cref.get("name") == obj and cref.get("namespace") == svc,
|
||||
f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} "
|
||||
f"— pre-binding must be pinned from BOTH ends or another PVC can win the race")
|
||||
|
||||
# ------------------------------------------------------------- (5) MOUNT SAFETY
|
||||
mopts = [str(o) for o in (pvs.get("mountOptions") or [])]
|
||||
check(bool(mopts),
|
||||
f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS "
|
||||
f"version are whatever the client happens to default to, and both are decisions "
|
||||
f"with consequences (silent short reads; how many firewall ports must open)")
|
||||
check("hard" in mopts,
|
||||
f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount "
|
||||
f"hands a server hiccup to the application as a short read or EIO — silent "
|
||||
f"corruption for a media library. `hard` hangs instead, which is recoverable")
|
||||
check("soft" not in mopts,
|
||||
f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the "
|
||||
f"one option that turns an outage into damaged data")
|
||||
vers = [o for o in mopts if o.startswith("nfsvers=")]
|
||||
check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]),
|
||||
f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). "
|
||||
f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port "
|
||||
f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the "
|
||||
f"operator has to open in pfSense for prod to reach this server at all")
|
||||
|
||||
# ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT
|
||||
want_mode = "ReadOnlyMany" if ro else "ReadWriteMany"
|
||||
for what, spec in (("PV", pvs), ("PVC", pvcs)):
|
||||
modes = spec.get("accessModes") or []
|
||||
check(modes == [want_mode],
|
||||
f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk "
|
||||
f"volume is a `...Many` (one export, several namespaces, several nodes — "
|
||||
f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write "
|
||||
f"posture stays legible in `kubectl get pv`")
|
||||
nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly"))
|
||||
check(nfs_ro == ro,
|
||||
f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. "
|
||||
f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared "
|
||||
f"posture is not the one the kernel enforces")
|
||||
|
||||
# The volumeMount half, read out of the rendered Deployment.
|
||||
dep = rendered.get(("Deployment", svc))
|
||||
check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against")
|
||||
if dep is not None:
|
||||
containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or []
|
||||
# The app container is containers[0] by the renderer's contract (a forward-auth
|
||||
# sidecar is appended AFTER it).
|
||||
mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else []
|
||||
check(len(mounts) == 1,
|
||||
f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, "
|
||||
f"expected exactly 1 — a bulk volume that is declared and never mounted is a "
|
||||
f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)")
|
||||
if len(mounts) == 1:
|
||||
m = mounts[0]
|
||||
check(bool(m.get("readOnly")) == ro,
|
||||
f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the "
|
||||
f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, "
|
||||
f"so that dropping one still leaves the other")
|
||||
check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"),
|
||||
f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the "
|
||||
f"declared subPath {vol.get('subPath')!r}")
|
||||
|
||||
# ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV
|
||||
server = (pvs.get("nfs") or {}).get("server")
|
||||
m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or ""))
|
||||
check(m is not None,
|
||||
f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars "
|
||||
f"token, not a literal address. dev and prod reach this storage host by DIFFERENT "
|
||||
f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct "
|
||||
f"in at most one environment and silently wrong in the other (R-ENV-02)")
|
||||
|
||||
cap = (pvs.get("capacity") or {}).get("storage")
|
||||
req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage")
|
||||
check(cap is not None and qty_to_bytes(cap) is not None,
|
||||
f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, "
|
||||
f"and it is what makes `kubectl get pv` report something true about this tier")
|
||||
check(cap == req,
|
||||
f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the "
|
||||
f"capacity never binds")
|
||||
|
||||
inventory.append(
|
||||
f" {label:<28} export={export} {'ro' if ro else 'rw'} {want_mode:<14} "
|
||||
f"cap={cap} server={server} opts={','.join(mopts)}"
|
||||
)
|
||||
|
||||
# ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes
|
||||
cap_b = qty_to_bytes(cap)
|
||||
reps = replicated_bytes.get(svc) or []
|
||||
if cap_b and reps:
|
||||
biggest = max(reps)
|
||||
# ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed:
|
||||
# the media library is ~460x its service's config volume, and the Nextcloud archive
|
||||
# ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate
|
||||
# bulk volume, and this check would be teaching people to weaken it. 10x cannot be
|
||||
# met by accident and still catches the regression that matters: a "bulk" volume
|
||||
# sized like a config PVC, i.e. the strategy named but never applied.
|
||||
check(cap_b >= 10 * biggest,
|
||||
f"{label}: bulk capacity {cap} is less than 10x this service's largest "
|
||||
f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the "
|
||||
f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps "
|
||||
f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A "
|
||||
f"bulk volume sized like a config volume means the tier was named, not applied")
|
||||
|
||||
# ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT
|
||||
# The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod.
|
||||
by_service: dict[str, list[dict]] = {}
|
||||
for svc, vol in corpus:
|
||||
by_service.setdefault(svc, []).append(vol)
|
||||
for svc, vols in sorted(by_service.items()):
|
||||
seen: dict[str, str] = {}
|
||||
for vol in vols:
|
||||
exp = vol.get("export")
|
||||
prev = seen.get(exp)
|
||||
check(prev is None,
|
||||
f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and "
|
||||
f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the "
|
||||
f"kernel, so link() between them fails EXDEV even though the server knows "
|
||||
f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they "
|
||||
f"degrade to copy-then-delete, rewriting every imported file over NFS and "
|
||||
f"destroying the seeding copy. Mount the common ancestor ONCE instead "
|
||||
f"(docs/BULK-NFS-STORAGE.md §4)")
|
||||
seen[exp] = vol.get("name")
|
||||
|
||||
# ------------------------------------- (8b) the knob has a base default AND every overlay
|
||||
tokens = set()
|
||||
for _svc, vol in corpus:
|
||||
pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}"))
|
||||
if pv:
|
||||
mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or ""))
|
||||
if mm:
|
||||
tokens.add(mm.group(1))
|
||||
|
||||
cv = ROOT / "clusters" / "base" / "cluster-vars.yaml"
|
||||
base_keys: set[str] = set()
|
||||
if cv.is_file():
|
||||
doc = yaml.safe_load(cv.read_text()) or {}
|
||||
base_keys = set((doc.get("data") or {}).keys())
|
||||
|
||||
overlays = sorted(
|
||||
d for d in (ROOT / "clusters").iterdir()
|
||||
if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file()
|
||||
) if (ROOT / "clusters").is_dir() else []
|
||||
|
||||
for tok in sorted(tokens):
|
||||
check(tok in base_keys,
|
||||
f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no "
|
||||
f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without "
|
||||
f"it the token ships into the cluster as a literal server name")
|
||||
for ov in overlays:
|
||||
text = (ov / "kustomization.yaml").read_text()
|
||||
check(f"/data/{tok}" in text,
|
||||
f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV "
|
||||
f"value by this repo's convention, so an overlay that omits it silently "
|
||||
f"inherits an address on the WRONG SIDE of pfSense — a mount that either "
|
||||
f"never connects or connects to something unintended (R-ENV-02)")
|
||||
|
||||
# --------------------------------------------------------------- (9) LOAD-BEARING
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism.
|
||||
# It used to assert `"bulk-tier-distinct-from-replicated/check.sh" in L0`: ci/l0.sh carried a hand-written
|
||||
# "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone
|
||||
# (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED
|
||||
# to write it). The same property is now asserted two ways: this test DECLARES
|
||||
# `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into
|
||||
# the run via `--include-scope`. Neither half alone is enough, so both are checked here; the
|
||||
# mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than
|
||||
# passing) is proven end-to-end and falsifiably by
|
||||
# conformance/tests/l0-static/whole-repo-invariants-wired.
|
||||
SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "bulk-tier-distinct-from-replicated" / "chainsaw-test.yaml"
|
||||
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
|
||||
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None,
|
||||
"bulk-tier-distinct-from-replicated/chainsaw-test.yaml does not declare "
|
||||
'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only '
|
||||
"when a change happens to touch one of the requirements it is labelled with. "
|
||||
"The path map keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement, so moving a multi-terabyte bulk volume onto the 3-replica tier could attribute no requirement at all and be gated by nothing")
|
||||
check("--include-scope" in L0,
|
||||
"ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is "
|
||||
"a purely requirement-scoped run again and the scope annotation is inert decoration — "
|
||||
"this check would then run only by coincidence")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
|
||||
# ------------------------------------------------------------------------- report
|
||||
print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):")
|
||||
for line in inventory:
|
||||
print(line)
|
||||
print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): "
|
||||
f"{', '.join(sorted(REPLICATED_CLASSES))}")
|
||||
print(f"bulk volumes: {len(corpus)} distinct exports: "
|
||||
f"{len({v.get('export') for _s, v in corpus})} "
|
||||
f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}")
|
||||
|
||||
# A floor on the assertion count, for the same reason the corpus may not be empty: a pass
|
||||
# must not be able to degenerate into "nothing was actually asked".
|
||||
MIN_CHECKS = 40
|
||||
check(CHECKS >= MIN_CHECKS,
|
||||
f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not "
|
||||
f"evidence; the corpus or the extraction has degraded")
|
||||
|
||||
print()
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)")
|
||||
PY
|
||||
@@ -57,14 +57,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-01"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# NO `scope` ANNOTATION, DELIBERATELY — this invariant IS path-scoped, and was audited as
|
||||
# such (2026-09-12). Its entire input surface is infra/ (OpenTofu + the Ansible roles that
|
||||
# provision the dev nodes); it reads nothing else. The gap was that
|
||||
# ci/touched-requirements.sh keyed infra/* to NOTHING, so an infra/ edit attributed no
|
||||
# requirement and this check was filtered out. The right fix for a path-scoped invariant is
|
||||
# a path-map entry, not an unconditional run: `infra/*) echo "R-ENV-01"` was added there in
|
||||
# the same change. Keep the two in sync — if this check ever grows an assertion outside
|
||||
# infra/, either widen that map entry or give this test a whole-repo scope.
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -74,11 +74,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. Universally quantified over tracked YAML: every manifest that carries
|
||||
# an egress-class label/annotation must use a value the policy set actually steers. The
|
||||
# file most likely to break it is a NEW workload under apps/ — which attributes no
|
||||
# requirement — not platform/vpn-egress/ (the only path the map keys to R-NET-04).
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -140,16 +140,8 @@ def docs(path: Path):
|
||||
return []
|
||||
|
||||
|
||||
# Skip dot-DIRECTORIES (.git, .claude, .cache) while still matching dot-FILES such as
|
||||
# .sops.yaml. A bare rglob from ROOT also descends into NESTED WORKTREES —
|
||||
# .claude/worktrees/agent-*/ created for isolated agent runs — which made this check see
|
||||
# 5 CiliumEgressGatewayPolicy objects instead of the 1 authored, failing R-NET-04 for a
|
||||
# reason that has nothing to do with egress. .claude/ is gitignored (.gitignore:43), so
|
||||
# nothing under it is part of the authored surface by definition.
|
||||
TRACKED_YAML = sorted(p for p in ROOT.rglob("*.yaml")
|
||||
if "_artifacts" not in p.parts
|
||||
and not any(part.startswith(".")
|
||||
for part in p.relative_to(ROOT).parent.parts))
|
||||
if ".git" not in p.parts and "_artifacts" not in p.parts)
|
||||
SCHEMA = ROOT / "schemas" / "servicedefinition.schema.json"
|
||||
RENDER = ROOT / "catalog" / "render.py"
|
||||
VPN_ROLE = ROOT / "infra" / "ansible" / "roles" / "vpn_egress_gateway"
|
||||
|
||||
@@ -43,21 +43,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-02, R-GITOPS-01"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE — relocated verbatim from ci/l0.sh's hand-written "Stage 3c
|
||||
# (UNCONDITIONAL)", which this annotation replaces:
|
||||
#
|
||||
# R-ENV-02's promote-once mechanism is `postBuild.substituteFrom cluster-vars`, and it
|
||||
# FAILS OPEN: an undefined ${TOKEN} becomes the empty string and the Kustomization still
|
||||
# reports Ready. That emptied Cilium's k8sServiceHost and broke the CNI on 2/3 dev nodes
|
||||
# (DECISIONS-LOG 2026-09-01, "substitution race that fails OPEN").
|
||||
#
|
||||
# WHY IT MUST NOT BE REQUIREMENT-SCOPED: ci/touched-requirements.sh's coarse path map
|
||||
# keys only clusters/* to R-ENV-02 — an edit to a SUBSTITUTED manifest under apps/ or
|
||||
# platform/ attributes no requirement, so the requirement-scoped selection would filter
|
||||
# this check out of exactly the change that introduces the bug. The invariant spans every
|
||||
# path any Flux Kustomization substitutes (apps/, platform/, platform-config/), so it runs
|
||||
# on EVERY gate invocation.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -55,8 +55,7 @@
|
||||
# (5) every knob an overlay patches (`/data/KEY`) exists in the BASE ConfigMap —
|
||||
# the "add the knob with a base default BEFORE referencing it" rule that the
|
||||
# incident's interim discipline depends on;
|
||||
# AND (6) this check is LOAD-BEARING: it DECLARES `conformance.home.k3s/scope: whole-repo`,
|
||||
# so ci/l0.sh's `--include-scope whole-repo` runs it on every gate run, and
|
||||
# AND (6) this check is LOAD-BEARING: ci/l0.sh runs it on every gate run and
|
||||
# ci/gate.sh sets the merge block when the l0 stage fails.
|
||||
# Assertion (1)-(3) are applied to COMMENTS as well as content: `kustomize build`
|
||||
# preserves body comments into the stream envsubst reads (verified: a `$`-carrying
|
||||
@@ -329,27 +328,10 @@ for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
|
||||
# ------------------------------------------------------- (6) the check is load-bearing
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism.
|
||||
# It used to assert `"flux-postbuild-substitution/check.sh" in L0`: ci/l0.sh carried a hand-written
|
||||
# "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone
|
||||
# (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED
|
||||
# to write it). The same property is now asserted two ways: this test DECLARES
|
||||
# `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into
|
||||
# the run via `--include-scope`. Neither half alone is enough, so both are checked here; the
|
||||
# mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than
|
||||
# passing) is proven end-to-end and falsifiably by
|
||||
# conformance/tests/l0-static/whole-repo-invariants-wired.
|
||||
SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "flux-postbuild-substitution" / "chainsaw-test.yaml"
|
||||
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
|
||||
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None,
|
||||
"flux-postbuild-substitution/chainsaw-test.yaml does not declare "
|
||||
'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only '
|
||||
"when a change happens to touch one of the requirements it is labelled with. "
|
||||
"The path map keys only clusters/* to R-ENV-02, so an edit to a SUBSTITUTED manifest under apps/ or platform/ attributes nothing and this check would be filtered out of exactly the change that empties a variable (the 2026-09-01 Cilium k8sServiceHost CNI outage)")
|
||||
check("--include-scope" in L0,
|
||||
"ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is "
|
||||
"a purely requirement-scoped run again and the scope annotation is inert decoration — "
|
||||
"this check would then run only by coincidence")
|
||||
check("flux-postbuild-substitution/check.sh" in L0,
|
||||
"ci/l0.sh does not run flux-postbuild-substitution/check.sh — the requirement-scoped "
|
||||
"conformance selection can filter this check out of a gate run, so l0.sh must invoke "
|
||||
"it directly and unconditionally or the gate is not actually enforcing it")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
|
||||
|
||||
@@ -68,16 +68,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-PLAT-01"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. The invariant spans THREE independent subtrees — infra/ (the Ansible
|
||||
# roles + OpenTofu that create the control-plane nodes and the VIP), clusters/ (the
|
||||
# CONTROL_PLANE_VIP knob and its per-overlay values) and platform/cilium/ (the HelmRelease
|
||||
# whose k8sServiceHost IS that VIP). ci/touched-requirements.sh keys infra/* to NOTHING,
|
||||
# clusters/* to R-GITOPS-01/R-ENV-02 and platform/cilium/* to the R-NET-* family: not one
|
||||
# of them attributes R-PLAT-01, so no edit that can break HA control-plane-as-code selects
|
||||
# this test. Note this is the same VIP whose empty substitution took the CNI down on 2/3
|
||||
# dev nodes (2026-09-01) — the two halves of that incident are guarded here and in
|
||||
# flux-postbuild-substitution.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — Keycloak client secrets are declared consistently, ordered by a REAL
|
||||
# Flux edge, and healed by a reconciler whose behaviour is proven offline
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# R-IDP-03 (identity config is declarative) and R-IDP-04 (IdP secrets SOPS-encrypted)
|
||||
# are both implemented through ONE mechanism that FAILS OPEN: the KeycloakRealmImport
|
||||
# resolves each client's secret from a `(env:NAME)` placeholder mapped to a key of the
|
||||
# SOPS-encrypted keycloak-realm-clients Secret. When the operator cannot resolve a
|
||||
# placeholder it stores the LITERAL placeholder text as the client secret AND THE IMPORT
|
||||
# JOB STILL REPORTS SUCCESS — then rests at Done=True / Started=False and never re-runs.
|
||||
#
|
||||
# Measured 2026-09-06: prod had all 12 clients holding the literal placeholder (every
|
||||
# OIDC login failed with "Invalid client or Invalid client credentials"); dev had exactly
|
||||
# one (oauth2-proxy), because that Secret key was added AFTER dev's import completed.
|
||||
# Declarative config that silently does not apply is not declarative config.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN every confidential client declares a placeholder, every placeholder is defined,
|
||||
# resolves to the keycloak-realm-clients Secret, and names a key that EXISTS in the
|
||||
# SOPS file under the `<clientId>-client-secret` contract the reconciler derives
|
||||
# from; public clients declare no secret; the Flux Kustomization applying the Secret
|
||||
# is a transitive `dependsOn` ancestor (wait: true) of the ones applying the import
|
||||
# and the reconciler — a real edge, not a comment; the reconciler is reconciled from
|
||||
# BOTH cluster entrypoints, mounts only what it needs, carries no ServiceAccount
|
||||
# token and contains no dollar sign; and its own `--selftest` (extracted from the
|
||||
# shipped ConfigMap and executed) proves compare-then-write, idempotence,
|
||||
# no-secret-logging, the endpoint allow-list and fail-closed behaviour.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git + PyYAML +
|
||||
# stdlib); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw
|
||||
# Test wraps the SAME script.
|
||||
#
|
||||
# NOT proven here: that the LIVE realm's stored secrets match git (that is
|
||||
# tests/l2-behavioral/keycloak-client-secrets-match-git), that the SOPS ciphertext
|
||||
# decrypts to a usable value (key PRESENCE is offline-checkable; plaintext is not), or
|
||||
# that the CronJob's schedule fires.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: keycloak-client-secret-reconciler
|
||||
labels:
|
||||
conformance.home.k3s/R-IDP-03: "true"
|
||||
conformance.home.k3s/R-IDP-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE — relocated verbatim from ci/l0.sh's hand-written "Stage 3d
|
||||
# (UNCONDITIONAL)", which this annotation replaces:
|
||||
#
|
||||
# The Keycloak realm import resolves each OIDC client's secret from an `(env:NAME)`
|
||||
# placeholder. When the operator cannot resolve one it stores the LITERAL placeholder as
|
||||
# the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS, then rests at Done=True and
|
||||
# never re-runs. On 2026-09-06 that had all 12 prod clients unusable ("Invalid client or
|
||||
# Invalid client credentials") behind an entirely green cluster; dev had one.
|
||||
#
|
||||
# WHY IT MUST NOT BE REQUIREMENT-SCOPED: this check's invariant spans platform/,
|
||||
# platform-config/ and clusters/, so a change that breaks it need not attribute
|
||||
# R-IDP-03/04 and the requirement-scoped selection would filter it out of exactly the
|
||||
# change that introduces the bug.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every OIDC client's secret is declared once, resolvable, ordered behind a real Flux
|
||||
dependency, and continuously reconciled — so an unresolved import placeholder can
|
||||
neither be merged nor silently persist in the realm.
|
||||
spec:
|
||||
steps:
|
||||
- name: audit-declaration-ordering-and-reconciler
|
||||
description: >-
|
||||
Assert the realm import, the SOPS Secret and the reconciler agree; that the Flux
|
||||
ordering is a real dependsOn edge; and run the reconciler's own selftest.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,759 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — keycloak-client-secret-reconciler
|
||||
# Serves: R-IDP-03 ("Identity config (realms, clients, groups, roles) is declarative."
|
||||
# — spec docs/15-identity-and-access.md)
|
||||
# R-IDP-04 ("IdP secrets SOPS-encrypted; MFA available."
|
||||
# — spec docs/15-identity-and-access.md)
|
||||
#
|
||||
# WHY THIS EXISTS — THE REALM IMPORT IS A ONE-SHOT THAT FAILS OPEN
|
||||
# ---------------------------------------------------------------
|
||||
# platform/keycloak/keycloak-realm-import.yaml declares 12 confidential OIDC clients
|
||||
# whose secrets are `(env:NAME)` import placeholders, mapped by spec.placeholders to
|
||||
# keys of the SOPS-encrypted keycloak-realm-clients Secret. Measured on both clusters
|
||||
# on 2026-09-06:
|
||||
#
|
||||
# * when the Keycloak Operator cannot resolve a placeholder it stores the LITERAL
|
||||
# placeholder text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS;
|
||||
# * the KeycloakRealmImport then rests at Done=True / Started=False and NEVER
|
||||
# re-runs, so it never self-heals.
|
||||
#
|
||||
# Consequence: on prod all 12 clients held the literal placeholder and every OIDC
|
||||
# login on the cluster failed ("Invalid client or Invalid client credentials"); on dev
|
||||
# exactly one (oauth2-proxy) was broken, because that Secret key was added AFTER dev's
|
||||
# import had already completed and a completed import is never re-run.
|
||||
#
|
||||
# Nothing in the repo asserted the declaration was self-consistent, and nothing
|
||||
# corrected the realm afterwards. This check is the static half of the fix (the
|
||||
# reconciler CronJob is the runtime half, and this check proves that too).
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (1) DECLARATION IS COMPLETE: every confidential client in the realm import
|
||||
# declares a secret placeholder; every placeholder it uses is defined in
|
||||
# spec.placeholders; every placeholder resolves to the keycloak-realm-clients
|
||||
# Secret; and every referenced key EXISTS in the SOPS file. A client whose
|
||||
# key is missing is exactly the dev failure, and it is now unmergeable.
|
||||
# (2) THE NAMING CONTRACT HOLDS: each key is `<clientId>-client-secret`. That is
|
||||
# what lets the reconciler derive the client set from the mounted Secret with
|
||||
# no second declaration — so it must be enforced, not assumed. No orphan
|
||||
# placeholder, and no orphan Secret key (an orphan key would make every
|
||||
# reconciler run fail with NOT FOUND).
|
||||
# (3) PUBLIC CLIENTS DECLARE NO SECRET (the `kubernetes` client is PKCE-public;
|
||||
# giving it a secret would be a silent downgrade).
|
||||
# (4) ORDERING IS A REAL FLUX EDGE, NOT A COMMENT: the Flux Kustomization that
|
||||
# renders the Secret is a transitive `dependsOn` ancestor — with `wait: true`
|
||||
# — of the one that renders the realm import and the reconciler. The repo's
|
||||
# hard-won rule: a dependency that is only DOCUMENTED is not a dependency.
|
||||
# (5) THE RECONCILER IS WIRED AND SAFE: it is reconciled from BOTH cluster
|
||||
# entrypoints, mounts exactly the two Secrets it needs at the paths its code
|
||||
# defaults to, carries no ServiceAccount token, and contains no dollar sign
|
||||
# (it ships inside a substituted Flux path, where an unescaped dollar-brace
|
||||
# expansion is rewritten before any container runs).
|
||||
# (6) THE RECONCILER'S BEHAVIOUR IS PROVEN, NOT ASSERTED: this check EXTRACTS the
|
||||
# script from the ConfigMap and executes its `--selftest`, which drives the
|
||||
# real code paths against an in-process stub Keycloak and asserts
|
||||
# compare-then-write, idempotence (zero writes on a converged realm),
|
||||
# no-secret-logging, the endpoint allow-list (users/groups/roles are
|
||||
# unreachable) and fail-closed behaviour. What CI proves is byte-for-byte
|
||||
# what ships.
|
||||
# (6b) AND THAT PROOF IS FALSIFIABLE: mutations.py injects plausible regressions
|
||||
# into the extracted script one at a time and requires --selftest to CATCH
|
||||
# every one. A passing suite proves nothing if it cannot fail, and "reported
|
||||
# success while the realm was broken" is the exact failure this component
|
||||
# exists to end — so the same standard is applied to its own test.
|
||||
# (8) THE URL DESIRED STATE IS DERIVED, NOT DECLARED TWICE: the reconciler's
|
||||
# second pass converges the realm's environment-derived client URLs
|
||||
# (redirectUris, webOrigins, rootUrl, baseUrl, adminUrl) on
|
||||
# platform/keycloak/client-urls.yaml. That file is GENERATED from the realm
|
||||
# import by ci/render-keycloak-client-urls.py, and this check RUNS the
|
||||
# renderer with --check, so a hand edit that makes the two disagree is
|
||||
# unmergeable. It further asserts the two files are rendered by the SAME Flux
|
||||
# Kustomization (postBuild.substituteFrom is per-Kustomization, so a split
|
||||
# would let one ${BASE_DOMAIN} resolve to two different values), that the
|
||||
# desired state uses no substitution token the realm definition does not,
|
||||
# that every client it names exists in the import, and that it declares no
|
||||
# field outside the reconciler's five-field write surface.
|
||||
# AND (7) THIS CHECK IS LOAD-BEARING: it DECLARES `conformance.home.k3s/scope: whole-repo`
|
||||
# so ci/l0.sh runs it unconditionally via --include-scope, ci/gate.sh
|
||||
# blocks the merge when the l0 stage fails, ci/touched-requirements.sh maps
|
||||
# platform/keycloak/ to these requirement IDs, and an L2 test exists that
|
||||
# audits the LIVE realm. An unreferenced test is precisely the "looks applied
|
||||
# and isn't" failure this repo keeps hitting.
|
||||
#
|
||||
# NOT proven here (named honestly, in the style of the sibling L0 checks):
|
||||
# - that the LIVE realm's stored secrets OR redirect URIs match git. That is cluster
|
||||
# state; it is conformance/tests/l2-behavioral/keycloak-client-secrets-match-git,
|
||||
# which runs the reconciler's own --audit against the live realm.
|
||||
# - that the hostnames the realm import declares are the ones the Gateway/HTTPRoutes
|
||||
# actually serve. This check proves the realm converges on what GIT says; it cannot
|
||||
# prove git is right about the domain. A BASE_DOMAIN flip that misses an HTTPRoute is
|
||||
# a different invariant.
|
||||
# - that the SOPS ciphertext decrypts to a usable secret. Key PRESENCE is checkable
|
||||
# offline; the plaintext is not, and must not be.
|
||||
# - that the CronJob's image can be pulled or its schedule fires.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
IMPORT_FILE="platform/keycloak/keycloak-realm-import.yaml"
|
||||
SOPS_FILE="platform/keycloak/keycloak-realm-clients.sops.yaml"
|
||||
RECON_FILE="platform/keycloak/client-secret-reconciler.yaml"
|
||||
URLS_FILE="platform/keycloak/client-urls.yaml"
|
||||
|
||||
# Honest SKIP (never green) if the thing under test is absent entirely.
|
||||
for f in "$IMPORT_FILE" "$SOPS_FILE" "$RECON_FILE" "$URLS_FILE"; do
|
||||
if [ ! -f "${ROOT}/${f}" ]; then
|
||||
echo "check SKIPPED: ${f} absent — nothing to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
"""Static audit of the Keycloak client-secret declaration, its Flux ordering, and the
|
||||
reconciler that heals it. PyYAML (the same dependency tests/l0-static/no-plaintext-secrets
|
||||
and least-privilege-rbac already take) parses the manifests; lib/status.py is IMPORTED for
|
||||
the Flux graph walk rather than forked, so there is one definition of "what a Flux path
|
||||
renders" in the harness."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree
|
||||
sys.path.insert(0, str(ROOT / "conformance" / "lib"))
|
||||
import status as st # noqa: E402 — reuse the walk, don't fork it
|
||||
|
||||
IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
|
||||
SOPS_FILE = "platform/keycloak/keycloak-realm-clients.sops.yaml"
|
||||
RECON_FILE = "platform/keycloak/client-secret-reconciler.yaml"
|
||||
URLS_FILE = "platform/keycloak/client-urls.yaml"
|
||||
SECRET_NAME = "keycloak-realm-clients"
|
||||
ADMIN_SECRET = "keycloak-admin-credentials"
|
||||
URLS_CM_NAME = "keycloak-client-urls"
|
||||
KEY_SUFFIX = "-client-secret"
|
||||
SECRETS_MOUNT = "/etc/keycloak-realm-clients"
|
||||
URLS_MOUNT = "/etc/keycloak-client-urls"
|
||||
URLS_DATA_KEY = "client-urls.json"
|
||||
RENDERER = "ci/render-keycloak-client-urls.py"
|
||||
DIR_PATH = ROOT / "conformance" / "tests" / "l0-static" / "keycloak-client-secret-reconciler"
|
||||
# The ONLY client fields the reconciler may write besides the secret. Kept in lockstep
|
||||
# with the renderer and the script; asserted below to be exactly what they agree on.
|
||||
URL_LIST_FIELDS = ("redirectUris", "webOrigins")
|
||||
URL_STR_FIELDS = ("rootUrl", "baseUrl", "adminUrl")
|
||||
URL_FIELDS = URL_LIST_FIELDS + URL_STR_FIELDS
|
||||
DOLLAR = chr(36)
|
||||
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def load(rel):
|
||||
return list(yaml.safe_load_all((ROOT / rel).read_text()))
|
||||
|
||||
|
||||
# ================================================================= (1)-(3) declaration
|
||||
realm_import = next((d for d in load(IMPORT_FILE)
|
||||
if d and d.get("kind") == "KeycloakRealmImport"), None)
|
||||
check(realm_import is not None, f"{IMPORT_FILE}: no KeycloakRealmImport document found")
|
||||
if realm_import is None:
|
||||
print("CHECK FAIL: cannot parse the realm import", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
placeholders = realm_import["spec"].get("placeholders") or {}
|
||||
clients = realm_import["spec"]["realm"].get("clients") or []
|
||||
check(len(clients) > 0, f"{IMPORT_FILE}: parsed zero clients — the file shape changed and "
|
||||
f"this check would silently audit nothing")
|
||||
|
||||
# The operator's placeholder syntax, spelled without a literal dollar so this file is
|
||||
# itself safe to grep for one. Matches `<dollar>(env:NAME)`.
|
||||
PLACEHOLDER_RE = re.compile(re.escape(DOLLAR) + r"\(env:([A-Za-z_][A-Za-z0-9_]*)\)")
|
||||
|
||||
used_placeholders: set[str] = set()
|
||||
declared_pairs: dict[str, str] = {} # clientId -> secret key in the Secret
|
||||
|
||||
for c in clients:
|
||||
cid = c.get("clientId", "<unnamed>")
|
||||
secret = c.get("secret")
|
||||
public = bool(c.get("publicClient"))
|
||||
if public:
|
||||
# (3) a PKCE public client must not carry a secret at all.
|
||||
check(secret is None,
|
||||
f"{IMPORT_FILE}: client '{cid}' is publicClient but declares a secret — a "
|
||||
f"public client holds no secret; declaring one is a silent downgrade")
|
||||
continue
|
||||
# (1) every confidential client must declare one.
|
||||
check(isinstance(secret, str) and secret,
|
||||
f"{IMPORT_FILE}: confidential client '{cid}' declares NO secret. The operator "
|
||||
f"would generate a random one that exists nowhere in git, so nothing could "
|
||||
f"ever reconcile it and the consuming app's mirrored credential would be wrong")
|
||||
if not isinstance(secret, str):
|
||||
continue
|
||||
m = PLACEHOLDER_RE.fullmatch(secret.strip())
|
||||
check(m is not None,
|
||||
f"{IMPORT_FILE}: client '{cid}' has a secret that is not an (env:NAME) "
|
||||
f"placeholder. A literal secret in git violates R-IDP-04/R-SEC-01; the only "
|
||||
f"permitted form is a placeholder resolved from the {SECRET_NAME} Secret")
|
||||
if m is None:
|
||||
continue
|
||||
name = m.group(1)
|
||||
used_placeholders.add(name)
|
||||
# (1) the placeholder must be DEFINED. An undefined one is stored literally by the
|
||||
# operator, with a successful-looking import — the whole incident.
|
||||
ph = placeholders.get(name)
|
||||
check(ph is not None,
|
||||
f"{IMPORT_FILE}: client '{cid}' uses placeholder '{name}' but spec.placeholders "
|
||||
f"does not define it. The operator stores the LITERAL placeholder as the client "
|
||||
f"secret and STILL reports the import successful (measured 2026-09-06)")
|
||||
if ph is None:
|
||||
continue
|
||||
ref = (ph or {}).get("secret") or {}
|
||||
check(ref.get("name") == SECRET_NAME,
|
||||
f"{IMPORT_FILE}: placeholder '{name}' reads Secret '{ref.get('name')}', not "
|
||||
f"'{SECRET_NAME}'. The reconciler mounts only {SECRET_NAME}, so a second source "
|
||||
f"would be declared but never healed")
|
||||
key = ref.get("key")
|
||||
check(isinstance(key, str) and key,
|
||||
f"{IMPORT_FILE}: placeholder '{name}' names no Secret key")
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
# (2) the naming contract the reconciler derives its client set from.
|
||||
check(key == cid + KEY_SUFFIX,
|
||||
f"{IMPORT_FILE}: client '{cid}' reads Secret key '{key}', but the reconciler "
|
||||
f"derives the clientId from the key name, so it MUST be "
|
||||
f"'{cid}{KEY_SUFFIX}'. Rename the key (and the SOPS entry) or the reconciler "
|
||||
f"would look up a client that does not exist")
|
||||
declared_pairs[cid] = key
|
||||
|
||||
# (2) no orphan placeholder: a defined-but-unused placeholder is dead config that will
|
||||
# quietly rot out of sync with the Secret.
|
||||
for name in sorted(set(placeholders) - used_placeholders):
|
||||
check(False,
|
||||
f"{IMPORT_FILE}: spec.placeholders defines '{name}' but no client uses it — "
|
||||
f"dead declaration; remove it or wire it to the client that needs it")
|
||||
|
||||
# ---- the SOPS Secret: keys are cleartext in a SOPS-encrypted file; values are not.
|
||||
sops_doc = next((d for d in load(SOPS_FILE) if d and d.get("kind") == "Secret"), None)
|
||||
check(sops_doc is not None, f"{SOPS_FILE}: no Secret document found")
|
||||
sops_keys = set((sops_doc or {}).get("stringData") or {})
|
||||
check(len(sops_keys) > 0,
|
||||
f"{SOPS_FILE}: parsed zero stringData keys — the file shape changed and this check "
|
||||
f"would silently audit nothing")
|
||||
check((sops_doc or {}).get("metadata", {}).get("name") == SECRET_NAME,
|
||||
f"{SOPS_FILE}: the Secret is not named {SECRET_NAME}")
|
||||
|
||||
for cid, key in sorted(declared_pairs.items()):
|
||||
# (1) THE DEV FAILURE, made unmergeable: a client declared in the import whose key
|
||||
# is absent from the Secret imports "successfully" with a literal placeholder.
|
||||
check(key in sops_keys,
|
||||
f"{SOPS_FILE}: client '{cid}' needs key '{key}' and it is NOT in the Secret. "
|
||||
f"The realm import would still report SUCCESS and store the literal placeholder "
|
||||
f"as that client's secret — this is exactly how dev's oauth2-proxy broke")
|
||||
|
||||
# (2) no orphan Secret key: the reconciler treats every `<x>-client-secret` key as a
|
||||
# client that MUST exist, and fails loudly if it does not. Catch it here instead.
|
||||
for key in sorted(k for k in sops_keys if k.endswith(KEY_SUFFIX)):
|
||||
cid = key[: -len(KEY_SUFFIX)]
|
||||
check(cid in declared_pairs,
|
||||
f"{SOPS_FILE}: key '{key}' has no matching client in {IMPORT_FILE}. The "
|
||||
f"reconciler would look up client '{cid}', not find it, and fail every run")
|
||||
|
||||
# ===================================================== (4) ordering is a real Flux edge
|
||||
# Parse contract mirrors lib/status.py's flux_paths(): docs split on a lone `---`; a doc
|
||||
# is an entrypoint only if it carries the kustomize.toolkit apiVersion, `kind:
|
||||
# Kustomization` and a spec-level `path:`. Overlay `patch: |-` bodies carry Kustomizations
|
||||
# WITHOUT a path and are correctly skipped (they are patches, not entrypoints).
|
||||
_DEPENDSON_RE = re.compile(r"^(\s*)dependsOn:\s*$", re.M)
|
||||
_DEP_NAME_RE = re.compile(r"^\s*(?:-\s*)?name:\s*([^\s#]+)")
|
||||
_WAIT_RE = re.compile(r"^\s{2,6}wait:\s*true\s*(?:#.*)?$", re.M)
|
||||
|
||||
|
||||
def depends_on(doc: str) -> list[str]:
|
||||
m = _DEPENDSON_RE.search(doc)
|
||||
if not m:
|
||||
return []
|
||||
indent = len(m.group(1))
|
||||
out: list[str] = []
|
||||
for raw in doc[m.end():].splitlines():
|
||||
if raw.strip() == "" or raw.lstrip().startswith("#"):
|
||||
continue
|
||||
if len(raw) - len(raw.lstrip()) <= indent and not raw.lstrip().startswith("-"):
|
||||
break # dedented out of the list
|
||||
nm = _DEP_NAME_RE.match(raw)
|
||||
if nm:
|
||||
out.append(nm.group(1))
|
||||
return out
|
||||
|
||||
|
||||
kustomizations: dict[str, dict] = {}
|
||||
for yfile in sorted((ROOT / "clusters").rglob("*.yaml")):
|
||||
text = yfile.read_text()
|
||||
if st.FLUX_API not in text:
|
||||
continue
|
||||
for doc in st._DOC_SPLIT_RE.split(text):
|
||||
if st.FLUX_API not in doc or not re.search(r"^kind:\s*Kustomization\s*$", doc, re.M):
|
||||
continue
|
||||
pm = st._PATH_RE.search(doc)
|
||||
if not pm:
|
||||
continue
|
||||
nm = st._NAME_RE.search(doc)
|
||||
name = nm.group(1) if nm else "?"
|
||||
kustomizations[name] = {
|
||||
"file": str(yfile.relative_to(ROOT)),
|
||||
"path": pm.group(1),
|
||||
"dependsOn": depends_on(doc),
|
||||
"wait": bool(_WAIT_RE.search(doc)),
|
||||
}
|
||||
|
||||
check(len(kustomizations) > 0,
|
||||
"no Flux Kustomization with a spec.path found under clusters/ — the parser or the "
|
||||
"repo layout changed; this check would silently audit nothing")
|
||||
|
||||
|
||||
def direct_render(path_spec: str) -> set[str]:
|
||||
"""Files a Flux `spec.path` renders ITSELF — a kustomize walk that deliberately does
|
||||
NOT expand nested Flux Kustomizations.
|
||||
|
||||
lib/status.py's reconciled_files() follows every `spec.path` it discovers, which is
|
||||
right for "is this file applied to the cluster at all" but wrong for OWNERSHIP: the
|
||||
root Kustomization's path is clusters/<env>, so a transitive walk makes the root the
|
||||
owner of every file in the repo and the ordering assertion below becomes vacuous
|
||||
(measured: it reported the root as an owner of all three keycloak files). `resources:`
|
||||
parsing itself is st.kustomize_resources, not forked.
|
||||
"""
|
||||
rel = path_spec.lstrip("./") or "."
|
||||
out: set[str] = set()
|
||||
seen: set[str] = set()
|
||||
|
||||
def walk(d: str) -> None:
|
||||
if d in seen:
|
||||
return
|
||||
seen.add(d)
|
||||
kfile = ROOT / d / "kustomization.yaml"
|
||||
if not kfile.is_file():
|
||||
return
|
||||
out.add(str(kfile.relative_to(ROOT)))
|
||||
for item in st.kustomize_resources(kfile):
|
||||
if "://" in item:
|
||||
continue # remote base: not repo state
|
||||
target = (ROOT / d / item).resolve()
|
||||
try:
|
||||
trel = target.relative_to(ROOT).as_posix()
|
||||
except ValueError:
|
||||
continue # outside the repo
|
||||
if target.is_dir():
|
||||
walk(trel)
|
||||
elif target.is_file():
|
||||
out.add(trel)
|
||||
|
||||
if (ROOT / rel).is_dir():
|
||||
walk(rel)
|
||||
return out
|
||||
|
||||
|
||||
rendered_by = {n: direct_render(k["path"]) for n, k in kustomizations.items()}
|
||||
|
||||
|
||||
def owners(rel_file: str) -> list[str]:
|
||||
return sorted(n for n, files in rendered_by.items() if rel_file in files)
|
||||
|
||||
|
||||
def ancestors(name: str, seen=None) -> set[str]:
|
||||
"""Transitive dependsOn closure of `name` (its dependencies, direct and indirect)."""
|
||||
seen = seen if seen is not None else set()
|
||||
for dep in kustomizations.get(name, {}).get("dependsOn", []):
|
||||
if dep in seen:
|
||||
continue
|
||||
seen.add(dep)
|
||||
ancestors(dep, seen)
|
||||
return seen
|
||||
|
||||
|
||||
secret_owners = owners(SOPS_FILE)
|
||||
import_owners = owners(IMPORT_FILE)
|
||||
recon_owners = owners(RECON_FILE)
|
||||
|
||||
check(bool(secret_owners),
|
||||
f"{SOPS_FILE} is rendered by NO Flux Kustomization — the client secrets are in git "
|
||||
f"but never applied")
|
||||
check(bool(import_owners),
|
||||
f"{IMPORT_FILE} is rendered by NO Flux Kustomization")
|
||||
# (5) the reconciler must actually be reconciled, or it is decoration.
|
||||
check(bool(recon_owners),
|
||||
f"{RECON_FILE} is rendered by NO Flux Kustomization — the healer would sit in git "
|
||||
f"and never run, which is the 'looks applied and isn't' failure it exists to end")
|
||||
|
||||
for consumer_file, consumer_owners in ((IMPORT_FILE, import_owners),
|
||||
(RECON_FILE, recon_owners)):
|
||||
for co in consumer_owners:
|
||||
for so in secret_owners:
|
||||
if so == co:
|
||||
continue # same Kustomization: one atomic apply, ordering moot
|
||||
anc = ancestors(co)
|
||||
check(so in anc,
|
||||
f"ORDERING IS ONLY DOCUMENTED: '{co}' renders {consumer_file} but does "
|
||||
f"not (transitively) dependsOn '{so}', which renders the "
|
||||
f"{SECRET_NAME} Secret. Without that edge the consumer can reconcile "
|
||||
f"before the secret exists, and the Keycloak Operator's response to a "
|
||||
f"missing placeholder is to store it LITERALLY and report success")
|
||||
if so in anc:
|
||||
check(kustomizations[so]["wait"],
|
||||
f"'{so}' (which renders the {SECRET_NAME} Secret) does not set "
|
||||
f"wait: true, so '{co}' can proceed before it is applied")
|
||||
|
||||
# ------ both environments must reconcile the reconciler, not just dev.
|
||||
for env in ("dev", "prod"):
|
||||
entry = f"clusters/{env}"
|
||||
if not (ROOT / entry).is_dir():
|
||||
continue
|
||||
applied, _ = st.reconciled_files(ROOT, entry)
|
||||
check(RECON_FILE in applied,
|
||||
f"{RECON_FILE} is not reachable from the {entry} entrypoint — the healer would "
|
||||
f"exist on one cluster only, and prod is where all 12 clients were broken")
|
||||
check(IMPORT_FILE in applied, f"{IMPORT_FILE} is not reachable from {entry}")
|
||||
|
||||
# ============================================ (5) the reconciler manifest is safe + wired
|
||||
recon_docs = [d for d in load(RECON_FILE) if d]
|
||||
cm = next((d for d in recon_docs if d.get("kind") == "ConfigMap"), None)
|
||||
cj = next((d for d in recon_docs if d.get("kind") == "CronJob"), None)
|
||||
check(cm is not None, f"{RECON_FILE}: no ConfigMap carrying the reconciler script")
|
||||
check(cj is not None, f"{RECON_FILE}: no CronJob running the reconciler")
|
||||
|
||||
script = ""
|
||||
if cm is not None:
|
||||
data = cm.get("data") or {}
|
||||
check(len(data) == 1,
|
||||
f"{RECON_FILE}: the reconciler ConfigMap must carry exactly one script key "
|
||||
f"(found {sorted(data)}) — a second copy would be untested by this gate")
|
||||
script = next(iter(data.values()), "")
|
||||
check(len(script.splitlines()) > 50,
|
||||
f"{RECON_FILE}: the extracted script is implausibly short "
|
||||
f"({len(script.splitlines())} lines) — the ConfigMap shape changed and the "
|
||||
f"selftest below would prove nothing")
|
||||
# THE nextcloud TRAP. This file is rendered by a Flux Kustomization with
|
||||
# postBuild.substituteFrom, so envsubst rewrites it before any container runs. The
|
||||
# script is written with NO dollar sign at all, so the hazard is structural rather
|
||||
# than escaped and no future edit has to reason about escaping.
|
||||
check(DOLLAR not in script,
|
||||
f"{RECON_FILE}: the reconciler script contains a dollar sign. It ships inside a "
|
||||
f"SUBSTITUTED Flux path, where a dollar-brace expansion is rewritten at "
|
||||
f"reconcile time (see apps/wave-1/nextcloud.yaml). Rewrite without it — Python "
|
||||
f"never needs one; chr(36) spells it where a literal is genuinely required")
|
||||
|
||||
if cj is not None:
|
||||
pod = cj["spec"]["jobTemplate"]["spec"]["template"]["spec"]
|
||||
ctr = pod["containers"][0]
|
||||
vols = {v["name"]: v for v in pod.get("volumes", [])}
|
||||
mounts = {m["name"]: m for m in ctr.get("volumeMounts", [])}
|
||||
secret_vols = [v for v in vols.values()
|
||||
if (v.get("secret") or {}).get("secretName") == SECRET_NAME]
|
||||
check(len(secret_vols) == 1,
|
||||
f"{RECON_FILE}: the CronJob must mount the {SECRET_NAME} Secret — that mount IS "
|
||||
f"the desired state it reconciles towards")
|
||||
if secret_vols:
|
||||
mp = mounts.get(secret_vols[0]["name"], {}).get("mountPath")
|
||||
check(mp == SECRETS_MOUNT,
|
||||
f"{RECON_FILE}: {SECRET_NAME} is mounted at {mp}, but the script defaults "
|
||||
f"to {SECRETS_MOUNT} and no env var overrides it")
|
||||
cm_vols = [v for v in vols.values()
|
||||
if (v.get("configMap") or {}).get("name") == (cm or {}).get(
|
||||
"metadata", {}).get("name")]
|
||||
check(len(cm_vols) == 1,
|
||||
f"{RECON_FILE}: the CronJob does not mount the reconciler ConfigMap, so it "
|
||||
f"would run nothing")
|
||||
# The URL desired state must be MOUNTED at the path the script defaults to, for the
|
||||
# same reason as the Secret: the pod holds no ServiceAccount token and cannot read
|
||||
# the KeycloakRealmImport CR, so an unmounted ConfigMap means pass 2 fails closed on
|
||||
# every run forever -- correct behaviour, but a permanently red reconciler.
|
||||
url_vols = [v for v in vols.values()
|
||||
if (v.get("configMap") or {}).get("name") == URLS_CM_NAME]
|
||||
check(len(url_vols) == 1,
|
||||
f"{RECON_FILE}: the CronJob must mount the {URLS_CM_NAME} ConfigMap — that "
|
||||
f"mount IS the desired state the URL pass reconciles towards, and the pod has "
|
||||
f"no Kubernetes API access to read the realm import instead")
|
||||
if url_vols:
|
||||
mp = mounts.get(url_vols[0]["name"], {}).get("mountPath")
|
||||
check(mp == URLS_MOUNT,
|
||||
f"{RECON_FILE}: {URLS_CM_NAME} is mounted at {mp}, but the script defaults "
|
||||
f"to {URLS_MOUNT}/{URLS_DATA_KEY} and no env var overrides it")
|
||||
check(mounts.get(url_vols[0]["name"], {}).get("readOnly") is True,
|
||||
f"{RECON_FILE}: the {URLS_CM_NAME} mount must be readOnly — the reconciler "
|
||||
f"reads its desired state, it never writes it")
|
||||
env_srcs = {e["name"]: e for e in ctr.get("env", [])}
|
||||
for var in ("KEYCLOAK_ADMIN_USER", "KEYCLOAK_ADMIN_PASSWORD"):
|
||||
ref = (env_srcs.get(var, {}).get("valueFrom") or {}).get("secretKeyRef") or {}
|
||||
check(ref.get("name") == ADMIN_SECRET,
|
||||
f"{RECON_FILE}: {var} must come from the {ADMIN_SECRET} Secret "
|
||||
f"(got {ref.get('name')!r}) — the admin credential is never inlined")
|
||||
for var in ("KEYCLOAK_BASE_URL", "KEYCLOAK_REALM"):
|
||||
check(env_srcs.get(var, {}).get("value"),
|
||||
f"{RECON_FILE}: {var} is not set; the script fails closed without it, but "
|
||||
f"the CronJob should not ship broken")
|
||||
# R-SEC-02: it talks only to Keycloak's REST API and needs no Kubernetes API rights.
|
||||
check(pod.get("automountServiceAccountToken") is False,
|
||||
f"{RECON_FILE}: the reconciler needs NO Kubernetes API access (its inputs are "
|
||||
f"mounted Secrets) — set automountServiceAccountToken: false")
|
||||
check(cj["spec"].get("concurrencyPolicy") == "Forbid",
|
||||
f"{RECON_FILE}: set concurrencyPolicy: Forbid so a run stalled on an "
|
||||
f"unavailable Keycloak is not lapped by the next tick")
|
||||
img = ctr.get("image", "")
|
||||
check(re.search(r":[0-9]", img),
|
||||
f"{RECON_FILE}: container image {img!r} is not pinned to an explicit version")
|
||||
|
||||
# ================================================ (8) the URL desired state is DERIVED
|
||||
# The reconciler's second pass makes the realm's environment-derived client URLs match
|
||||
# platform/keycloak/client-urls.yaml. That file must not be an independent declaration of
|
||||
# every public hostname: two declarations that can disagree is the same shape as the bug
|
||||
# this whole change closes. It is GENERATED from the realm import, and the generation is
|
||||
# gated HERE -- a hand edit to either file that makes them disagree is unmergeable.
|
||||
urls_docs = [d for d in load(URLS_FILE) if d]
|
||||
urls_cm = next((d for d in urls_docs if d.get("kind") == "ConfigMap"), None)
|
||||
check(urls_cm is not None, f"{URLS_FILE}: no ConfigMap document found")
|
||||
check((urls_cm or {}).get("metadata", {}).get("name") == URLS_CM_NAME,
|
||||
f"{URLS_FILE}: the ConfigMap must be named {URLS_CM_NAME} (the CronJob mounts it "
|
||||
f"by that name)")
|
||||
|
||||
urls_want: dict[str, dict] = {}
|
||||
if urls_cm is not None:
|
||||
data = urls_cm.get("data") or {}
|
||||
check(set(data) == {URLS_DATA_KEY},
|
||||
f"{URLS_FILE}: the ConfigMap must carry exactly the key {URLS_DATA_KEY} "
|
||||
f"(found {sorted(data)}) — the script reads that filename from its mount")
|
||||
raw = data.get(URLS_DATA_KEY, "")
|
||||
try:
|
||||
urls_want = json.loads(raw)
|
||||
except ValueError as e:
|
||||
check(False, f"{URLS_FILE}: {URLS_DATA_KEY} is not valid JSON ({e}) — the "
|
||||
f"reconciler would fail closed on every run")
|
||||
check(isinstance(urls_want, dict) and bool(urls_want),
|
||||
f"{URLS_FILE}: {URLS_DATA_KEY} must be a non-empty JSON object of clients")
|
||||
|
||||
# THE GATE: the committed file must be exactly what the renderer produces from the realm
|
||||
# import. This is what makes the realm import the single source of truth for hostnames
|
||||
# rather than merely the place they were first typed.
|
||||
if (ROOT / RENDERER).is_file():
|
||||
proc = subprocess.run([sys.executable, str(ROOT / RENDERER), "--check"],
|
||||
capture_output=True, text=True, timeout=120, cwd=str(ROOT))
|
||||
check(proc.returncode == 0,
|
||||
f"{URLS_FILE} is NOT what {RENDERER} renders from {IMPORT_FILE}. The URL "
|
||||
f"reconciler would converge the live realm on a desired state the realm import "
|
||||
f"does not declare — two sources of truth for every public hostname, which is "
|
||||
f"the failure class this change exists to close. Regenerate with: "
|
||||
f"python3 {RENDERER} --write\n "
|
||||
+ "\n ".join((proc.stdout + proc.stderr).strip().splitlines()[:12]))
|
||||
else:
|
||||
check(False, f"{RENDERER} is missing — {URLS_FILE} would become a hand-maintained "
|
||||
f"second declaration of every hostname with nothing keeping it honest")
|
||||
|
||||
# Every client named in the desired state must exist in the realm import (otherwise every
|
||||
# reconciler run reports NOT FOUND forever), and every field must be one the reconciler is
|
||||
# actually allowed to write.
|
||||
import_client_ids = {c.get("clientId") for c in clients}
|
||||
for cid in sorted(urls_want):
|
||||
check(cid in import_client_ids,
|
||||
f"{URLS_FILE}: declares URLs for client '{cid}', which {IMPORT_FILE} does not "
|
||||
f"declare. The reconciler would look it up, not find it, and fail every run")
|
||||
entry = urls_want[cid] if isinstance(urls_want.get(cid), dict) else {}
|
||||
check(bool(entry), f"{URLS_FILE}: client '{cid}' declares no URL fields")
|
||||
for field in sorted(entry):
|
||||
check(field in URL_FIELDS,
|
||||
f"{URLS_FILE}: client '{cid}' declares field '{field}', which is not one of "
|
||||
f"the writable URL fields ({', '.join(URL_FIELDS)}). Anything else would be "
|
||||
f"a client-representation field written into the realm by a file edit")
|
||||
|
||||
# The three places that name the writable field set must agree, or the gate is checking a
|
||||
# different contract than the one that ships.
|
||||
for field in URL_FIELDS:
|
||||
check(field in script,
|
||||
f"{RECON_FILE}: the shipped script never mentions the writable URL field "
|
||||
f"'{field}', but this check and {RENDERER} both treat it as writable — the "
|
||||
f"three definitions of the write surface have diverged")
|
||||
|
||||
# A redirect URI is only as good as the hostname it points at. If the realm import uses a
|
||||
# Flux substitution token, the desired-state ConfigMap must carry the SAME token (not a
|
||||
# resolved value) and must be rendered by the SAME Kustomization, or the two are
|
||||
# substituted differently and disagree about the domain at reconcile time.
|
||||
# Scan the DATA PAYLOAD, not the whole file: both files carry header prose that discusses
|
||||
# substitution tokens by name, and matching those would be a false positive.
|
||||
TOKEN_RE = re.compile(re.escape(DOLLAR) + r"\{([A-Z_][A-Z0-9_]*)\}")
|
||||
import_tokens = set(TOKEN_RE.findall(
|
||||
yaml.safe_dump((realm_import or {}).get("spec", {}).get("realm", {}))))
|
||||
urls_tokens = set(TOKEN_RE.findall(
|
||||
((urls_cm or {}).get("data") or {}).get(URLS_DATA_KEY, "")))
|
||||
check(urls_tokens <= import_tokens,
|
||||
f"{URLS_FILE} uses substitution token(s) {sorted(urls_tokens - import_tokens)} that "
|
||||
f"the realm definition in {IMPORT_FILE} does not — the desired state would resolve "
|
||||
f"to a hostname the realm import never declared")
|
||||
|
||||
urls_owners = owners(URLS_FILE)
|
||||
check(bool(urls_owners),
|
||||
f"{URLS_FILE} is rendered by NO Flux Kustomization — the reconciler would mount a "
|
||||
f"ConfigMap that does not exist and fail closed on every run")
|
||||
check(set(urls_owners) == set(import_owners),
|
||||
f"{URLS_FILE} is rendered by {urls_owners} but {IMPORT_FILE} by {import_owners}. "
|
||||
f"They MUST be rendered by the same Flux Kustomization: postBuild.substituteFrom is "
|
||||
f"per-Kustomization, so a split would let the same "
|
||||
f"{DOLLAR}{{BASE_DOMAIN}} resolve to two different values — the realm converging on "
|
||||
f"a hostname the HTTPRoutes do not serve")
|
||||
for env in ("dev", "prod"):
|
||||
entry = f"clusters/{env}"
|
||||
if not (ROOT / entry).is_dir():
|
||||
continue
|
||||
applied, _ = st.reconciled_files(ROOT, entry)
|
||||
check(URLS_FILE in applied,
|
||||
f"{URLS_FILE} is not reachable from the {entry} entrypoint — the URL pass would "
|
||||
f"fail closed on that cluster forever")
|
||||
|
||||
# ============================== (6) the reconciler's BEHAVIOUR, proven by its own selftest
|
||||
if script:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "reconcile.py"
|
||||
p.write_text(script)
|
||||
proc = subprocess.run([sys.executable, str(p), "--selftest"],
|
||||
capture_output=True, text=True, timeout=180)
|
||||
tail = (proc.stdout + proc.stderr).strip().splitlines()
|
||||
check(proc.returncode == 0,
|
||||
"the reconciler's --selftest FAILED (compare-then-write / idempotence / "
|
||||
"no-secret-logging / endpoint allow-list / fail-closed):\n "
|
||||
+ "\n ".join(tail[-12:]))
|
||||
if proc.returncode == 0:
|
||||
print(" reconciler --selftest: " + (tail[-1] if tail else "(no output)"))
|
||||
|
||||
# ---- (6b) THE SELFTEST IS FALSIFIABLE, not decorative -----------------------
|
||||
# A passing suite proves nothing unless it can be shown to FAIL. mutations.py
|
||||
# injects plausible regressions one at a time -- order-sensitive comparison, a
|
||||
# fat round-tripped PUT, additive-only URI sets, a widened allow-list, coupled
|
||||
# passes, fail-open on an unreadable input -- and requires --selftest to catch
|
||||
# every one. A mutation that SURVIVES means the suite does not check that
|
||||
# property, and this check fails naming it.
|
||||
#
|
||||
# This matters more than usual here: the whole incident was a component that
|
||||
# REPORTED SUCCESS while the realm was broken. A green test that cannot fail is
|
||||
# the same lie one level up. ~20s, no network, no cluster.
|
||||
mut = DIR_PATH / "mutations.py"
|
||||
if mut.is_file():
|
||||
mproc = subprocess.run([sys.executable, str(mut), str(p)],
|
||||
capture_output=True, text=True, timeout=600)
|
||||
mtail = (mproc.stdout + mproc.stderr).strip().splitlines()
|
||||
check(mproc.returncode == 0,
|
||||
"the reconciler's --selftest is NOT FALSIFIABLE — at least one injected "
|
||||
"regression survived it, so the suite does not prove what it claims:\n"
|
||||
" " + "\n ".join(mtail[-14:]))
|
||||
if mproc.returncode == 0:
|
||||
print(" reconciler mutations: "
|
||||
+ (mtail[-1] if mtail else "(no output)"))
|
||||
else:
|
||||
check(False,
|
||||
"conformance/tests/l0-static/keycloak-client-secret-reconciler/"
|
||||
"mutations.py is missing — nothing proves the --selftest can fail, and "
|
||||
"an unfalsifiable proof is the failure mode this component exists to end")
|
||||
|
||||
# ======================================================= (7) this check is load-bearing
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
TOUCHED = ((ROOT / "ci" / "touched-requirements.sh").read_text()
|
||||
if (ROOT / "ci" / "touched-requirements.sh").is_file() else "")
|
||||
# THIS CHECK IS LOAD-BEARING — asserted against the DECLARATIVE mechanism.
|
||||
# It used to assert `"keycloak-client-secret-reconciler/check.sh" in L0`: ci/l0.sh carried a hand-written
|
||||
# "UNCONDITIONAL" stage per whole-repo check, hardcoding this path. Those stages are gone
|
||||
# (they did not scale, and the next whole-repo invariant only got one if someone REMEMBERED
|
||||
# to write it). The same property is now asserted two ways: this test DECLARES
|
||||
# `conformance.home.k3s/scope: "whole-repo"`, and ci/l0.sh unions every declaring test into
|
||||
# the run via `--include-scope`. Neither half alone is enough, so both are checked here; the
|
||||
# mechanism ITSELF (the union really selects, and a scope-declared SKIP exits 3 rather than
|
||||
# passing) is proven end-to-end and falsifiably by
|
||||
# conformance/tests/l0-static/whole-repo-invariants-wired.
|
||||
SELF_YAML = ROOT / "conformance" / "tests" / "l0-static" / "keycloak-client-secret-reconciler" / "chainsaw-test.yaml"
|
||||
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
|
||||
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M) is not None,
|
||||
"keycloak-client-secret-reconciler/chainsaw-test.yaml does not declare "
|
||||
'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check only '
|
||||
"when a change happens to touch one of the requirements it is labelled with. "
|
||||
"This invariant spans platform/, platform-config/ and clusters/, so a change that breaks it need not attribute R-IDP-03/04 — which is how 12 unusable prod OIDC clients sat behind an entirely green cluster on 2026-09-06")
|
||||
check("--include-scope" in L0,
|
||||
"ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is "
|
||||
"a purely requirement-scoped run again and the scope annotation is inert decoration — "
|
||||
"this check would then run only by coincidence")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
check(re.search(r"^\s*platform/keycloak/\*\)", TOUCHED, re.M),
|
||||
"ci/touched-requirements.sh has no platform/keycloak/* entry in its path->requirement "
|
||||
"map, so editing the realm import or this reconciler attributes NO requirement and "
|
||||
"the L2 keycloak tests are filtered out of the merge gate's touched subset")
|
||||
l2 = ROOT / "conformance/tests/l2-behavioral/keycloak-client-secrets-match-git/chainsaw-test.yaml"
|
||||
check(l2.is_file(),
|
||||
"the LIVE-realm counterpart (conformance/tests/l2-behavioral/"
|
||||
"keycloak-client-secrets-match-git) is missing — this static check cannot see a "
|
||||
"realm that drifted after apply, which is the whole failure mode")
|
||||
if l2.is_file():
|
||||
l2_text = l2.read_text()
|
||||
check("KEYCLOAK_CLIENT_SECRETS_AUDIT_OK" in l2_text,
|
||||
"the L2 test does not assert the reconciler's secret audit sentinel, so it could "
|
||||
"pass on a Job that never ran the audit")
|
||||
check("KEYCLOAK_CLIENT_URLS_AUDIT_OK" in l2_text,
|
||||
"the L2 test does not assert the reconciler's URL audit sentinel. The live realm "
|
||||
"could hold pre-migration redirect URIs and the test would still pass — which is "
|
||||
"exactly the state prod was measured in")
|
||||
check(URLS_CM_NAME in l2_text,
|
||||
f"the L2 audit Job does not mount the {URLS_CM_NAME} ConfigMap, so its URL pass "
|
||||
f"would fail closed and the test would report a false failure")
|
||||
|
||||
# The alert pair must cover the reconciler's silence. Extended, not duplicated: one CronJob
|
||||
# gets one Failing/Stale pair, because two pairs keyed to the same job_name regex would
|
||||
# both fire for a single cause.
|
||||
ALERTS = "platform/monitoring/alerts.yaml"
|
||||
alerts_text = (ROOT / ALERTS).read_text() if (ROOT / ALERTS).is_file() else ""
|
||||
for rule in ("KeycloakClientSecretReconcilerFailing", "KeycloakClientSecretReconcilerStale"):
|
||||
check(rule in alerts_text,
|
||||
f"{ALERTS} has no {rule} rule — the reconciler's silence would be unobservable, "
|
||||
f"and a healer nobody notices is absent is the 'looks applied and isn't' failure")
|
||||
check("redirect" in alerts_text.lower(),
|
||||
f"{ALERTS}: the reconciler alert descriptions never mention redirect URIs, so an "
|
||||
f"operator paged at 3am would look only for a client-secret problem while the actual "
|
||||
f"drift was a hostname")
|
||||
|
||||
# The renderer must be a real, runnable, tracked file — not a documented intention.
|
||||
check((ROOT / RENDERER).is_file(),
|
||||
f"{RENDERER} is missing; without it {URLS_FILE} cannot be regenerated and the "
|
||||
f"single-source-of-truth gate above has nothing to compare against")
|
||||
|
||||
# ------------------------------------------------------------------------------ report
|
||||
print(f"realm import: {len(clients)} client(s), {len(declared_pairs)} confidential with a "
|
||||
f"declared secret, {len(placeholders)} placeholder(s)")
|
||||
print(f"{SECRET_NAME} Secret: {len(sops_keys)} key(s) in git (names only; values stay "
|
||||
f"SOPS-encrypted)")
|
||||
print("flux ordering: " + ", ".join(
|
||||
f"{f.split('/')[-1]} <- {owners(f)}" for f in (SOPS_FILE, IMPORT_FILE, RECON_FILE)))
|
||||
|
||||
print()
|
||||
if not FAILURES:
|
||||
print(f"check OK: keycloak-client-secret-reconciler — {CHECKS} assertions "
|
||||
f"(R-IDP-03, R-IDP-04)")
|
||||
raise SystemExit(0)
|
||||
print(f"CHECK FAIL (keycloak-client-secret-reconciler): {len(FAILURES)} failed, "
|
||||
f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" FAIL {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
@@ -1,190 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# =============================================================================
|
||||
# mutations.py -- prove the reconciler's --selftest IS FALSIFIABLE.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# ---------------
|
||||
# A selftest that passes proves nothing on its own: a selftest asserting `True ==
|
||||
# True` also passes. The reconciler exists because a one-shot import REPORTED
|
||||
# SUCCESS while the realm was broken, and the same trap applies one level up -- a
|
||||
# green test suite that cannot fail is the same lie in a different place.
|
||||
#
|
||||
# So this injects a series of PLAUSIBLE regressions into the shipped script, one at
|
||||
# a time, and requires that --selftest FAILS for every one. A mutation that
|
||||
# SURVIVES means the suite does not actually check that property, and the check
|
||||
# fails, naming it.
|
||||
#
|
||||
# Each mutation is a mistake a reasonable person could make while editing this
|
||||
# code -- not a nonsense edit. Several were live design questions while it was
|
||||
# written (order-sensitivity, additive-vs-exact URI sets, whether the two passes
|
||||
# should share a failure path); the mutation is the record that the answer is
|
||||
# enforced and not merely commented.
|
||||
#
|
||||
# Invoked by check.sh with the script EXTRACTED from the shipped ConfigMap, so what
|
||||
# is proven falsifiable is byte-for-byte what runs on the cluster.
|
||||
#
|
||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||
#
|
||||
# Runtime is ~20s (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster).
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def dedent4(text: str) -> str:
|
||||
"""Anchors are written with the YAML file's indentation. The script extracted from
|
||||
the ConfigMap block scalar has 4 fewer leading spaces on every line, so anchors are
|
||||
authored to match the YAML (easy to diff against the manifest) and dedented here."""
|
||||
return "\n".join(ln[4:] if ln.startswith(" ") else ln for ln in text.split("\n"))
|
||||
|
||||
|
||||
# (name, the mistake being modelled, anchor, replacement)
|
||||
MUTATIONS: list[tuple[str, str, str, str]] = [
|
||||
("order-sensitive URL comparison",
|
||||
"compare redirectUris in the order Keycloak returns them -- Keycloak does not "
|
||||
"preserve the authored order, so a converged realm is rewritten on every run "
|
||||
"forever while every run still reports success",
|
||||
"if sorted(stored) != sorted(declared_value):",
|
||||
"if stored != declared_value:"),
|
||||
|
||||
("fat round-tripped URL PUT",
|
||||
"PUT the whole client representation back instead of only the drifted fields -- "
|
||||
"the live server rejects it with 400 'Cannot parse the JSON', and even if it did "
|
||||
"not, the blast radius would cover every field of the client",
|
||||
" rep = {\"clientId\": client.get(\"clientId\")}\n"
|
||||
" for field in sorted(drift):\n"
|
||||
" rep[field] = want[cid][field]",
|
||||
" rep = dict(client)\n"
|
||||
" for field in sorted(drift):\n"
|
||||
" rep[field] = want[cid][field]"),
|
||||
|
||||
("write unconditionally (compare-then-write removed)",
|
||||
"drop the converged short-circuit -- every client is rewritten on every run, so "
|
||||
"the CronJob becomes a permanent write load against the IdP database",
|
||||
" drift = url_drift(client, want[cid])\n if not drift:",
|
||||
" drift = url_drift(client, want[cid]) or dict(\n"
|
||||
" (f, (\"\", want[cid][f])) for f in want[cid])\n if False:"),
|
||||
|
||||
("additive-only URI reconciliation",
|
||||
"union the declared URIs with the stored ones -- a redirect URI REMOVED from git "
|
||||
"is never removed from the realm, so a stale wildcard keeps working and quietly "
|
||||
"widens what the realm will redirect to",
|
||||
" rep[field] = want[cid][field]",
|
||||
" rep[field] = (sorted(set(want[cid][field]) | set(client.get(field) or []))\n"
|
||||
" if field in URL_LIST_FIELDS else want[cid][field])"),
|
||||
|
||||
("URL pass fails OPEN on an unreadable input",
|
||||
"treat a missing ConfigMap as 'nothing declared' -- the run reports success while "
|
||||
"reconciling nothing, which is precisely the fail-open of the realm import",
|
||||
" except OSError as e:\n"
|
||||
" url_input_fail = (\"cannot read %s (%s) -- the keycloak-client-urls ConfigMap \"\n"
|
||||
" \"is not mounted\" % (URLS_FILE, e.__class__.__name__))",
|
||||
" except OSError:\n want_urls = {}"),
|
||||
|
||||
("URL pass skips public clients",
|
||||
"copy the secret pass's publicClient skip into the URL pass -- the `kubernetes` "
|
||||
"client is PKCE-public, so kubectl's redirect URIs would never converge",
|
||||
" drift = url_drift(client, want[cid])",
|
||||
" if client.get(\"publicClient\"):\n continue\n"
|
||||
" drift = url_drift(client, want[cid])"),
|
||||
|
||||
("the two passes are coupled",
|
||||
"abort the whole run when either input is bad -- a typo in a hostname ConfigMap "
|
||||
"would stop client-SECRET repair, making credential healing depend on a file that "
|
||||
"has nothing to do with credentials",
|
||||
" if want_secrets is None and want_urls is None:",
|
||||
" if want_secrets is None or want_urls is None:"),
|
||||
|
||||
("unmanaged fields forced to empty",
|
||||
"iterate every writable field instead of only the declared ones -- a rootUrl or "
|
||||
"adminUrl the realm holds and git does not declare would be WIPED, turning "
|
||||
"'not declared' into 'must be empty'",
|
||||
" for field, declared_value in sorted(wanted.items()):",
|
||||
" for field in URL_FIELDS:\n"
|
||||
" declared_value = wanted.get(\n"
|
||||
" field, [] if field in URL_LIST_FIELDS else \"\")"),
|
||||
|
||||
("no readback verification",
|
||||
"trust the 204 -- a server that accepts the PUT and does not apply it is reported "
|
||||
"as REPAIRED, which is 'reports success while the realm never converged' again",
|
||||
" still = url_drift(after[0], want[cid])",
|
||||
" still = {}"),
|
||||
|
||||
("allow-list widened to the whole realm",
|
||||
"add a realm-level endpoint 'just in case' -- /users, /groups, /roles and "
|
||||
"/authentication stop being structurally unreachable",
|
||||
" r\"/admin/realms/[^/]+/clients\",",
|
||||
" r\"/admin/realms/[^/]+\",\n r\"/admin/realms/[^/]+/clients\","),
|
||||
|
||||
("desired-state field validation removed",
|
||||
"accept any field name from the ConfigMap -- an arbitrary client field becomes "
|
||||
"writable through a file edit, bypassing the five-field write surface entirely",
|
||||
" if field not in URL_FIELDS:",
|
||||
" if False:"),
|
||||
|
||||
("URL drift detected but never repaired",
|
||||
"leave the audit-mode early-continue in place on the write path -- drift is "
|
||||
"reported in the log and silently not fixed",
|
||||
" if audit:\n continue\n # MINIMAL representation",
|
||||
" if True:\n continue\n # MINIMAL representation"),
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||
return 2
|
||||
src = Path(sys.argv[1]).read_text()
|
||||
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
for name, why, old, new in MUTATIONS:
|
||||
old, new = dedent4(old), dedent4(new)
|
||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||
# moved and this mutation silently stopped proving anything.
|
||||
n = src.count(old)
|
||||
if n != 1:
|
||||
problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
"code moved and this mutation no longer proves anything. "
|
||||
"Re-anchor it against the current source." % (name, n))
|
||||
continue
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(src.replace(old, new, 1))
|
||||
path = fh.name
|
||||
try:
|
||||
proc = subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=240)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-42s %s" % (name, how))
|
||||
|
||||
print()
|
||||
if problems:
|
||||
print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven"
|
||||
% (len(problems), len(MUTATIONS)), file=sys.stderr)
|
||||
for p in problems:
|
||||
print(" FAIL %s" % p, file=sys.stderr)
|
||||
return 1
|
||||
print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest "
|
||||
"(the suite is falsifiable, not decorative)" % killed)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -28,12 +28,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-SEC-02"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. Universally quantified over tracked YAML: a Role/ClusterRole/binding
|
||||
# or a wildcard verb can be authored in ANY subtree, and an over-broad ServiceAccount is
|
||||
# exactly the kind of thing added alongside a new app under apps/ (which the path map
|
||||
# keys to nothing). Scoping this to R-SEC-02 would run it only when someone happens to
|
||||
# retype that ID.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -26,13 +26,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-SEC-01"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. The invariant is literally universally quantified over the tracked
|
||||
# tree: `git ls-files` -> every file must be free of plaintext credentials. ANY new or
|
||||
# edited file in ANY subtree can break it, and most subtrees (apps/, docs/, infra/,
|
||||
# runbooks/, most of platform/) attribute NO requirement at all in
|
||||
# ci/touched-requirements.sh's path map — so requirement-scoped selection would filter
|
||||
# secret hygiene out of exactly the commit that pastes a password into a new manifest.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -1,71 +1,43 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — no environment overlay ships a placeholder value, and the guard that owns
|
||||
# one refuses it
|
||||
# L0 STATIC — a placeholder value in cluster-vars is REFUSED by the guard that owns it
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# THE BUG CLASS THIS CLOSES (it has now bitten three times in one file). A guard in
|
||||
# `bootstrap/standup.sh` keys on the SHAPE of a value that lives in
|
||||
# `clusters/base/cluster-vars.yaml`, and nothing tied the two together. On 2026-09-05 the
|
||||
# S3 placeholder changed from `TBD-PROD-S3-BACKUP-ENDPOINT` to
|
||||
# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com` (commit 32d6766); the three guards
|
||||
# matched the glob `TBD-*`, a PREFIX match, and `fnmatch('https://TBD-R2-…','TBD-*')` is
|
||||
# False. All three guards silently stopped matching, and `standup.sh resume` would have
|
||||
# resumed `velero` on prod against a nonexistent hostname: installs clean, reports healthy,
|
||||
# NO BACKUPS EXIST. It was found by accident.
|
||||
#
|
||||
# WHY THIS TEST WAS RESTRUCTURED (2026-09-13). It was PERMANENTLY SKIPPED. `check.sh` exited
|
||||
# 3 when `bootstrap/standup.sh` was absent — and that file has NEVER been on `main`: it was
|
||||
# not deleted (no deletion commit exists anywhere), it landed only on unmerged prod-standup
|
||||
# branches. So an incident-derived test reported SKIP on every gate run, and SKIPPED IS NOT
|
||||
# GREEN here. The skip was also hiding a RED: run against the real `standup.sh` from
|
||||
# `work/prod-standup-campaign`, the old check FAILED 2 of 34 assertions on false positives
|
||||
# from treating the empty string as a placeholder for knobs cluster-vars authors empty on
|
||||
# purpose. Both are fixed. The invariant is now split into a half that needs no guard and a
|
||||
# half that activates when one lands, and THIS CHECK NEVER EXITS 3.
|
||||
# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`; the three guards matched the glob
|
||||
# `TBD-*`, a PREFIX match, and `fnmatch('https://TBD-R2-…','TBD-*')` is False. All three
|
||||
# guards silently stopped matching, and `standup.sh resume` would have resumed `velero` on
|
||||
# prod against a nonexistent hostname: installs clean, reports healthy, NO BACKUPS EXIST.
|
||||
# It was found by accident. The guard is fixed (`s3_is_placeholder`, substring); the
|
||||
# COUPLING was untested, which is what makes the class reintroducible.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN HALF A — always, needs no guard, input surface `clusters/`:
|
||||
# (A1) no environment overlay's EFFECTIVE cluster-vars value is placeholder-shaped by
|
||||
# an INDEPENDENT detector, for any knob no guard refuses (base is exempt: it is
|
||||
# not a bootstrap target and its loud sentinels exist to be overridden — but a
|
||||
# sentinel an overlay forgets to override IS caught, as that overlay's value);
|
||||
# (A2) an empty effective value is STATED explicitly by the overlay, never inherited
|
||||
# in silence — cluster-vars' own documented convention, and the rule that
|
||||
# separates "authored empty" from "somebody forgot";
|
||||
# (A3) a LOUD base sentinel is explicitly overridden by every overlay, so the
|
||||
# `TBD-PROD-*` convention cannot be defeated by an omitted patch op;
|
||||
# (A4) the detector itself recognises the exact 2026-09-05 string MID-STRING, accepts
|
||||
# both real endpoints the repo ships, and does not call the empty string loud;
|
||||
# (A5) ci/l0.sh runs the L0 layer and ci/gate.sh blocks the merge when it fails.
|
||||
# HALF B — when a `*_is_placeholder` predicate exists ANYWHERE in the tree (the guard
|
||||
# is DISCOVERED, not hardcoded, so a guard landing at bootstrap/standup.sh or moving to
|
||||
# a sourceable library activates this half with no edit here):
|
||||
# (B1) every authored value of a guarded knob the detector calls placeholder-shaped is
|
||||
# REFUSED by the REAL predicate, extracted from the real file and executed (never
|
||||
# reimplemented — see check.sh "HOW THE REAL GUARD IS EXERCISED"), and every
|
||||
# usable value is ACCEPTED;
|
||||
# (B2) every function that reads a guarded knob applies the guard to it (the "three
|
||||
# guards died at once" regression);
|
||||
# (B3) the guard matches its own vocabulary ANYWHERE in a value, not just as a prefix
|
||||
# — the exact anchoring defect above — and still accepts a real endpoint;
|
||||
# (B4) the extraction is faithful, exactly one file and one definition own the
|
||||
# vocabulary, and the check fails LOUDLY rather than silently testing a copy if
|
||||
# the guard stops being a self-contained predicate.
|
||||
# WHEN the REAL predicate is extracted from bootstrap/standup.sh and executed by bash
|
||||
# (never reimplemented — see check.sh "HOW THE REAL GUARD IS EXERCISED"),
|
||||
# THEN (1) every authored value of a guarded knob that is placeholder-shaped by an
|
||||
# INDEPENDENT detector is actually refused by that guard, in base and in every
|
||||
# environment overlay;
|
||||
# (2) no environment overlay carries a placeholder-shaped value for a knob no guard
|
||||
# refuses;
|
||||
# (3) every function in standup.sh that reads a guarded knob applies the guard to it
|
||||
# (the "three guards died at once" regression);
|
||||
# (4) the guard matches its own vocabulary ANYWHERE in a value, not just as a prefix
|
||||
# — the exact anchoring defect above — and still accepts a real endpoint;
|
||||
# AND (5) the extraction is faithful, and the check fails LOUDLY rather than silently
|
||||
# testing a copy if the guard stops being a self-contained predicate.
|
||||
#
|
||||
# STATUS: EXECUTABLE, and it RUNS — never SKIP. `check.sh` is a pure offline assertion
|
||||
# script (standard library + PyYAML + bash); conformance/run.sh --layer l0 runs it directly
|
||||
# and this Chainsaw Test wraps the SAME script.
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library +
|
||||
# PyYAML + bash); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test
|
||||
# wraps the SAME script.
|
||||
#
|
||||
# NOT proven: while no guard exists on `main`, HALF B IS NOT PROVEN, and the check SAYS SO on
|
||||
# stdout every run rather than swallowing it as a skip — what is proven instead is the
|
||||
# stronger-at-the-source half, that the values a guard would be asked about are not
|
||||
# placeholders at all. Also not proven: that the endpoint is REACHABLE or that backups exist
|
||||
# (runtime facts — `standup.sh s3check` probes the former from a pod, l3-fault-injection/
|
||||
# dr-restore proves the latter); that guards for knobs OTHER than the placeholder-refusing
|
||||
# ones are correct; and that the guard runs at all in a real standup (the dispatch path is
|
||||
# shell, executed only against a live cluster). A1 reads cluster-vars, not every manifest: a
|
||||
# placeholder hardcoded in a manifest rather than in a knob is out of scope. R-STATE-03 is
|
||||
# tagged for the narrow static slice only: the authored backup target cannot be a placeholder.
|
||||
# NOT proven: that the endpoint is REACHABLE or that backups exist (runtime facts —
|
||||
# `standup.sh s3check` probes the former from a pod, l3-fault-injection/dr-restore proves
|
||||
# the latter); that guards for knobs OTHER than the placeholder-refusing ones are correct;
|
||||
# and that the guard runs at all in a real standup (the dispatch path is shell, executed
|
||||
# only against a live cluster). R-STATE-03 is tagged for the narrow static slice only: the
|
||||
# authored backup target cannot be a placeholder that the guard fails to recognise.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
@@ -77,33 +49,20 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-02, R-STATE-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# NO `scope` ANNOTATION, DELIBERATELY — this invariant IS requirement-scoped, and was
|
||||
# re-audited as such on 2026-09-13 after the restructure changed the input surface.
|
||||
# HALF A reads clusters/base/cluster-vars.yaml plus clusters/<env>/kustomization.yaml;
|
||||
# ci/touched-requirements.sh keys clusters/* to R-ENV-02 + R-GITOPS-01, so every path
|
||||
# that can break HALF A attributes a requirement declared above. HALF B reads whichever
|
||||
# file defines the guard; bootstrap/* is keyed to R-STATE-03 + R-GITOPS-01, which covers
|
||||
# the place a standup guard lands. Requirement-scoped selection therefore cannot hide a
|
||||
# breaking change, and declaring whole-repo scope would only add runtime, not coverage.
|
||||
# The one residual: (A5) reads ci/l0.sh and ci/gate.sh, and ci/* attributes neither
|
||||
# R-ENV-02 nor R-STATE-03 — but that wiring is separately and unconditionally asserted by
|
||||
# the whole-repo-scoped tests/l0-static/whole-repo-invariants-wired, so (A5) here is
|
||||
# belt-and-braces rather than the only proof.
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
No environment overlay can ship a placeholder-shaped cluster-vars value that nothing
|
||||
refuses, and where a real bootstrap guard owns such a knob it actually recognises
|
||||
every authored value as a placeholder — so a value change in one file cannot silently
|
||||
disarm a guard in another and enable velero against an endpoint that does not exist.
|
||||
Every placeholder-shaped value authored in cluster-vars, for any environment
|
||||
overlay, is actually recognised as a placeholder by the real bootstrap guard that
|
||||
is supposed to refuse it — so a value change in one file cannot silently disarm a
|
||||
guard in another and enable velero against an endpoint that does not exist.
|
||||
spec:
|
||||
steps:
|
||||
- name: refuse-authored-placeholders-and-couple-them-to-any-real-guard
|
||||
- name: couple-authored-placeholders-to-the-real-guard
|
||||
description: >-
|
||||
Screen every authored effective cluster-vars value of every environment overlay
|
||||
against an independent placeholder detector, assert empty values are stated rather
|
||||
than inherited, and — when a *_is_placeholder predicate exists anywhere in the tree
|
||||
— extract and execute the REAL guard against those same values.
|
||||
Extract the real placeholder predicate from bootstrap/standup.sh, run it against
|
||||
every authored cluster-vars value the repo actually carries, and assert the guard
|
||||
refuses each one an independent detector calls a placeholder.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -12,74 +12,20 @@
|
||||
# placeholder. Three of those guards keyed on the SHAPE of `S3_BACKUP_ENDPOINT`, which is
|
||||
# authored in `clusters/base/cluster-vars.yaml`. On 2026-09-05 the placeholder changed
|
||||
# from `TBD-PROD-S3-BACKUP-ENDPOINT` to `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`
|
||||
# (the MinIO -> Cloudflare R2 move, commit 32d6766). The guards matched the glob `TBD-*` —
|
||||
# a PREFIX match — and `fnmatch('https://TBD-R2-…','TBD-*')` is FALSE. All three guards
|
||||
# silently stopped matching. `standup.sh resume` would then have resumed `velero` on prod
|
||||
# against a hostname that does not exist: the HelmRelease installs clean, the Kustomization
|
||||
# reports Ready, and NO BACKUPS EXIST. It was found by accident.
|
||||
# (the MinIO -> Cloudflare R2 move). The guards matched the glob `TBD-*` — a PREFIX match —
|
||||
# and `fnmatch('https://TBD-R2-…','TBD-*')` is FALSE. All three guards silently stopped
|
||||
# matching. `standup.sh resume` would then have resumed `velero` on prod against a hostname
|
||||
# that does not exist: the HelmRelease installs clean, the Kustomization reports Ready, and
|
||||
# NO BACKUPS EXIST. The only reason it was not shipped is that someone noticed by accident.
|
||||
#
|
||||
# WHY THIS CHECK WAS REWRITTEN (2026-09-13) — IT WAS SKIPPING FOREVER
|
||||
# ------------------------------------------------------------------
|
||||
# The original version keyed its ENTIRE existence on `bootstrap/standup.sh` and exited 3
|
||||
# (SKIP) when that file was absent. Established by measurement, not assumption:
|
||||
#
|
||||
# * `bootstrap/standup.sh` HAS NEVER EXISTED ON `main`. It was not deleted — there is no
|
||||
# deletion commit for it anywhere (`git log --all --diff-filter=D -- bootstrap/standup.sh`
|
||||
# is empty). It lives on unmerged prod-standup branches (`work/prod-standup-campaign`,
|
||||
# `prod/*`, `feat/prod-hardening`) and never landed. `git ls-tree main bootstrap/` lists
|
||||
# only README.md, flux/, gitea/, sops/. docs/BULK-NFS-STORAGE.md §7 already recorded
|
||||
# this, including the consequence: "nothing in the gate currently refuses a
|
||||
# placeholder-shaped overlay value".
|
||||
# * So this check reported SKIP on every gate run since it landed, and SKIPPED IS NOT
|
||||
# GREEN in this repo. An incident-derived test that can only ever skip is the quiet
|
||||
# version of the failure it was written to prevent.
|
||||
# * Worse, the skip was HIDING A RED. Run against the real `standup.sh` from
|
||||
# `work/prod-standup-campaign`, the original check FAILED 2 of 34 assertions — both
|
||||
# false positives from treating the EMPTY STRING as placeholder-shaped for knobs whose
|
||||
# emptiness is deliberately authored (`BACKUP_CREDENTIALS_SUFFIX` and
|
||||
# `VELERO_BACKUP_PREFIX`, both `""` on dev). cluster-vars documents that emptiness as
|
||||
# load-bearing ("THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING … an
|
||||
# EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions"). So the check would have
|
||||
# false-blocked the moment its own input landed. That is fixed below.
|
||||
#
|
||||
# THE HAZARD IS NOT GONE, so this coverage is NOT obsolete and was NOT deleted. The
|
||||
# invariant has two independent halves, and only one of them needs `standup.sh`:
|
||||
#
|
||||
# HALF A — THE AUTHORED VALUES (provable on `main` today, needs NO guard).
|
||||
# No environment overlay may ship a placeholder-shaped effective cluster-vars value.
|
||||
# This is the OUTCOME the 2026-09-05 guard existed to prevent, asserted at the source
|
||||
# instead of at the guard: if prod's `S3_BACKUP_ENDPOINT` is never a placeholder, velero
|
||||
# cannot be pointed at a hostname that does not exist, whether or not a guard catches
|
||||
# it. Its entire input surface is `clusters/`, all of which is on `main`.
|
||||
# HALF B — THE COUPLING (activates automatically when a guard lands).
|
||||
# The real predicate, extracted from the real file and executed, agrees with an
|
||||
# INDEPENDENT detector about every authored value; every consumer of a guarded knob
|
||||
# applies the guard; the guard matches its vocabulary ANYWHERE, not as a prefix.
|
||||
#
|
||||
# THIS CHECK NEVER EXITS 3. Half A always runs, against real files, with an assertion
|
||||
# floor. Half B runs when a guard exists and is reported as NOT PROVEN when one does not —
|
||||
# stated loudly in the output rather than swallowed as a skip.
|
||||
#
|
||||
# THE GUARD IS DISCOVERED, NOT HARDCODED (this is the "re-point" made automatic)
|
||||
# -----------------------------------------------------------------------------
|
||||
# The original hardcoded `bootstrap/standup.sh`, which is exactly why a file that never
|
||||
# landed could disable the whole check. Instead, every tracked `*.sh` in the tree is
|
||||
# scanned for a `<name>_is_placeholder()` definition:
|
||||
# * exactly one such FILE -> that is the guard; Half B runs against it, wherever it is.
|
||||
# A guard that lands at `bootstrap/standup.sh`, or moves to `bootstrap/lib/placeholder.sh`,
|
||||
# is picked up with no edit to this check.
|
||||
# * none -> Half A only, with the gap named in the output.
|
||||
# * more than one file -> FAIL. Two files defining the placeholder vocabulary is the
|
||||
# 2026-09-05 divergence with a second file's name on it; this check cannot say which
|
||||
# one a standup run would source.
|
||||
# `PLACEHOLDER_GUARD_FILE` overrides discovery (repo-relative) for negative fixtures.
|
||||
# NOTE: discovery is shell-only, because the guard family is shell. If the refusal ever
|
||||
# moves to a non-shell mechanism (a Kyverno policy, a CEL admission rule), discovery finds
|
||||
# nothing and says so — re-point this check deliberately in that change.
|
||||
# The predicate is fixed (`s3_is_placeholder`, substring match). The COUPLING is what was
|
||||
# never tested: a guard in one file keying on a value in another, with nothing tying them
|
||||
# together. This check is that tie, and it is derived from BOTH real files — no current
|
||||
# string is hardcoded here, so it does not rot when the placeholder or the guard changes.
|
||||
#
|
||||
# HOW THE REAL GUARD IS EXERCISED (and why not a copy)
|
||||
# ---------------------------------------------------
|
||||
# `standup.sh` is NOT sourceable: at top level it parses argv, hard-exits when
|
||||
# `bootstrap/standup.sh` is NOT sourceable: at top level it parses argv, hard-exits when
|
||||
# --kubeconfig/--expect-server/<step> are absent, `cd`s to the repo root, exports
|
||||
# KUBECONFIG, installs an EXIT trap and finally dispatches steps against a live cluster.
|
||||
# So this check EXTRACTS the predicate's own source lines from the file and sources ONLY
|
||||
@@ -89,119 +35,105 @@
|
||||
# * the extracted body must be a SELF-CONTAINED predicate — `case`/`return` and nothing
|
||||
# else, no command substitution, no call to another function. If the guard ever grows
|
||||
# a dependency, extraction stops being faithful and THIS CHECK FAILS LOUDLY, naming the
|
||||
# remedy (move the predicate into a small sourceable library) rather than quietly
|
||||
# falling back to testing a copy of the logic;
|
||||
# remedy (move the predicate into a small sourceable library that standup.sh sources)
|
||||
# rather than quietly falling back to testing a copy of the logic;
|
||||
# * a CANARY pins the extraction itself: the extracted function must refuse the empty
|
||||
# string and accept a realistic endpoint.
|
||||
# string and accept a realistic endpoint. An extraction that grabbed the wrong lines,
|
||||
# or a predicate degraded to always-true / always-false, fails there — this repo has
|
||||
# already been bitten by a check that passed when perturbed because it asked a question
|
||||
# that was always satisfiable.
|
||||
# There is deliberately NO second implementation of the matching logic anywhere in this file.
|
||||
#
|
||||
# THE INDEPENDENT DETECTOR (and the empty-string distinction the old version got wrong)
|
||||
# ------------------------------------------------------------------------------------
|
||||
# Asking "does the guard agree with the guard?" proves nothing, so placeholder-shaped is
|
||||
# decided by a detector written HERE, whose vocabulary is the UNION of two independent
|
||||
# sources:
|
||||
# THE INDEPENDENT DETECTOR (the other half of the comparison)
|
||||
# ----------------------------------------------------------
|
||||
# Asking "does the guard agree with the guard?" proves nothing, so the set of
|
||||
# placeholder-shaped values is decided by a detector written HERE, whose vocabulary is the
|
||||
# UNION of two independent sources:
|
||||
# * SEED — universal loud-placeholder idioms (TBD, TODO, FIXME, PLACEHOLDER, REPLACE-ME,
|
||||
# CHANGE-ME, FILL-ME, SET-ME, XXXX), plus `<angle-bracket>` templates. NOT derived from
|
||||
# any guard, so DELETING a guard arm cannot make a value stop being placeholder-shaped.
|
||||
# * the guard's OWN vocabulary (when a guard exists), read out of its case-arms with the
|
||||
# glob anchors STRIPPED. Stripping the anchors is what makes the comparison non-circular
|
||||
# in the dimension that actually broke: the detector matches a word ANYWHERE,
|
||||
# case-insensitively, and the guard must agree. A prefix-anchored guard cannot pass.
|
||||
#
|
||||
# LOUD vs EMPTY — the correction. A LOUD placeholder is a positive statement that a value
|
||||
# was never decided, and it is always a defect in an overlay. The EMPTY STRING is not:
|
||||
# cluster-vars authors `""` deliberately for suffix/prefix knobs, and Flux strict
|
||||
# substitution fails on an UNDEFINED variable, not on one defined as `""`. So emptiness is
|
||||
# NOT treated as a loud placeholder. It is held to a different, real rule instead (A2): an
|
||||
# empty effective value must be STATED EXPLICITLY by that overlay, never inherited in
|
||||
# silence — which is cluster-vars' own documented convention ("Both overlays state this
|
||||
# knob explicitly … so neither inherits it silently"). Emptiness still counts as
|
||||
# placeholder-shaped for a knob a GUARD covers, because a `""` case-arm in the guard is the
|
||||
# guard author's declaration that empty is not a usable target for that knob.
|
||||
# CHANGE-ME, FILL-ME, SET-ME, XXXX), plus `<angle-bracket>` templates and the empty
|
||||
# string. These are NOT derived from the guard, so DELETING a guard arm cannot make a
|
||||
# value stop being placeholder-shaped — it makes assertion (1) fail, which is the point.
|
||||
# A value using an idiom the guard does not know is a finding either way: fix it by
|
||||
# giving the knob a real value, or by teaching the guard the idiom.
|
||||
# * the guard's OWN vocabulary, read out of its case-arms with the glob anchors STRIPPED.
|
||||
# Stripping the anchors is what makes the comparison non-circular in the dimension that
|
||||
# actually broke: the detector matches a word ANYWHERE, case-insensitively, and the
|
||||
# guard must agree. A prefix-anchored guard cannot pass.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN HALF A (always):
|
||||
# (A1) VALUES: no environment overlay's EFFECTIVE cluster-vars value is LOUD
|
||||
# placeholder-shaped, for any knob that no guard refuses. (Base alone is exempt:
|
||||
# it is not a bootstrap target — `flux bootstrap --path=clusters/<env>` — and its
|
||||
# loud sentinels exist to be overridden. A base placeholder an overlay forgets to
|
||||
# override IS caught, because it becomes that overlay's effective value.)
|
||||
# (A2) EXPLICITNESS: an empty effective value is stated explicitly by that overlay.
|
||||
# (A3) SENTINEL DISCIPLINE: if base carries a LOUD sentinel for a knob, every overlay
|
||||
# states that knob explicitly — the `TBD-PROD-*` convention only works if no
|
||||
# overlay can inherit the sentinel by omission.
|
||||
# (A4) DETECTOR CANARIES: the detector recognises the exact 2026-09-05 string
|
||||
# MID-STRING, accepts both real endpoints the repo actually ships, and does NOT
|
||||
# call the empty string loud.
|
||||
# (A5) LOAD-BEARING: ci/l0.sh runs the L0 conformance layer on every gate run and
|
||||
# ci/gate.sh sets the merge block when that stage fails.
|
||||
# HALF B (only when a guard file is discovered):
|
||||
# (B1) VALUES: for every knob a guard is responsible for, every authored value — the
|
||||
# base default AND each overlay's value — that the detector calls placeholder-
|
||||
# shaped is REFUSED by the real guard, and every value it calls usable is
|
||||
# ACCEPTED (a guard that refuses a real endpoint leaves velero suspended forever,
|
||||
# the same class of silent failure).
|
||||
# (B2) CONSUMERS: every function that reads a guarded knob (via `cvar`, following one
|
||||
# hop of variable aliasing) applies a placeholder guard to it. This is the "all
|
||||
# three guards died at once" regression stated as a property.
|
||||
# (B3) ANCHORING: for each word in the guard's own vocabulary, the guard refuses a
|
||||
# value carrying that word in the MIDDLE (not at the start) — the precise defect
|
||||
# of 2026-09-05 — while still accepting a realistic endpoint.
|
||||
# (B4) HARNESS: exactly one definition per predicate, one guard FILE in the tree, the
|
||||
# extracted body verbatim and self-contained, the canary holds, no divergent
|
||||
# second copy of the matching logic, and assertion floors keep a pass from
|
||||
# degenerating into a no-op.
|
||||
# WHEN the real predicate(s) are extracted from bootstrap/standup.sh and executed,
|
||||
# THEN (1) VALUES: for every knob a guard is responsible for, every authored value — the
|
||||
# base default AND each environment overlay's value — that the independent
|
||||
# detector calls placeholder-shaped is REFUSED by the real guard, and every value
|
||||
# it calls usable is ACCEPTED (a guard that refuses a real endpoint leaves velero
|
||||
# suspended forever, which is the same class of silent failure);
|
||||
# (2) OVERLAYS: no environment overlay's EFFECTIVE cluster-vars carries a
|
||||
# placeholder-shaped value for a knob that no guard refuses. (Base alone is
|
||||
# exempt: it is not a bootstrap target — `flux bootstrap --path=clusters/<env>` —
|
||||
# and its loud sentinels exist precisely to be overridden. A base placeholder an
|
||||
# overlay forgets to override IS caught, because it becomes that overlay's
|
||||
# effective value.)
|
||||
# (3) CONSUMERS: every function in standup.sh that reads a guarded knob (via `cvar`,
|
||||
# following one hop of variable aliasing) applies a placeholder guard to it. This
|
||||
# is the "all three guards died at once" regression stated as a property: the
|
||||
# guard being correct is worthless if a consumer stops calling it.
|
||||
# (4) ANCHORING: for each word in the guard's own vocabulary, the guard refuses a
|
||||
# value carrying that word in the MIDDLE (not at the start) — the precise defect
|
||||
# of 2026-09-05 — while still accepting a realistic endpoint.
|
||||
# (5) HARNESS: exactly one definition per predicate, the extracted body is verbatim
|
||||
# from the file and self-contained, the canary holds, no divergent second copy of
|
||||
# the matching logic exists elsewhere in standup.sh, and a floor on the number of
|
||||
# assertions run keeps a pass from degenerating into a no-op.
|
||||
# AND (6) LOAD-BEARING: ci/l0.sh runs the L0 conformance layer on every gate run and
|
||||
# ci/gate.sh sets the merge block when that stage fails.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps)
|
||||
# - While no guard exists on `main`, HALF B IS NOT PROVEN AT ALL, and the check says so on
|
||||
# stdout every run. What is proven is the stronger-at-the-source half: the values a
|
||||
# guard would be asked about are not placeholders in the first place. This is a real
|
||||
# narrowing of coverage versus the intent, and it is reported, not hidden.
|
||||
# - Nothing about RUNTIME: not that the endpoint resolves, not that credentials work, not
|
||||
# that a backup exists. `standup.sh s3check` probes reachability from a pod (operator-
|
||||
# run, cluster-gated) and conformance/tests/l3-fault-injection/dr-restore proves restore.
|
||||
# - Half B does not execute the dispatch path, so it cannot prove the guard is REACHED in
|
||||
# a real run — only that, when reached, it refuses what it must. (B2) is a text-level
|
||||
# taint trace over one file, not an interpreter.
|
||||
# - It does not execute standup.sh's dispatch path, so it cannot prove the guard is
|
||||
# REACHED in a real run — only that, when reached, it refuses what it must. The
|
||||
# consumer property (3) is a static approximation of reachability: it is a text-level
|
||||
# taint trace over one file, not an interpreter, and a guard whose result is computed
|
||||
# and then ignored would satisfy it.
|
||||
# - It says nothing about guards keyed on shapes OTHER than placeholders (e.g. the
|
||||
# EXTERNALDNS_TXT_OWNER_ID == 'default' test), nor about placeholder-shaped values
|
||||
# outside cluster-vars (the Ansible inventory's BLOCKED/NOT-A-DEVICE sentinels are
|
||||
# asserted by the node_baseline role itself, at play time).
|
||||
# - A1 reads cluster-vars, not every manifest. A placeholder hardcoded in a manifest
|
||||
# rather than in a knob is out of scope here.
|
||||
#
|
||||
# SELECTION. `ci/touched-requirements.sh` keys `clusters/*` -> R-ENV-02 (Half A's whole
|
||||
# input surface) and `bootstrap/*` -> R-STATE-03 (where a guard lands), so every path that
|
||||
# can break this check attributes a requirement it declares. Deliberately requirement-scoped,
|
||||
# not whole-repo — see chainsaw-test.yaml.
|
||||
# - Stage 3b of ci/l0.sh scopes the conformance run to the TOUCHED requirements. An edit
|
||||
# to bootstrap/* attributes NO requirement in ci/touched-requirements.sh's path map, so
|
||||
# the touched set is empty and the FULL L0 layer runs (verified: ci/lib.sh
|
||||
# load_touched_requirements) — this check runs. An edit to clusters/* maps to
|
||||
# R-ENV-02, which this check carries, so a cluster-vars value change also selects it.
|
||||
# A change that touches standup.sh ALONGSIDE some unrelated mapped path (say
|
||||
# platform/cilium/) would scope the run away from this check; closing that needs a
|
||||
# `bootstrap/*` entry in the path map, which is not this check's file to edit.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
GUARD_OVERRIDE="${PLACEHOLDER_GUARD_FILE:-}"
|
||||
STANDUP_REL="${STANDUP_REL:-bootstrap/standup.sh}"
|
||||
|
||||
cd "${ROOT}"
|
||||
|
||||
# The authored half of the coupling is the one thing this check cannot do without.
|
||||
if [ ! -f "${ROOT}/clusters/base/cluster-vars.yaml" ]; then
|
||||
echo "CHECK FAIL (placeholder-guard-coupling): clusters/base/cluster-vars.yaml is absent — the authored values this check audits do not exist. That is not a SKIP: the knobs this invariant is about have been moved or deleted, and this check must be re-pointed in the SAME change." >&2
|
||||
exit 1
|
||||
if [ ! -f "${ROOT}/${STANDUP_REL}" ]; then
|
||||
echo "check SKIPPED (placeholder-guard-coupling): ${STANDUP_REL} is absent — nothing to couple." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "${WORK}"' EXIT
|
||||
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# PHASE 1 (python): discover the guard, derive the corpus from the real files, and run
|
||||
# every assertion that does not need a shell. Writes:
|
||||
# ${WORK}/guards.sh the predicate source lines, verbatim, and nothing else (may be empty)
|
||||
# ${WORK}/corpus.tsv <guard>\t<expect: placeholder|usable>\t<origin>\t<value> (may be empty)
|
||||
# PHASE 1 (python): extract the real predicate(s), derive the corpus from the real files,
|
||||
# and run every assertion that does not need a shell. Writes:
|
||||
# ${WORK}/guards.sh the predicate source lines, verbatim, and nothing else
|
||||
# ${WORK}/corpus.tsv <guard>\t<expect: placeholder|usable>\t<origin>\t<value>
|
||||
# ${WORK}/findings.txt one line per failed assertion
|
||||
# ${WORK}/counts.txt <assertions-run>
|
||||
# ${WORK}/mode.txt the discovered guard's repo-relative path, or empty
|
||||
# ---------------------------------------------------------------------------------------
|
||||
"${PYTHON}" - "${ROOT}" "${WORK}" "${GUARD_OVERRIDE}" <<'PY'
|
||||
"${PYTHON}" - "${ROOT}" "${WORK}" "${STANDUP_REL}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -213,7 +145,7 @@ except ImportError: # pragma: no cover
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT, WORK, OVERRIDE = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]
|
||||
ROOT, WORK, STANDUP_REL = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
@@ -232,159 +164,116 @@ def bail(msg):
|
||||
(WORK / "counts.txt").write_text("0\n")
|
||||
(WORK / "corpus.tsv").write_text("")
|
||||
(WORK / "guards.sh").write_text("")
|
||||
(WORK / "mode.txt").write_text("")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
TEXT = (ROOT / STANDUP_REL).read_text()
|
||||
LINES = TEXT.splitlines()
|
||||
|
||||
# ------------------------------------------------------------------ (5) extract the guard
|
||||
DEF_RE = re.compile(r"^([a-z0-9_]+_is_placeholder)\(\)\s*\{")
|
||||
SELF_DIR = ROOT / "conformance" / "tests" / "l0-static" / "placeholder-guard-coupling"
|
||||
SKIP_DIRS = {".git", "_artifacts", "node_modules", ".venv"}
|
||||
defs: dict[str, list[int]] = {}
|
||||
for i, line in enumerate(LINES):
|
||||
m = DEF_RE.match(line)
|
||||
if m:
|
||||
defs.setdefault(m.group(1), []).append(i)
|
||||
|
||||
# ------------------------------------------------------------------ discover the guard FILE
|
||||
# Hardcoding a path is what let a file that never landed disable this whole check.
|
||||
if OVERRIDE:
|
||||
cand = [ROOT / OVERRIDE]
|
||||
if not cand[0].is_file():
|
||||
bail(f"PLACEHOLDER_GUARD_FILE={OVERRIDE} does not exist")
|
||||
else:
|
||||
cand = []
|
||||
for p in sorted(ROOT.rglob("*.sh")):
|
||||
if SKIP_DIRS & set(p.parts) or SELF_DIR in p.parents:
|
||||
continue
|
||||
try:
|
||||
text = p.read_text()
|
||||
except (OSError, UnicodeDecodeError): # pragma: no cover
|
||||
continue
|
||||
if any(DEF_RE.match(line) for line in text.splitlines()):
|
||||
cand.append(p)
|
||||
if not defs:
|
||||
bail(f"{STANDUP_REL} defines no *_is_placeholder predicate. Either the guard was renamed "
|
||||
"(update this check's discovery pattern in the SAME change) or it was DELETED, which "
|
||||
"is the 2026-09-05 failure with the guard removed instead of disarmed.")
|
||||
|
||||
check(len(cand) <= 1,
|
||||
"more than one file defines a *_is_placeholder predicate ("
|
||||
+ ", ".join(str(p.relative_to(ROOT)) for p in cand)
|
||||
+ ") — two files owning the placeholder vocabulary is the 2026-09-05 divergence with a "
|
||||
"second file's name on it, and this check cannot say which one a standup run would "
|
||||
"source. Keep exactly one, and source it from the other.")
|
||||
|
||||
GUARD_REL = str(cand[0].relative_to(ROOT)) if len(cand) == 1 else ""
|
||||
(WORK / "mode.txt").write_text(GUARD_REL + "\n")
|
||||
|
||||
# --------------------------------------------------------- HALF B setup: extract the guard
|
||||
guard_src: list[str] = []
|
||||
guard_words: dict[str, list[str]] = {}
|
||||
funcs: dict[str, list[str]] = {}
|
||||
# An "argument-shaped" call: <name> "<something>"
|
||||
CALL_RE = re.compile(r"\b([a-z0-9_]+_is_placeholder)\s+\"([^\"]*)\"")
|
||||
|
||||
if GUARD_REL:
|
||||
TEXT = (ROOT / GUARD_REL).read_text()
|
||||
LINES = TEXT.splitlines()
|
||||
guard_src: list[str] = []
|
||||
guard_words: dict[str, list[str]] = {}
|
||||
ARM_RE = re.compile(r"^(?P<pats>[^()]*?)\)\s*return\s+[01]\s*;;$")
|
||||
ALLOWED = (
|
||||
re.compile(r"^$"),
|
||||
re.compile(r"^#"),
|
||||
re.compile(r"^case\s+\"\$\{?[0-9A-Za-z_]+\}?\"\s+in$"),
|
||||
re.compile(r"^esac$"),
|
||||
re.compile(r"^return\s+[01]$"),
|
||||
ARM_RE,
|
||||
)
|
||||
|
||||
defs: dict[str, list[int]] = {}
|
||||
for i, line in enumerate(LINES):
|
||||
m = DEF_RE.match(line)
|
||||
if m:
|
||||
defs.setdefault(m.group(1), []).append(i)
|
||||
for name, at in sorted(defs.items()):
|
||||
check(len(at) == 1,
|
||||
f"{STANDUP_REL} defines {name}() {len(at)} times (lines "
|
||||
f"{', '.join(str(a + 1) for a in at)}) — which one runs depends on file order, and "
|
||||
"this check can no longer say which one it extracted. Keep exactly one definition.")
|
||||
if len(at) != 1:
|
||||
continue
|
||||
start = at[0]
|
||||
end = next((j for j in range(start + 1, len(LINES)) if LINES[j] == "}"), None)
|
||||
if end is None:
|
||||
bail(f"{name}() at {STANDUP_REL}:{start + 1} has no closing brace at column 0 — the "
|
||||
"extraction boundary this check relies on is gone. Restore the conventional "
|
||||
"layout, or move the predicate into a sourceable library and point this check at it.")
|
||||
body = LINES[start:end + 1]
|
||||
block = "\n".join(body)
|
||||
check(block in TEXT,
|
||||
f"extracted {name}() is not a verbatim substring of {STANDUP_REL} — extraction bug; "
|
||||
"the harness would be testing something the script does not contain")
|
||||
|
||||
ARM_RE = re.compile(r"^(?P<pats>[^()]*?)\)\s*return\s+[01]\s*;;$")
|
||||
ALLOWED = (
|
||||
re.compile(r"^$"),
|
||||
re.compile(r"^#"),
|
||||
re.compile(r"^case\s+\"\$\{?[0-9A-Za-z_]+\}?\"\s+in$"),
|
||||
re.compile(r"^esac$"),
|
||||
re.compile(r"^return\s+[01]$"),
|
||||
ARM_RE,
|
||||
)
|
||||
|
||||
for name, at in sorted(defs.items()):
|
||||
check(len(at) == 1,
|
||||
f"{GUARD_REL} defines {name}() {len(at)} times (lines "
|
||||
f"{', '.join(str(a + 1) for a in at)}) — which one runs depends on file order, and "
|
||||
"this check can no longer say which one it extracted. Keep exactly one definition.")
|
||||
if len(at) != 1:
|
||||
words: list[str] = []
|
||||
for raw in body[1:-1]:
|
||||
s = raw.strip()
|
||||
if not any(rx.match(s) for rx in ALLOWED) or "$(" in s or "`" in s:
|
||||
bail(f"{name}() at {STANDUP_REL}:{start + 1} is no longer a SELF-CONTAINED "
|
||||
f"case/return predicate (offending line: {s!r}). This check exercises the REAL "
|
||||
"function by sourcing its own lines; a body that calls out to anything else "
|
||||
"cannot be extracted faithfully, and silently testing a COPY of the logic is "
|
||||
"exactly the failure mode this check exists to prevent. REMEDY: move the "
|
||||
"predicate into a tiny library that standup.sh sources (e.g. "
|
||||
"bootstrap/lib/placeholder.sh) and point STANDUP_REL/this extractor at it — do "
|
||||
"NOT reimplement the matching here.")
|
||||
m = ARM_RE.match(s)
|
||||
if not m:
|
||||
continue
|
||||
start = at[0]
|
||||
end = next((j for j in range(start + 1, len(LINES)) if LINES[j] == "}"), None)
|
||||
if end is None:
|
||||
bail(f"{name}() at {GUARD_REL}:{start + 1} has no closing brace at column 0 — the "
|
||||
"extraction boundary this check relies on is gone. Restore the conventional "
|
||||
"layout, or move the predicate into a sourceable library.")
|
||||
body = LINES[start:end + 1]
|
||||
block = "\n".join(body)
|
||||
check(block in TEXT,
|
||||
f"extracted {name}() is not a verbatim substring of {GUARD_REL} — extraction bug; "
|
||||
"the harness would be testing something the script does not contain")
|
||||
for pat in m.group("pats").split("|"):
|
||||
pat = pat.strip()
|
||||
if pat in ('""', "''", ""):
|
||||
continue # the empty-value arm: covered by the canary
|
||||
words.append(pat.strip("'\"").strip("*"))
|
||||
check(len(words) >= 1,
|
||||
f"{name}() has no non-empty placeholder pattern left — every arm that recognises a "
|
||||
"loud placeholder string has been removed, so the guard now only refuses the empty "
|
||||
"value and any 'TBD'-style endpoint would sail straight through it")
|
||||
guard_words[name] = words
|
||||
guard_src.append(block)
|
||||
|
||||
words: list[str] = []
|
||||
for raw in body[1:-1]:
|
||||
s = raw.strip()
|
||||
if not any(rx.match(s) for rx in ALLOWED) or "$(" in s or "`" in s:
|
||||
bail(f"{name}() at {GUARD_REL}:{start + 1} is no longer a SELF-CONTAINED "
|
||||
f"case/return predicate (offending line: {s!r}). This check exercises the "
|
||||
"REAL function by sourcing its own lines; a body that calls out to anything "
|
||||
"else cannot be extracted faithfully, and silently testing a COPY of the "
|
||||
"logic is exactly the failure mode this check exists to prevent. REMEDY: "
|
||||
"move the predicate into a tiny library that the script sources (e.g. "
|
||||
"bootstrap/lib/placeholder.sh) — discovery will find it — do NOT "
|
||||
"reimplement the matching here.")
|
||||
m = ARM_RE.match(s)
|
||||
if not m:
|
||||
continue
|
||||
for pat in m.group("pats").split("|"):
|
||||
pat = pat.strip()
|
||||
if pat in ('""', "''", ""):
|
||||
continue # the empty-value arm: covered by the canary
|
||||
words.append(pat.strip("'\"").strip("*"))
|
||||
check(len(words) >= 1,
|
||||
f"{name}() has no non-empty placeholder pattern left — every arm that recognises a "
|
||||
"loud placeholder string has been removed, so the guard now only refuses the empty "
|
||||
"value and any 'TBD'-style endpoint would sail straight through it")
|
||||
guard_words[name] = words
|
||||
guard_src.append(block)
|
||||
(WORK / "guards.sh").write_text("\n".join(guard_src) + "\n")
|
||||
|
||||
if not guard_words:
|
||||
bail(f"{GUARD_REL} was discovered as the guard file but yielded no usable predicate — "
|
||||
"the definitions are there but none could be extracted. Fix the extraction in the "
|
||||
"SAME change; do not let this degrade to a pass.")
|
||||
|
||||
# ------------------------------------------- no divergent second copy of the logic
|
||||
# The guard's own vocabulary in EXECUTABLE text outside the predicate means a second,
|
||||
# independently-drifting implementation of the same test (which is how three guards came
|
||||
# to disagree with one value in the first place).
|
||||
guard_line_nums = {j for name, at in defs.items() if len(at) == 1
|
||||
for j in range(at[0], next(k for k in range(at[0] + 1, len(LINES))
|
||||
if LINES[k] == "}") + 1)}
|
||||
divergent: list[str] = []
|
||||
_words = sorted({w.upper() for ws in guard_words.values() for w in ws if w})
|
||||
for j, line in enumerate(LINES):
|
||||
if j in guard_line_nums or line.strip().startswith("#"):
|
||||
continue
|
||||
code = line.split("#", 1)[0]
|
||||
hit = [w for w in _words if w in code.upper()]
|
||||
if hit:
|
||||
divergent.append(f"{GUARD_REL}:{j + 1} ({', '.join(hit)}): {line.strip()!r}")
|
||||
check(not divergent,
|
||||
"the guard's own placeholder vocabulary appears in EXECUTABLE text outside the "
|
||||
"predicate — a SECOND implementation of the same test, which is precisely how a value "
|
||||
"change disarmed three guards at once. Call the predicate instead. Sites: "
|
||||
+ " | ".join(divergent))
|
||||
|
||||
# function bodies, for the consumer trace
|
||||
FUNC_RE = re.compile(r"^([a-z0-9_]+)\(\)\s*\{")
|
||||
i = 0
|
||||
while i < len(LINES):
|
||||
m = FUNC_RE.match(LINES[i])
|
||||
if m:
|
||||
end = next((j for j in range(i + 1, len(LINES)) if LINES[j] == "}"), len(LINES) - 1)
|
||||
funcs[m.group(1)] = LINES[i + 1:end]
|
||||
i = end + 1
|
||||
continue
|
||||
i += 1
|
||||
check(len(funcs) > 5, f"parsed only {len(funcs)} function(s) out of {GUARD_REL} — the "
|
||||
"consumer assertion (B2) would be vacuous")
|
||||
|
||||
(WORK / "guards.sh").write_text(("\n".join(guard_src) + "\n") if guard_src else "")
|
||||
# ------------------------------------------------- no divergent second copy of the logic
|
||||
# Any of the guard's own vocabulary appearing in EXECUTABLE text outside the predicate means
|
||||
# a second, independently-drifting implementation of the same test (which is how three
|
||||
# guards came to disagree with one value in the first place).
|
||||
guard_line_nums = {j for name, at in defs.items() if len(at) == 1
|
||||
for j in range(at[0], next(k for k in range(at[0] + 1, len(LINES))
|
||||
if LINES[k] == "}") + 1)}
|
||||
all_words = sorted({w.upper() for ws in guard_words.values() for w in ws if w})
|
||||
divergent: list[str] = []
|
||||
for j, line in enumerate(LINES):
|
||||
if j in guard_line_nums or line.strip().startswith("#"):
|
||||
continue
|
||||
code = line.split("#", 1)[0]
|
||||
hit = [w for w in all_words if w in code.upper()]
|
||||
if hit:
|
||||
divergent.append(f"{STANDUP_REL}:{j + 1} ({', '.join(hit)}): {line.strip()!r}")
|
||||
check(not divergent,
|
||||
"the guard's own placeholder vocabulary appears in EXECUTABLE text outside the "
|
||||
f"predicate — a SECOND implementation of the same test, which is precisely how a value "
|
||||
f"change disarmed three guards at once. Call the predicate instead. Sites: "
|
||||
+ " | ".join(divergent))
|
||||
|
||||
# ------------------------------------------------------------- cluster-vars, as authored
|
||||
CV = ROOT / "clusters" / "base" / "cluster-vars.yaml"
|
||||
if not CV.is_file():
|
||||
bail("clusters/base/cluster-vars.yaml is absent — the authored half of the coupling is "
|
||||
"missing and this check cannot mean anything")
|
||||
|
||||
base: dict[str, str] = {}
|
||||
for d in yaml.safe_load_all(CV.read_text()):
|
||||
if isinstance(d, dict) and d.get("kind") == "ConfigMap" \
|
||||
@@ -393,16 +282,12 @@ for d in yaml.safe_load_all(CV.read_text()):
|
||||
check(len(base) > 5, f"clusters/base/cluster-vars.yaml yielded {len(base)} knob(s) — the "
|
||||
"ConfigMap parse failed, so every value assertion below would be vacuous")
|
||||
|
||||
# Effective values per overlay, and WHICH knobs that overlay states explicitly. The second
|
||||
# half is what lets emptiness be judged as authored-or-forgotten instead of guessed.
|
||||
overlays: dict[str, dict[str, str]] = {}
|
||||
stated: dict[str, set[str]] = {}
|
||||
for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
|
||||
env = overlay.parent.name
|
||||
if env == "base":
|
||||
continue
|
||||
eff = dict(base)
|
||||
explicit: set[str] = set()
|
||||
try:
|
||||
k = yaml.safe_load(overlay.read_text()) or {}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
@@ -423,15 +308,27 @@ for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
|
||||
m = re.match(r"^/data/([A-Za-z0-9_]+)$", str(op.get("path", "")))
|
||||
if m and str(op.get("op", "")) in ("replace", "add"):
|
||||
eff[m.group(1)] = "" if op.get("value") is None else str(op["value"])
|
||||
explicit.add(m.group(1))
|
||||
overlays[env] = eff
|
||||
stated[env] = explicit
|
||||
check(len(overlays) >= 1,
|
||||
"no clusters/<env>/kustomization.yaml overlay was parsed — assertion (A1) would be vacuous")
|
||||
"no clusters/<env>/kustomization.yaml overlay was parsed — assertion (2) would be vacuous")
|
||||
|
||||
# ------------------------------------------------ which knobs is a guard responsible for?
|
||||
# Function bodies of standup.sh, then a one-hop taint trace from `x="$(cvar KEY)"` to a
|
||||
# `*_is_placeholder "${x}"` call. Both halves are read out of the file; nothing is named here.
|
||||
FUNC_RE = re.compile(r"^([a-z0-9_]+)\(\)\s*\{")
|
||||
funcs: dict[str, list[str]] = {}
|
||||
i = 0
|
||||
while i < len(LINES):
|
||||
m = FUNC_RE.match(LINES[i])
|
||||
if m:
|
||||
end = next((j for j in range(i + 1, len(LINES)) if LINES[j] == "}"), len(LINES) - 1)
|
||||
funcs[m.group(1)] = LINES[i + 1:end]
|
||||
i = end + 1
|
||||
continue
|
||||
i += 1
|
||||
check(len(funcs) > 5, f"parsed only {len(funcs)} function(s) out of {STANDUP_REL} — the "
|
||||
"consumer assertion (3) would be vacuous")
|
||||
|
||||
# ------------------------------------------ which knobs is a guard responsible for? (HALF B)
|
||||
# A one-hop taint trace from `x="$(cvar KEY)"` to a `*_is_placeholder "${x}"` call. Both
|
||||
# halves are read out of the file; nothing is named here.
|
||||
CVAR_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)=\"\$\(\s*cvar\s+([A-Z0-9_]+)\s*\)\"")
|
||||
ALIAS_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)=\"\$\{([A-Za-z_][A-Za-z0-9_]*)[:}]")
|
||||
|
||||
@@ -450,6 +347,8 @@ for fname, body in funcs.items():
|
||||
keys_of_var.setdefault(m.group(1), set()).update(src)
|
||||
called_on = {m.group(2) for m in CALL_RE.finditer(text)}
|
||||
called_vars = {re.sub(r"^\$\{?|\}?$", "", a) for a in called_on}
|
||||
# Aggregate per (function, knob): the knob is guarded HERE if ANY variable carrying its
|
||||
# value is passed to a predicate — `live="$(cvar K)"; ep="${live}"; guard "${ep}"` counts.
|
||||
guarded_here: dict[str, bool] = {}
|
||||
for var, keys in keys_of_var.items():
|
||||
for key in keys:
|
||||
@@ -459,119 +358,43 @@ for fname, body in funcs.items():
|
||||
if g:
|
||||
guarded_keys.add(key)
|
||||
|
||||
if GUARD_REL and not guarded_keys:
|
||||
bail(f"no cluster-vars knob in {GUARD_REL} is passed to a *_is_placeholder predicate. "
|
||||
if not guarded_keys:
|
||||
bail(f"no cluster-vars knob in {STANDUP_REL} is passed to a *_is_placeholder predicate. "
|
||||
"Either the guards were removed, or the way the script reads cluster-vars changed and "
|
||||
"this check's derivation must be updated in the SAME change — it must never silently "
|
||||
"verify an empty set.")
|
||||
|
||||
# (B2) CONSUMERS: every reader of a guarded knob must apply the guard to it.
|
||||
# (3) CONSUMERS: every reader of a guarded knob must apply the guard to it.
|
||||
for fname, key, guarded in sorted(consumers):
|
||||
if key not in guarded_keys:
|
||||
continue # nothing claims responsibility for this knob at all
|
||||
check(guarded,
|
||||
f"{GUARD_REL}: {fname}() reads cluster-vars knob {key} but never passes it to a "
|
||||
f"{STANDUP_REL}: {fname}() reads cluster-vars knob {key} but never passes it to a "
|
||||
"*_is_placeholder guard, while other functions do. On 2026-09-05 all three consumers "
|
||||
"of S3_BACKUP_ENDPOINT were disarmed at once; a consumer that reads the knob and skips "
|
||||
"the guard is that failure in one function.")
|
||||
|
||||
# ------------------------------------------------------------------ independent detector
|
||||
# Deliberately NOT derived from the guard alone: SEED is universal loud-placeholder
|
||||
# vocabulary, and the guard's own words are added so that a guard whose ANCHORING is wrong
|
||||
# still has its vocabulary tested against real values.
|
||||
# ---------------------------------------------------------------- independent detector
|
||||
# Deliberately NOT derived from the guard alone (see the header): SEED is universal
|
||||
# loud-placeholder vocabulary, and the guard's own words are added so that a guard whose
|
||||
# ANCHORING is wrong still has its vocabulary tested against real values.
|
||||
SEED = ["TBD", "TODO", "FIXME", "PLACEHOLDER", "REPLACE-ME", "REPLACEME",
|
||||
"CHANGE-ME", "CHANGEME", "FILL-ME", "FILLME", "SET-ME", "XXXX"]
|
||||
all_words = sorted({w.upper() for ws in guard_words.values() for w in ws if w})
|
||||
VOCAB = sorted({w.upper() for w in SEED} | set(all_words))
|
||||
ANGLE = re.compile(r"<[^>\s]+>")
|
||||
|
||||
|
||||
def loud(value: str) -> bool:
|
||||
"""A positive statement that the value was never decided. The EMPTY STRING IS NOT LOUD —
|
||||
see the header: cluster-vars authors "" deliberately, and treating it as a placeholder is
|
||||
the false positive that made the original check red against its own input."""
|
||||
def placeholder_shaped(value: str) -> bool:
|
||||
if value.strip() == "":
|
||||
return True # an unset endpoint is not a target
|
||||
up = value.upper()
|
||||
return any(w in up for w in VOCAB) or bool(ANGLE.search(value))
|
||||
|
||||
|
||||
def guard_shaped(value: str) -> bool:
|
||||
"""What a GUARD must refuse: loud, or empty. Empty counts here only because a `""`
|
||||
case-arm is the guard author's declaration that empty is not a usable target."""
|
||||
return value.strip() == "" or loud(value)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- (A4) DETECTOR CANARIES
|
||||
# The detector is now the thing doing the refusing in HALF A, so it gets the same
|
||||
# no-op interrogation the guard does. These strings are the real ones from the incident.
|
||||
check(loud("https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com"),
|
||||
"the detector does not recognise the exact 2026-09-05 placeholder "
|
||||
"'https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com' — it is prefix-anchored, which is "
|
||||
"the defect this check exists to catch, now in the check itself")
|
||||
check(loud("TBD-PROD-BULK-NFS-SERVER"),
|
||||
"the detector does not recognise the repo's own TBD-PROD-* sentinel convention "
|
||||
"(clusters/README.md) — assertion (A1) would pass over the very values it is about")
|
||||
check(loud("<your-account-id>"),
|
||||
"the detector does not recognise an <angle-bracket> template as a placeholder")
|
||||
check(not loud("https://0123456789abcdef0123456789abcd.r2.cloudflarestorage.com"),
|
||||
"the detector calls a REAL Cloudflare R2 endpoint a placeholder — it would false-block "
|
||||
"every prod backup target")
|
||||
check(not loud("http://192.168.1.9:9000"),
|
||||
"the detector calls a REAL MinIO endpoint a placeholder — it would false-block dev")
|
||||
check(not loud(""),
|
||||
"the detector calls the EMPTY STRING loud. cluster-vars authors \"\" deliberately for "
|
||||
"suffix/prefix knobs and documents it as load-bearing; emptiness is held to the "
|
||||
"explicitness rule (A2) instead. This exact false positive made the original check FAIL "
|
||||
"2 of 34 assertions against the real guard.")
|
||||
|
||||
# --------------------------------- (A1) no overlay ships a LOUD value nothing refuses
|
||||
a1_rows = 0
|
||||
for env, eff in sorted(overlays.items()):
|
||||
for key, value in sorted(eff.items()):
|
||||
if key in guarded_keys:
|
||||
continue # a guard owns this knob: HALF B (B1) asserts it refuses it
|
||||
a1_rows += 1
|
||||
check(not loud(value),
|
||||
f"clusters/{env}: knob {key} resolves to the placeholder-shaped value {value!r}, "
|
||||
"and nothing refuses it"
|
||||
+ (f" — no guard in {GUARD_REL} covers this knob." if GUARD_REL
|
||||
else " — and no placeholder guard exists in the tree at all.")
|
||||
+ " Either give it a real value before it reaches an overlay, or guard it the way "
|
||||
"the S3 endpoint is guarded. An overlay that ships a placeholder no guard "
|
||||
"recognises is the 2026-09-05 failure with a different knob's name on it: it "
|
||||
"applies clean, reports Ready, and the thing it configures does not exist.")
|
||||
|
||||
# ------------------------- (A2) an empty effective value must be STATED, never inherited
|
||||
a2_rows = 0
|
||||
for env, eff in sorted(overlays.items()):
|
||||
for key, value in sorted(eff.items()):
|
||||
if value.strip() != "":
|
||||
continue
|
||||
a2_rows += 1
|
||||
check(key in stated[env],
|
||||
f"clusters/{env}: knob {key} resolves to the EMPTY STRING by INHERITANCE from "
|
||||
"clusters/base — the overlay never states it. An empty value can be correct (dev's "
|
||||
"BACKUP_CREDENTIALS_SUFFIX and VELERO_BACKUP_PREFIX are authored empty on purpose), "
|
||||
"but only when the overlay SAYS SO: cluster-vars' own convention is that both "
|
||||
"overlays state such a knob explicitly so neither inherits it silently. An empty "
|
||||
"value nobody stated is indistinguishable from a knob somebody forgot.")
|
||||
|
||||
# ---------------- (A3) a LOUD base sentinel must be overridden EXPLICITLY by every overlay
|
||||
a3_rows = 0
|
||||
for key, value in sorted(base.items()):
|
||||
if not loud(value):
|
||||
continue
|
||||
for env in sorted(overlays):
|
||||
a3_rows += 1
|
||||
check(key in stated[env],
|
||||
f"clusters/base/cluster-vars.yaml authors the LOUD sentinel {key}={value!r}, and "
|
||||
f"clusters/{env} does not state that knob — so that overlay INHERITS the sentinel. "
|
||||
"The TBD-PROD-* convention only works if a sentinel cannot be inherited by "
|
||||
"omission; an omitted op is how prod silently ships a value nobody decided.")
|
||||
|
||||
# --------------------------------------- HALF B corpus: authored values + anchoring + canaries
|
||||
rows: list[tuple[str, str, str, str]] = [] # guard, expect, origin, value
|
||||
GUARD = sorted(guard_words)[0] if len(guard_words) == 1 else None
|
||||
|
||||
# (B1) VALUES: base default and every overlay's value, for every guarded knob.
|
||||
# (1) VALUES: base default and every overlay's value, for every guarded knob.
|
||||
for key in sorted(guarded_keys):
|
||||
seen: list[tuple[str, str]] = []
|
||||
if key in base:
|
||||
@@ -581,12 +404,24 @@ for key in sorted(guarded_keys):
|
||||
check(bool(seen), f"guarded knob {key} is not defined in cluster-vars at all — the guard "
|
||||
"protects a knob nothing authors")
|
||||
for origin, value in seen:
|
||||
expect = "placeholder" if guard_shaped(value) else "usable"
|
||||
expect = "placeholder" if placeholder_shaped(value) else "usable"
|
||||
for name in sorted(guard_words):
|
||||
rows.append((name, expect, origin, value))
|
||||
|
||||
# (B3) ANCHORING: the guard's own vocabulary must match ANYWHERE, not just as a prefix,
|
||||
# and a realistic endpoint must still be accepted.
|
||||
# (2) OVERLAYS: a placeholder-shaped effective value for an UNGUARDED knob.
|
||||
for env, eff in sorted(overlays.items()):
|
||||
for key, value in sorted(eff.items()):
|
||||
if key in guarded_keys or not placeholder_shaped(value):
|
||||
continue
|
||||
check(False,
|
||||
f"clusters/{env}: knob {key} resolves to the placeholder-shaped value {value!r}, "
|
||||
f"and NO guard in {STANDUP_REL} refuses it. Either give it a real value before it "
|
||||
"reaches an overlay, or guard it the way the S3 endpoint is guarded — an overlay "
|
||||
"that ships a placeholder no guard recognises is the 2026-09-05 failure with a "
|
||||
"different knob's name on it.")
|
||||
|
||||
# (4) ANCHORING: the guard's own vocabulary must match ANYWHERE, not just as a prefix,
|
||||
# and a realistic endpoint must still be accepted.
|
||||
for name, words in sorted(guard_words.items()):
|
||||
for w in sorted(set(words)):
|
||||
rows.append((name, "placeholder",
|
||||
@@ -601,16 +436,15 @@ for name, words in sorted(guard_words.items()):
|
||||
|
||||
# Every guarded knob must reach the oracle once per authored source (base + each overlay),
|
||||
# so a knob silently dropping out of the corpus is a failure rather than a quiet pass.
|
||||
if GUARD_REL:
|
||||
authored = [r for r in rows if not r[2].startswith(("anchoring:", "canary:"))]
|
||||
expected_rows = len(guarded_keys) * (1 + len(overlays)) * len(guard_words)
|
||||
check(len(authored) == expected_rows,
|
||||
f"{len(authored)} authored value(s) reached the guard, expected {expected_rows} "
|
||||
f"({len(guarded_keys)} guarded knob(s) x base+{len(overlays)} overlay(s) x "
|
||||
f"{len(guard_words)} predicate(s)) — a guarded knob is missing from cluster-vars or "
|
||||
"from an overlay's effective values, and the coupling is only partly exercised")
|
||||
authored = [r for r in rows if not r[2].startswith(("anchoring:", "canary:"))]
|
||||
expected_rows = len(guarded_keys) * (1 + len(overlays)) * len(guard_words)
|
||||
check(len(authored) == expected_rows,
|
||||
f"{len(authored)} authored value(s) reached the guard, expected {expected_rows} "
|
||||
f"({len(guarded_keys)} guarded knob(s) x base+{len(overlays)} overlay(s) x "
|
||||
f"{len(guard_words)} predicate(s)) — a guarded knob is missing from cluster-vars or from "
|
||||
"an overlay's effective values, and the coupling is only partly exercised")
|
||||
|
||||
# --------------------------------------------------- (A5) the check is load-bearing
|
||||
# ------------------------------------------------------- (6) the check is load-bearing
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
check("--layer l0" in L0,
|
||||
@@ -618,14 +452,6 @@ check("--layer l0" in L0,
|
||||
check(bool(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M)) and bool(re.search(r"fail\).*BLOCK=1", GATE)),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
|
||||
# ------------------------------------------------------- non-vacuity floor for HALF A
|
||||
# HALF A is the half that runs unconditionally, so it carries its own floor: if the
|
||||
# authored-value sweep shrinks to nothing, a pass here means nothing.
|
||||
check(a1_rows >= 20,
|
||||
f"only {a1_rows} authored overlay value(s) were audited by (A1) — the sweep over "
|
||||
"clusters/<env> effective cluster-vars has collapsed, and a pass would be meaningless. "
|
||||
"Expected roughly (knobs x overlays); check the ConfigMap and overlay parse above.")
|
||||
|
||||
# --------------------------------------------------------------------------- handoff
|
||||
with (WORK / "corpus.tsv").open("w") as fh:
|
||||
for name, expect, origin, value in rows:
|
||||
@@ -633,31 +459,17 @@ with (WORK / "corpus.tsv").open("w") as fh:
|
||||
(WORK / "findings.txt").write_text("".join(f"{f}\n" for f in FAILURES))
|
||||
(WORK / "counts.txt").write_text(f"{CHECKS}\n")
|
||||
|
||||
if GUARD_REL:
|
||||
print(f"guard file: {GUARD_REL} predicates: {', '.join(sorted(guard_words))}")
|
||||
print(f"guarded knobs: {', '.join(sorted(guarded_keys))} "
|
||||
f"consumers traced: {len(consumers)}")
|
||||
else:
|
||||
print("guard file: NONE FOUND in the tree — HALF B (guard/value coupling) IS NOT PROVEN "
|
||||
"ON THIS TREE. No *_is_placeholder predicate exists to extract: bootstrap/standup.sh "
|
||||
"has never been on main (no deletion commit exists; it lives on unmerged prod-standup "
|
||||
"branches). HALF A below runs against the real authored values and is what currently "
|
||||
"prevents the 2026-09-05 outcome. When a guard lands ANYWHERE in the tree, discovery "
|
||||
"picks it up and HALF B activates with no edit to this check.")
|
||||
print(f"detector vocabulary: {', '.join(VOCAB)}")
|
||||
print(f"HALF A: overlays {', '.join(sorted(overlays))}; {len(base)} base knob(s); "
|
||||
f"(A1) {a1_rows} authored overlay value(s) screened; (A2) {a2_rows} empty value(s) "
|
||||
f"checked for explicit statement; (A3) {a3_rows} base-sentinel inheritance check(s)")
|
||||
print(f"guards: {', '.join(sorted(guard_words))} "
|
||||
f"vocabulary: {', '.join(VOCAB)}")
|
||||
print(f"guarded knobs: {', '.join(sorted(guarded_keys))} "
|
||||
f"consumers traced: {len(consumers)} overlays: {', '.join(sorted(overlays))}")
|
||||
for name, expect, origin, value in rows:
|
||||
if not origin.startswith(("anchoring:", "canary:")):
|
||||
print(f" authored {origin} = {value!r} -> expect {expect}")
|
||||
PY
|
||||
|
||||
GUARD_REL="$(tr -d '\n' < "${WORK}/mode.txt")"
|
||||
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# PHASE 2 (bash): source the REAL predicate lines and run the corpus through them.
|
||||
# Empty when no guard exists — HALF A has already run and carries its own floor.
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# shellcheck disable=SC1090,SC1091
|
||||
source "${WORK}/guards.sh"
|
||||
@@ -671,9 +483,9 @@ while IFS=$'\t' read -r guard expect origin value; do
|
||||
if [ "${verdict}" != "${expect}" ]; then
|
||||
oracle_failures=$((oracle_failures + 1))
|
||||
if [ "${expect}" = placeholder ]; then
|
||||
printf '%s\n' "${guard}() in ${GUARD_REL} ACCEPTS '${value}' (from ${origin}) as a usable value, but it is placeholder-shaped. This is the 2026-09-05 failure exactly: the guard is asked about a value it does not recognise, returns 'fine', and the tier it protects is enabled against something that does not exist. Fix the predicate (match the idiom anywhere, not as a prefix) or stop authoring that idiom." >> "${WORK}/findings.txt"
|
||||
printf '%s\n' "${guard}() in ${STANDUP_REL} ACCEPTS '${value}' (from ${origin}) as a usable value, but it is placeholder-shaped. This is the 2026-09-05 failure exactly: the guard is asked about a value it does not recognise, returns 'fine', and the tier it protects is enabled against something that does not exist. Fix the predicate (match the idiom anywhere, not as a prefix) or stop authoring that idiom." >> "${WORK}/findings.txt"
|
||||
else
|
||||
printf '%s\n' "${guard}() in ${GUARD_REL} REFUSES '${value}' (from ${origin}), which is a usable value. An over-matching guard leaves the tier it protects suspended forever and reports it as an operator decision — a silent failure in the other direction." >> "${WORK}/findings.txt"
|
||||
printf '%s\n' "${guard}() in ${STANDUP_REL} REFUSES '${value}' (from ${origin}), which is a usable value. An over-matching guard leaves the tier it protects suspended forever and reports it as an operator decision — a silent failure in the other direction." >> "${WORK}/findings.txt"
|
||||
fi
|
||||
fi
|
||||
done < "${WORK}/corpus.tsv"
|
||||
@@ -683,13 +495,11 @@ py_checks="$(cat "${WORK}/counts.txt")"
|
||||
total=$((py_checks + oracle_rows))
|
||||
findings="$(cat "${WORK}/findings.txt")"
|
||||
|
||||
# NO-OP GUARDS: this repo has been bitten by a check that passed when perturbed because it
|
||||
# NO-OP GUARD: this repo has been bitten by a check that passed when perturbed because it
|
||||
# asked a question that was always satisfiable. The floors below are the minimum evidence
|
||||
# for the verdict to mean anything, not a style rule. The oracle floor applies only when a
|
||||
# guard exists — when none does, HALF A's own floor (a1_rows >= 20, asserted in phase 1) is
|
||||
# what stops this from degenerating, and the absence is reported on stdout.
|
||||
if [ -n "${GUARD_REL}" ] && [ "${oracle_rows}" -lt 6 ]; then
|
||||
findings="${findings}only ${oracle_rows} value(s) were put to the real guard in ${GUARD_REL} — the coupling is not actually being exercised
|
||||
# for the verdict to mean anything, not a style rule.
|
||||
if [ "${oracle_rows}" -lt 6 ]; then
|
||||
findings="${findings}only ${oracle_rows} value(s) were put to the real guard — the coupling is not actually being exercised
|
||||
"
|
||||
fi
|
||||
if [ "${total}" -lt 20 ]; then
|
||||
@@ -703,8 +513,4 @@ if [ -n "${findings}" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${GUARD_REL}" ]; then
|
||||
echo "check OK: placeholder-guard-coupling — ${total} assertions. HALF A: no environment overlay ships a placeholder-shaped effective cluster-vars value, every empty value is stated explicitly rather than inherited, and the detector canaries hold (including the exact 2026-09-05 string, matched mid-string). HALF B: ${oracle_rows} value(s) run through the REAL extracted predicate(s) from ${GUARD_REL} (no reimplementation), covering every authored value of every guarded knob in base and each overlay, the guard's vocabulary matched mid-string, and the empty/real-endpoint canaries. Static slice only: refusal, not reachability — see the header."
|
||||
else
|
||||
echo "check OK: placeholder-guard-coupling — ${total} assertions. HALF A ONLY: no environment overlay ships a placeholder-shaped effective cluster-vars value, every empty value is stated explicitly rather than inherited, no LOUD base sentinel is inherited by omission, and the detector canaries hold (including the exact 2026-09-05 string 'https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com', matched mid-string). HALF B (guard/value coupling) IS NOT PROVEN: no *_is_placeholder predicate exists anywhere in the tree to extract — see the stdout note above and the header. This is a REPORTED gap, not a skip: what a guard would be asked about is proven not to be a placeholder in the first place."
|
||||
fi
|
||||
echo "check OK: placeholder-guard-coupling — ${total} assertions; ${oracle_rows} value(s) run through the REAL extracted predicate(s) from ${STANDUP_REL} (no reimplementation), covering every authored cluster-vars value of every guarded knob in base and each overlay, the guard's own vocabulary matched mid-string, and the empty/real-endpoint canaries. Static slice only: refusal, not reachability — see the header."
|
||||
@@ -58,12 +58,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. Universally quantified over the tracked tree (`git ls-files`): its
|
||||
# core assertion is that NO dev-only surface leaks into what prod renders, over
|
||||
# platform/, platform-config/, apps/, catalog/ and clusters/base/ together, plus infra/
|
||||
# (which the path map keys to nothing at all). No single requirement can select the set
|
||||
# of files that can break it.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -48,10 +48,7 @@
|
||||
# sync.yaml, every patch it applies targets an ENVIRONMENT-SCOPED object
|
||||
# (cluster-vars, a Flux Kustomization's sops decryption, or the Git source),
|
||||
# prod's Git ref is the SAME ref dev tracks (no prod-only branch to develop on),
|
||||
# and no prod-only manifest exists anywhere in the rendered configuration — a file
|
||||
# whose NAME carries the environment token is excused ONLY when the tree proves it
|
||||
# is an OVERLAY-SELECTED half that every environment applies (see the four
|
||||
# conditions at "THE ONE STRUCTURAL EXCEPTION" below); anything else still fails;
|
||||
# and no prod-only manifest exists anywhere in the rendered configuration;
|
||||
# (6) prod artifacts exist only WITH a declaration of readiness: the spec roadmap
|
||||
# carries the phase that opens prod build (checked when the spec repo is mounted),
|
||||
# and the repository records that the operator opened it.
|
||||
@@ -440,172 +437,12 @@ tracked = [f for f in subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"],
|
||||
capture_output=True, text=True, check=True
|
||||
).stdout.split("\0") if f]
|
||||
RENDERED_DIRS = ("platform/", "platform-config", "apps/", "catalog/", "clusters/base/")
|
||||
|
||||
# THE ONE STRUCTURAL EXCEPTION: AN OVERLAY-SELECTED ENVIRONMENT HALF.
|
||||
# ------------------------------------------------------------------
|
||||
# A filename carrying an environment token is a HEURISTIC for the harm R-ENV-03 names
|
||||
# (content only prod runs, in a path every environment renders). It is not the harm
|
||||
# itself. There is exactly one shape where a `-prod` name is the OPPOSITE of a prod-only
|
||||
# fork: a credential that is TWO Secret objects, BOTH applied to BOTH clusters as base
|
||||
# resources, where a cluster-vars knob decides only which one the shared consumer
|
||||
# REFERENCES (clusters/base/cluster-vars.yaml, BACKUP_CREDENTIALS_SUFFIX: dev backs up to
|
||||
# MinIO, prod to Cloudflare R2, and one shared Secret meant either environment's rotation
|
||||
# broke the other). Nothing is forked: every environment renders the identical manifests
|
||||
# and the identical object set, which is precisely the R-ENV-02 contract.
|
||||
#
|
||||
# So the exception is not a path allowlist — it is the PROPERTY that makes that shape
|
||||
# safe, asserted from the tree, and a file is excused ONLY when all of it holds:
|
||||
#
|
||||
# (A) it declares nothing but Secrets whose names end in the environment suffix — the
|
||||
# only kind of object this repo has a selection mechanism for;
|
||||
# (1) it is listed in its own directory's kustomization.yaml, so EVERY environment
|
||||
# applies it, and every environment's value of the selector resolves to an object
|
||||
# that some resource of that same kustomization declares (same namespace) — no
|
||||
# environment is left selecting a Secret that does not exist;
|
||||
# (2) nothing references the suffixed name literally: every reference is a substituted
|
||||
# expression `<base-name>${VAR}`, and the suffixed FILE is named only by the
|
||||
# kustomization that applies it — so no shared manifest hardcodes prod's half;
|
||||
# (3) that VAR is defined for the base AND for every overlay (StrictPostBuildSubstitutions
|
||||
# fails on an UNDEFINED variable, so an overlay that forgot it would break that
|
||||
# cluster's reconciliation), and prod's value is the one that selects THIS file.
|
||||
#
|
||||
# References are read from PARSED YAML VALUES, never from raw text: the mechanism is
|
||||
# documented by name in comments beside the resources (cluster-vars says so explicitly),
|
||||
# and a name in a comment references nothing. Anything that fails any condition still
|
||||
# fails, naming the condition it broke.
|
||||
ENV_SUFFIX = f"-{prod_token}"
|
||||
# Every environment that renders the shared configuration.
|
||||
ENVS = ["base"] + sorted(p.parent.name
|
||||
for p in (ROOT / "clusters").glob("*/kustomization.yaml")
|
||||
if p.parent.name != "base")
|
||||
|
||||
|
||||
def _docs(rel: str):
|
||||
p = ROOT / rel
|
||||
if not p.is_file():
|
||||
return None
|
||||
try:
|
||||
return [d for d in yaml.safe_load_all(p.read_text(errors="replace"))
|
||||
if isinstance(d, dict)]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _strings(node):
|
||||
if isinstance(node, str):
|
||||
yield node
|
||||
elif isinstance(node, dict):
|
||||
for k, v in node.items():
|
||||
yield from _strings(k)
|
||||
yield from _strings(v)
|
||||
elif isinstance(node, (list, tuple)):
|
||||
for v in node:
|
||||
yield from _strings(v)
|
||||
|
||||
|
||||
RENDERED_YAML = [f for f in tracked
|
||||
if f.startswith(RENDERED_DIRS) and f.endswith((".yaml", ".yml"))]
|
||||
VALUE_STRINGS = {f: list(_strings(_docs(f) or [])) for f in RENDERED_YAML}
|
||||
|
||||
|
||||
def overlay_selected(f: str) -> list[str]:
|
||||
"""[] if f is an overlay-SELECTED environment half; else the conditions it breaks."""
|
||||
docs = _docs(f)
|
||||
objs = [((d.get("kind")), (d.get("metadata") or {}).get("name"),
|
||||
(d.get("metadata") or {}).get("namespace")) for d in (docs or [])]
|
||||
if not objs or any(k != "Secret" or not isinstance(n, str) or not n.endswith(ENV_SUFFIX)
|
||||
for k, n, _ in objs):
|
||||
return [f"(A) it does not consist solely of Secret objects whose names end in "
|
||||
f"{ENV_SUFFIX!r} — an overlay can select which SECRET a shared manifest "
|
||||
f"references, and nothing else, so any other prod-suffixed object in a "
|
||||
f"shared path is prod-only content"]
|
||||
|
||||
reasons: list[str] = []
|
||||
d = str(Path(f).parent)
|
||||
kfile = f"{d}/kustomization.yaml"
|
||||
kdocs = _docs(kfile) or []
|
||||
res = [str(r) for r in ((kdocs[0].get("resources") if kdocs else None) or [])]
|
||||
if Path(f).name not in res:
|
||||
reasons.append(f"(1) {kfile} does not list it in `resources`, so it is not applied "
|
||||
f"in every environment — one environment would carry an object the "
|
||||
f"other does not")
|
||||
|
||||
# Every Secret the applying kustomization declares: name -> namespace.
|
||||
declared: dict[str, object] = {}
|
||||
for r in res:
|
||||
for dd in (_docs(f"{d}/{r}") or []):
|
||||
if dd.get("kind") == "Secret":
|
||||
md = dd.get("metadata") or {}
|
||||
declared[str(md.get("name"))] = md.get("namespace")
|
||||
|
||||
for _, name, ns in objs:
|
||||
base_name = name[:-len(ENV_SUFFIX)]
|
||||
pat = re.escape(base_name) + r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}"
|
||||
sel = sorted({m for ss in VALUE_STRINGS.values() for s in ss
|
||||
for m in re.findall(pat, s)})
|
||||
# (2) literal references to the suffixed OBJECT — anywhere but this file, except
|
||||
# the applying kustomization naming this FILE (which is how it gets applied).
|
||||
lit = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss
|
||||
if name in s and f2 != f
|
||||
and not (f2 == kfile and s == Path(f).name)})
|
||||
if lit:
|
||||
reasons.append(f"(2) {', '.join(lit)} names the Secret {name!r} literally — a "
|
||||
f"shared manifest that hardcodes prod's half makes every "
|
||||
f"environment use prod's credential; reference "
|
||||
f"{base_name}${{...}} instead")
|
||||
# (2) literal references to the suffixed FILE from anywhere but its kustomization.
|
||||
stray = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss
|
||||
if Path(f).name in s and f2 not in (f, kfile)})
|
||||
if stray:
|
||||
reasons.append(f"(2) {', '.join(stray)} names the file {Path(f).name!r} "
|
||||
f"literally — only {kfile} may apply it")
|
||||
if not sel:
|
||||
reasons.append(f"(2) no manifest references {base_name}${{...}}: nothing selects "
|
||||
f"between the halves, so this file is prod-only content that "
|
||||
f"only prod could ever consume")
|
||||
continue
|
||||
if len(sel) > 1:
|
||||
reasons.append(f"(2) {base_name} is selected by more than one variable {sel} — "
|
||||
f"they can drift apart and leave one consumer on the other "
|
||||
f"environment's credential")
|
||||
var = sel[0]
|
||||
vals = env_values(var)
|
||||
missing = [e for e in ENVS if e not in vals]
|
||||
if missing:
|
||||
reasons.append(f"(3) ${{{var}}} is not defined for {missing} (defined for "
|
||||
f"{sorted(vals)}) — StrictPostBuildSubstitutions fails on an "
|
||||
f"undefined variable, so that environment would stop "
|
||||
f"reconciling, or silently inherit the other's credential")
|
||||
for e, v in sorted(vals.items()):
|
||||
want = base_name + v
|
||||
if want not in declared:
|
||||
reasons.append(f"(1) environment {e!r} selects Secret {want!r} via "
|
||||
f"${{{var}}}={v!r}, which no resource of {kfile} declares — "
|
||||
f"that environment renders a dangling reference")
|
||||
elif declared[want] != ns:
|
||||
reasons.append(f"(1) environment {e!r} selects Secret {want!r}, declared in "
|
||||
f"namespace {declared[want]!r}, but this half is in {ns!r} — "
|
||||
f"the halves are not interchangeable")
|
||||
if prod_overlay and prod_overlay in vals and base_name + vals[prod_overlay] != name:
|
||||
reasons.append(f"(3) the prod overlay's ${{{var}}}={vals[prod_overlay]!r} selects "
|
||||
f"{base_name + vals[prod_overlay]!r}, not this file's {name!r} — "
|
||||
f"prod's half is applied but no environment selects it")
|
||||
return reasons
|
||||
|
||||
|
||||
flagged = [f for f in tracked
|
||||
if f.startswith(RENDERED_DIRS)
|
||||
and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)", f, re.I)]
|
||||
for f in flagged:
|
||||
why = overlay_selected(f)
|
||||
check(not why,
|
||||
f"{f} is a prod-specific file inside the shared configuration — every environment "
|
||||
f"renders these paths, so prod-only content here is development against prod. It is "
|
||||
f"not an overlay-selected environment half either: " + " AND ".join(why))
|
||||
if not why:
|
||||
note(f"{f} carries the {prod_token!r} token but is an OVERLAY-SELECTED half: both "
|
||||
f"halves are applied in every environment and only the cluster-vars selector "
|
||||
f"differs (R-ENV-02), so it is not prod-only content")
|
||||
for f in tracked:
|
||||
if f.startswith(RENDERED_DIRS) and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)",
|
||||
f, re.I):
|
||||
check(False,
|
||||
f"{f} is a prod-specific file inside the shared configuration — every environment "
|
||||
"renders these paths, so prod-only content here is development against prod")
|
||||
check(len(tracked) > 50, f"git ls-files returned only {len(tracked)} paths — not a real checkout")
|
||||
|
||||
# --------------------------- (6) prod work exists only with a declaration of readiness
|
||||
|
||||
@@ -30,14 +30,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-TEST-06"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# NO `scope` ANNOTATION, DELIBERATELY — this invariant IS path-scoped, and was audited as
|
||||
# such (2026-09-12). Its entire input surface is ci/ (spec-immutability.sh, SPEC_PIN,
|
||||
# gate.sh) plus the read-only spec repo, which no commit here can change. The gap was that
|
||||
# ci/touched-requirements.sh keyed .gitea/*|ci/* to R-GITOPS-01 + R-TEST-02 + R-TEST-03 but
|
||||
# NOT to R-TEST-06 — a leftover from when R-TEST-06 was enforced only by the gate STAGE and
|
||||
# carried by no test, so attributing it false-blocked the coverage gate. This test now
|
||||
# carries the R-TEST-06 label, so R-TEST-06 was added to that path-map entry in the same
|
||||
# change; a path-scoped invariant belongs in the path map, not in an unconditional run.
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -50,13 +50,6 @@ metadata:
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-TLS-02"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE. Universally quantified over tracked YAML: it inventories every
|
||||
# cert-manager Certificate/Issuer and every Gateway TLS listener wherever they are
|
||||
# authored, and cross-checks each against the issuers and cluster-vars knobs that exist.
|
||||
# A new HTTPRoute/Gateway or a hand-rolled Certificate under apps/ or platform/<x>/
|
||||
# attributes no requirement, so requirement-scoped selection would skip the check that
|
||||
# would have caught a listener whose secret nothing ever issues.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — every WHOLE-REPO invariant is actually wired into the gate (R-TEST-03) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# WHAT THIS REPLACES, AND WHY IT HAS TO EXIST
|
||||
# ------------------------------------------------------------------------------
|
||||
# R-TEST-03 gates a merge on the conformance subset the change TOUCHES. That selection has one
|
||||
# blind spot, and it is the worst-shaped kind: for an invariant that quantifies over the WHOLE
|
||||
# tracked tree (or over subtrees the coarse path->requirement map keys to NOTHING — apps/,
|
||||
# infra/, most of platform/), the change most likely to break it attributes no requirement at
|
||||
# all, so requirement-scoped selection filters the check out of EXACTLY that change.
|
||||
#
|
||||
# The repo's first answer was a hand-written "UNCONDITIONAL" stage per such check in ci/l0.sh
|
||||
# (3c subst-safety, 3d keycloak-client-secrets, 3e bulk-tier), and each of those three checks
|
||||
# asserted its OWN wiring by grepping ci/l0.sh for its own check.sh path. That pattern did not
|
||||
# scale — and it failed SILENTLY in the worst way, because whoever added the NEXT whole-repo
|
||||
# invariant had to REMEMBER to hand-write a stage, and if they forgot, the check ran only when
|
||||
# some unrelated requirement happened to be touched.
|
||||
#
|
||||
# The stages are gone. A test now DECLARES `conformance.home.k3s/scope: "whole-repo"` and the
|
||||
# runner unions every declaring test into the selection (`run.sh --include-scope whole-repo`).
|
||||
# This test is what keeps that declarative mechanism honest, for ALL declaring tests at once
|
||||
# instead of three of them separately:
|
||||
#
|
||||
# (1) the mechanism EXISTS end to end — lib/common.sh resolves the annotation, run.sh's
|
||||
# --include-scope unions it into select_dirs, and ci/l0.sh passes it;
|
||||
# (2) it WORKS, proven empirically and falsifiably: with a requirement filter that matches
|
||||
# NO test, `run.sh --layer l0 --print-selection --include-scope whole-repo` must still
|
||||
# resolve to every whole-repo-scoped L0 test — and WITHOUT the flag, to none of them;
|
||||
# (3) a SKIP of a scope-declared check is UNWIRED, NOT GREEN — proven by running the offline
|
||||
# runner against synthetic checks and asserting exit 3 for the declared one and exit 0
|
||||
# for the undeclared one. This is the rc-3 semantics the deleted stages encoded
|
||||
# ("check SKIPPED (clusters/ absent) — not a pass"), which is the half most likely to
|
||||
# be lost in a refactor because nothing else notices when it disappears;
|
||||
# (4) the bespoke pattern has not crept back: a ci/*.sh that hardcodes a test's check.sh path
|
||||
# must at least belong to a test that ALSO declares the scope, so a hand-written stage
|
||||
# can never be the ONLY thing running a whole-repo check;
|
||||
# (5) ci/gate.sh still BLOCKS the merge when the l0 stage fails (the assertion each of the
|
||||
# three deleted stages' checks carried, kept here once);
|
||||
# (6) the declared set is non-empty and every member is executable — a mechanism that
|
||||
# resolves to nothing would pass every other assertion here while checking nothing.
|
||||
#
|
||||
# This test declares whole-repo scope ITSELF, for the obvious reason: an invariant about the
|
||||
# gate's own selection must not be selectable away by the gate's own selection.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: whole-repo-invariants-wired
|
||||
labels:
|
||||
conformance.home.k3s/R-TEST-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-TEST-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE, and necessarily so: this asserts a property OF the merge gate's
|
||||
# requirement-scoped selection. If it could itself be filtered out by that selection, the
|
||||
# first change to break the mechanism would also be the change that stops checking it.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every conformance test that declares `conformance.home.k3s/scope: whole-repo` is
|
||||
unconditionally selected by the merge gate regardless of which requirements a change
|
||||
touches, and a SKIP of such a test is reported as UNWIRED (exit 3), never as green.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-whole-repo-scope-mechanism-is-wired-and-falsifiable
|
||||
description: >-
|
||||
Resolve the gate's own L0 selection with a requirement filter that matches no test
|
||||
and assert every whole-repo-scoped test is still selected; then prove a scope-declared
|
||||
SKIP exits 3 while an undeclared SKIP exits 0.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,208 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 STATIC CHECK — the whole-repo scope mechanism is wired, works, and stays honest.
|
||||
# See chainsaw-test.yaml's header for WHY this test exists and what it replaced.
|
||||
#
|
||||
# Exit 0 = all assertions pass. Exit 1 = a failure. This check has NO exit-3 path: the
|
||||
# mechanism it audits is source in this repo, so "nothing to audit" is not a state it can
|
||||
# legitimately be in.
|
||||
#
|
||||
# IT MUST NEVER EXECUTE THE SUITE IT IS PART OF. `run.sh --print-selection` resolves the
|
||||
# selection and exits without running a single check, which is what makes assertion (2)
|
||||
# possible without infinite recursion.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
CONF="${ROOT}/conformance"
|
||||
SCOPE="${WHOLE_REPO_SCOPE:-whole-repo}"
|
||||
PASS=0
|
||||
FAILED=0
|
||||
|
||||
note() { printf ' ok %s\n' "$1"; PASS=$((PASS+1)); }
|
||||
bad() { printf ' FAIL %s\n' "$1" >&2; FAILED=$((FAILED+1)); }
|
||||
chk() { if [ "$1" = 0 ]; then note "$2"; else bad "$3"; fi }
|
||||
|
||||
L0_SH="${ROOT}/ci/l0.sh"
|
||||
GATE_SH="${ROOT}/ci/gate.sh"
|
||||
RUN_SH="${CONF}/run.sh"
|
||||
COMMON_SH="${CONF}/lib/common.sh"
|
||||
OFFLINE_PY="${CONF}/lib/run_l1_offline.py"
|
||||
|
||||
# ---- (1) the mechanism exists, end to end -----------------------------------------------
|
||||
for f in "$L0_SH" "$GATE_SH" "$RUN_SH" "$COMMON_SH" "$OFFLINE_PY"; do
|
||||
[ -f "$f" ] || { bad "missing ${f#"$ROOT"/} — the mechanism cannot be audited"; }
|
||||
done
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo "CHECK FAIL (whole-repo-invariants-wired): the gate's own sources are missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -q 'CONF_SCOPE_ANNOTATION' "$COMMON_SH" \
|
||||
&& grep -q 'conf_dirs_for_scope' "$COMMON_SH"
|
||||
chk $? "conformance/lib/common.sh resolves the scope annotation (conf_dirs_for_scope)" \
|
||||
"conformance/lib/common.sh no longer resolves the scope annotation — no test can be
|
||||
selected BY declaring a scope, so every whole-repo invariant is back to being
|
||||
selectable away by the requirement filter"
|
||||
|
||||
grep -q -- '--include-scope' "$RUN_SH" && grep -q 'conf_dirs_for_scopes' "$RUN_SH"
|
||||
chk $? "conformance/run.sh --include-scope unions scope-declared tests into select_dirs" \
|
||||
"conformance/run.sh does not union scope-declared tests (--include-scope /
|
||||
conf_dirs_for_scopes missing) — the annotation would be inert decoration"
|
||||
|
||||
grep -Eq -- '--include-scope[[:space:]]+"?\$\{?WHOLE_REPO_SCOPE' "$L0_SH" \
|
||||
|| grep -Eq -- "--include-scope[[:space:]]+['\"]?${SCOPE}" "$L0_SH"
|
||||
chk $? "ci/l0.sh passes --include-scope ${SCOPE} to the conformance runner" \
|
||||
"ci/l0.sh does not pass --include-scope ${SCOPE}: the L0 merge-gate stage is back to a
|
||||
purely requirement-scoped run, and every whole-repo invariant is filtered out of
|
||||
exactly the change that breaks it"
|
||||
|
||||
# ---- (6) the declared set is non-empty and executable -----------------------------------
|
||||
declared_dirs=()
|
||||
while IFS= read -r d; do [ -n "$d" ] && declared_dirs+=("$d"); done < <(
|
||||
CONF_ROOT="$CONF" bash -c 'source "${CONF_ROOT}/lib/common.sh"; conf_dirs_for_scope "$1"' _ "$SCOPE"
|
||||
)
|
||||
MIN_DECLARED="${MIN_DECLARED:-5}"
|
||||
[ "${#declared_dirs[@]}" -ge "$MIN_DECLARED" ]
|
||||
chk $? "${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED})" \
|
||||
"only ${#declared_dirs[@]} test(s) declare scope='${SCOPE}' (floor ${MIN_DECLARED}) — either the
|
||||
annotation was stripped from the whole-repo checks, or the resolver stopped matching
|
||||
it. Every assertion below would then pass while checking nothing"
|
||||
|
||||
for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do
|
||||
rel="${d#"$ROOT"/}"
|
||||
[ -f "$d/check.sh" ] || [ -f "$d/render-check.sh" ]
|
||||
chk $? "${rel}: declares the scope AND carries an executable offline check" \
|
||||
"${rel} declares scope='${SCOPE}' but carries neither check.sh nor render-check.sh — an
|
||||
unconditional run of a test with nothing to run is a guaranteed SKIP, i.e. a
|
||||
permanent not-green, not a guard"
|
||||
done
|
||||
|
||||
# ---- (2) IT WORKS: prove the selection cannot filter a declaring test out ---------------
|
||||
# A requirement ID no test carries. With it alone the L0 selection MUST be empty; adding
|
||||
# --include-scope MUST bring back every declaring test. Both halves are asserted, so this is
|
||||
# falsifiable: if --include-scope became a no-op, the first half would still pass and the
|
||||
# second would fail.
|
||||
# Deliberately LOWERCASE: conf_normalize_reqs() upper-cases it, so the selection behaves
|
||||
# identically — but the literal here does not match ci/touched-requirements.sh's content-scan
|
||||
# pattern `R-[A-Z]+-[0-9]+`, so editing this file does not attribute a nonexistent requirement
|
||||
# to the change. (Same reason the synthetic fixture below carries no requirement label at all.)
|
||||
BOGUS_REQ="r-nosuchreq-99"
|
||||
sel_without="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --print-selection 2>/dev/null || true)"
|
||||
sel_with="$(bash "$RUN_SH" --layer l0 --requirements "$BOGUS_REQ" --include-scope "$SCOPE" --print-selection 2>/dev/null || true)"
|
||||
|
||||
[ -z "$sel_without" ]
|
||||
chk $? "a requirement filter matching no test resolves to an EMPTY L0 selection (control)" \
|
||||
"the control case is broken: --requirements ${BOGUS_REQ} still selected tests
|
||||
($(printf '%s' "$sel_without" | tr '\n' ' ')), so the assertion below proves nothing
|
||||
about --include-scope"
|
||||
|
||||
missing=""
|
||||
for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do
|
||||
printf '%s\n' "$sel_with" | grep -qxF "$d" || missing="${missing} ${d#"$ROOT"/}"
|
||||
done
|
||||
[ -z "$missing" ]
|
||||
chk $? "every scope-declared L0 test is selected DESPITE a requirement filter matching none" \
|
||||
"the scope union does not actually select:${missing}. This is the blind spot itself: a
|
||||
change that attributes no requirement would not run these checks"
|
||||
|
||||
# The union must not smuggle in another layer's tests when --layer bounds the run.
|
||||
strays="$(printf '%s\n' "$sel_with" | grep -v '/tests/l0-static/' || true)"
|
||||
[ -z "$strays" ]
|
||||
chk $? "the scope union stays inside --layer l0 (no cross-layer leakage)" \
|
||||
"--layer l0 --include-scope ${SCOPE} selected non-L0 test(s): $(printf '%s' "$strays" | tr '\n' ' ')"
|
||||
|
||||
# ---- (3) a SKIP of a scope-declared check is UNWIRED, NOT GREEN -------------------------
|
||||
# Proven by running the real offline runner over two synthetic test dirs: one DECLARING the
|
||||
# scope, one not. Both checks exit 3. The declaring one must make the runner exit 3; the
|
||||
# other must leave it at 0. This is the half a refactor loses most easily, because nothing
|
||||
# else in the suite notices when "skip" quietly starts meaning "pass".
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "${WORK}"' EXIT
|
||||
mk_synth() { # mk_synth <dir> <declare-scope: yes|no>
|
||||
local d="$1" declare="$2"
|
||||
mkdir -p "$d"
|
||||
{
|
||||
printf 'apiVersion: chainsaw.kyverno.io/v1alpha1\nkind: Test\nmetadata:\n'
|
||||
printf ' name: %s\n annotations:\n' "$(basename "$d")"
|
||||
[ "$declare" = yes ] && printf ' conformance.home.k3s/scope: "%s"\n' "$SCOPE"
|
||||
printf ' conformance.home.k3s/layer: "L0"\nspec: {}\n'
|
||||
} > "$d/chainsaw-test.yaml"
|
||||
printf '#!/usr/bin/env bash\necho "synthetic: nothing to audit" >&2\nexit 3\n' > "$d/check.sh"
|
||||
}
|
||||
mk_synth "${WORK}/l0-static/synth-declared" yes
|
||||
mk_synth "${WORK}/l0-static/synth-undeclared" no
|
||||
|
||||
run_offline() { # run_offline <dir...> -> echoes rc
|
||||
local rc=0
|
||||
python3 "$OFFLINE_PY" --junit "${WORK}/j.xml" \
|
||||
--scope-annotation "conformance.home.k3s/scope" \
|
||||
--unwired-scope "$SCOPE" "$@" >/dev/null 2>&1 || rc=$?
|
||||
echo "$rc"
|
||||
}
|
||||
rc_declared="$(run_offline "${WORK}/l0-static/synth-declared")"
|
||||
rc_undeclared="$(run_offline "${WORK}/l0-static/synth-undeclared")"
|
||||
|
||||
[ "$rc_declared" = 3 ]
|
||||
chk $? "a SKIPPED check that DECLARES scope='${SCOPE}' makes the runner exit 3 (UNWIRED, not green)" \
|
||||
"a SKIPPED scope-declared check made the offline runner exit ${rc_declared}, not 3. A
|
||||
whole-repo invariant reporting 'nothing to audit' means the tree it audits has
|
||||
VANISHED; the deleted ci/l0.sh stages called that 'not a pass' and set the gate's
|
||||
rc=3. Exit 0 here turns that back into a silent green"
|
||||
|
||||
[ "$rc_undeclared" = 0 ]
|
||||
chk $? "a SKIPPED check with NO scope declaration still exits 0 (unchanged for scaffolds)" \
|
||||
"a SKIPPED undeclared check made the runner exit ${rc_undeclared}, not 0 — the strict
|
||||
rule leaked onto ordinary scaffolds, which would make every unfinished test block"
|
||||
|
||||
# ...and ci/l0.sh must map that rc 3 to the GATE's rc 3, never to a pass.
|
||||
l0_static_stage="$(sed -n '/Stage 3b: L0 STATIC/,/^case "\$rc" in/p' "$L0_SH")"
|
||||
printf '%s\n' "$l0_static_stage" | grep -Eq '^[[:space:]]*3\)' \
|
||||
&& printf '%s\n' "$l0_static_stage" | grep -Eq 'rc=3'
|
||||
chk $? "ci/l0.sh maps the runner's rc 3 to the gate's own rc 3 (SKIPPED, not a pass)" \
|
||||
"ci/l0.sh's L0-static stage has no rc-3 arm setting the gate's rc=3, so a scope-declared
|
||||
check that SKIPPED would be reported as a PASS — the exact false green the deleted
|
||||
stages were written to prevent"
|
||||
|
||||
# ---- (4) the bespoke per-check stage pattern has not crept back -------------------------
|
||||
# Not forbidden outright: a ci/*.sh MAY hardcode a test's check.sh. But if it does, that test
|
||||
# must ALSO declare the scope, so a hand-written stage can never be the only thing running a
|
||||
# whole-repo check (which is how the mechanism silently stops covering a new invariant).
|
||||
# Two exclusions, both the repo's existing "reference, not declaration" rule (see
|
||||
# ci/touched-requirements.sh changed_diff() rules 2-3):
|
||||
# * whole-line `#` comments — a path NAMED in prose is documentation, not an invocation;
|
||||
# * ci/tests/ — those files embed example test paths as unit-test FIXTURE DATA.
|
||||
hardcoded="$(find "${ROOT}/ci" -name '*.sh' -not -path '*/ci/tests/*' -print0 2>/dev/null \
|
||||
| xargs -0 -r grep -hE -v '^[[:space:]]*#' \
|
||||
| grep -hoE 'conformance/tests/[a-z0-9-]+/[a-zA-Z0-9._-]+/(check|render-check)\.sh' \
|
||||
| sort -u || true)"
|
||||
for rel in $hardcoded; do
|
||||
tdir="${ROOT}/$(dirname "$rel")"
|
||||
y="${tdir}/chainsaw-test.yaml"
|
||||
if [ -f "$y" ] && grep -Eq "^[[:space:]]*conformance\.home\.k3s/scope:[[:space:]]*[\"']?${SCOPE}" "$y"; then
|
||||
note "ci/ hardcodes ${rel}, and that test also DECLARES the scope (belt and braces, fine)"
|
||||
else
|
||||
bad "ci/ hardcodes ${rel} as a bespoke stage, but that test does NOT declare
|
||||
conformance.home.k3s/scope: \"${SCOPE}\". That is the pattern this mechanism replaced: the
|
||||
stage is then the ONLY thing running the check, it must be hand-maintained, and the
|
||||
runner's own selection still does not know the invariant is whole-repo. Add the
|
||||
annotation to ${rel%/*}/chainsaw-test.yaml and delete the stage."
|
||||
fi
|
||||
done
|
||||
[ -z "$hardcoded" ] && note "no ci/*.sh hardcodes a conformance check path (fully declarative)"
|
||||
|
||||
# ---- (5) the gate still BLOCKS on an l0 failure -----------------------------------------
|
||||
# Carried over from the three deleted stages' own checks — asserted once, here.
|
||||
grep -Eq '^for s in .*\bl0\b.*; do' "$GATE_SH" && grep -Eq 'fail\).*BLOCK=1' "$GATE_SH"
|
||||
chk $? "ci/gate.sh sets the merge block when the l0 stage fails" \
|
||||
"ci/gate.sh does not set the merge block on an l0 stage failure — every assertion above
|
||||
would hold while a red L0 stage merged anyway"
|
||||
|
||||
echo
|
||||
echo "whole-repo scope: ${#declared_dirs[@]} declaring test(s): $(for d in ${declared_dirs[@]+"${declared_dirs[@]}"}; do printf '%s ' "$(basename "$d")"; done)"
|
||||
if [ "$FAILED" -eq 0 ]; then
|
||||
echo "check OK: whole-repo-invariants-wired — ${PASS} assertions (R-TEST-03)"
|
||||
exit 0
|
||||
fi
|
||||
echo "CHECK FAIL (whole-repo-invariants-wired): ${FAILED} failed, ${PASS} passed" >&2
|
||||
exit 1
|
||||
@@ -1,242 +0,0 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL — every OIDC client's STORED secret AND redirect URIs equal what git
|
||||
# declares (R-IDP-02, R-IDP-04) [RUNNABLE]
|
||||
# Spec: docs/15 identity-and-access; ADR-0014.
|
||||
# Impl: platform/keycloak/client-secret-reconciler.yaml (the CronJob + its script).
|
||||
#
|
||||
# WHY THIS TEST EXISTS
|
||||
# --------------------
|
||||
# The KeycloakRealmImport resolves each confidential client's secret from a `(env:NAME)`
|
||||
# placeholder mapped to a key of the SOPS-encrypted keycloak-realm-clients Secret. When
|
||||
# the Keycloak Operator cannot resolve a placeholder it stores the LITERAL placeholder
|
||||
# text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS; the CR then rests
|
||||
# at Done=True / Started=False and never re-runs. Measured 2026-09-06: prod had all 12
|
||||
# clients holding the literal placeholder — every OIDC login on the cluster failed with
|
||||
# "Invalid client or Invalid client credentials" — and dev had one (oauth2-proxy), whose
|
||||
# Secret key was added after dev's import had already completed.
|
||||
#
|
||||
# Every SIGNAL the cluster offered was green: the import Job succeeded, the CR read
|
||||
# Done=True, the Keycloak pods were healthy, and the existing keycloak-config-as-code
|
||||
# test (which asserts Done=True) PASSED throughout. The only observable was a human
|
||||
# failing to log in. This test is that observable, automated: it compares STORED against
|
||||
# DECLARED for every client and fails on any difference, so the class cannot sit
|
||||
# undetected for months again.
|
||||
#
|
||||
# GIVEN the `homelab` realm and the SOPS-encrypted keycloak-realm-clients Secret,
|
||||
# WHEN the reconciler's script is run in --audit mode (READ-ONLY: it cannot write, and
|
||||
# its endpoint allow-list makes users/groups/roles unreachable),
|
||||
# -> THEN every declared client's stored secret matches git, and none holds an
|
||||
# unresolved import placeholder. Emits KEYCLOAK_CLIENT_SECRETS_AUDIT_OK.
|
||||
# -> AND every declared client URL — redirectUris, webOrigins, rootUrl, baseUrl,
|
||||
# adminUrl — matches git. Emits KEYCLOAK_CLIENT_URLS_AUDIT_OK.
|
||||
#
|
||||
# THE URL HALF EXISTS FOR THE SAME REASON, ONE FIELD OVER. The realm import is templated
|
||||
# per environment, so moving prod's BASE_DOMAIN re-renders every declared redirect URI in
|
||||
# git — and reaches the realm never, because a completed import does not re-run. Measured
|
||||
# on prod: stored redirect URIs still on the PRE-migration hostname while git declared the
|
||||
# new one. Every OIDC login would fail with a redirect_uri mismatch, and once again every
|
||||
# signal would be green: the CR reads Done=True, the pods are healthy, and the
|
||||
# secret-only version of this very test would have PASSED. Asserting only the secret
|
||||
# sentinel is what would have let that ship.
|
||||
#
|
||||
# WHY IT RUNS THE SHIPPED SCRIPT rather than re-implementing the comparison: a detector
|
||||
# that duplicates the healer's logic drifts from it. This audits with the same code, in
|
||||
# the same mode the L0 gate selftests, so a passing audit is evidence about the healer
|
||||
# too. It runs as a Job IN the keycloak namespace (not in the ephemeral chainsaw
|
||||
# namespace) because the desired state is the mounted keycloak-realm-clients Secret plus
|
||||
# the keycloak-client-urls ConfigMap, and the admin credential is
|
||||
# keycloak-admin-credentials — all namespace-local, which is also why neither this Job nor
|
||||
# the CronJob needs any Kubernetes API access.
|
||||
#
|
||||
# chainsaw v0.2.12: Job + status assert; the leftover-Job delete keeps a re-run from
|
||||
# colliding with an immutable Job left by a previous failure.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: keycloak-client-secrets-match-git
|
||||
labels:
|
||||
conformance.home.k3s/R-IDP-02: "true"
|
||||
conformance.home.k3s/R-IDP-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-IDP-02, R-IDP-04"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every OIDC client's stored secret equals the SOPS-declared one and none holds an
|
||||
unresolved import placeholder — the observable that was missing when all 12 prod
|
||||
clients were broken behind a green realm import.
|
||||
spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 5m
|
||||
exec: 5m
|
||||
steps:
|
||||
- name: given-the-reconciler-is-reconciled-onto-the-cluster
|
||||
description: >-
|
||||
The audit Job runs the CronJob's own script from its ConfigMap; if Flux has not
|
||||
applied the reconciler, there is nothing to audit WITH and that is the failure.
|
||||
try:
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: keycloak-client-secret-reconciler
|
||||
namespace: keycloak
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: keycloak-client-secret-reconciler
|
||||
namespace: keycloak
|
||||
spec:
|
||||
concurrencyPolicy: Forbid
|
||||
- name: then-every-stored-client-secret-matches-git
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n keycloak delete job keycloak-client-secret-audit \
|
||||
--ignore-not-found --wait=true
|
||||
- apply:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: keycloak-client-secret-audit
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak-client-secret-audit
|
||||
app.kubernetes.io/part-of: conformance
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 240
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak-client-secret-audit
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: audit
|
||||
image: python:3.13.15-alpine3.23
|
||||
command:
|
||||
- python3
|
||||
- /opt/reconciler/reconcile-keycloak-clients.py
|
||||
# READ-ONLY. --audit never issues a write on any path; the
|
||||
# script's --selftest proves that (L0 gate).
|
||||
- --audit
|
||||
env:
|
||||
- name: KEYCLOAK_BASE_URL
|
||||
value: http://keycloak-service.keycloak.svc.cluster.local:8080
|
||||
- name: KEYCLOAK_REALM
|
||||
value: homelab
|
||||
- name: PYTHONDONTWRITEBYTECODE
|
||||
value: "1"
|
||||
- name: PYTHONUNBUFFERED
|
||||
value: "1"
|
||||
- name: KEYCLOAK_ADMIN_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: keycloak-admin-credentials
|
||||
key: admin-user
|
||||
- name: KEYCLOAK_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: keycloak-admin-credentials
|
||||
key: admin-password
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /opt/reconciler
|
||||
readOnly: true
|
||||
- name: client-secrets
|
||||
mountPath: /etc/keycloak-realm-clients
|
||||
readOnly: true
|
||||
# The URL pass's desired state. Without this mount the audit
|
||||
# fails closed on pass 2 and this test would report a false
|
||||
# failure rather than auditing the realm's redirect URIs.
|
||||
- name: client-urls
|
||||
mountPath: /etc/keycloak-client-urls
|
||||
readOnly: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: keycloak-client-secret-reconciler
|
||||
defaultMode: 0444
|
||||
- name: client-secrets
|
||||
secret:
|
||||
secretName: keycloak-realm-clients
|
||||
defaultMode: 0400
|
||||
- name: client-urls
|
||||
configMap:
|
||||
name: keycloak-client-urls
|
||||
defaultMode: 0444
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: keycloak-client-secret-audit
|
||||
namespace: keycloak
|
||||
status:
|
||||
succeeded: 1
|
||||
# The Job's exit status already encodes the verdict, but the sentinel is what the
|
||||
# run LEDGER records (conformance/runs/*/run.log): a JUnit alone cannot tell a real
|
||||
# audit from a Job that succeeded without auditing anything.
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
out="$(kubectl -n keycloak logs job/keycloak-client-secret-audit \
|
||||
--tail=-1 2>&1 || true)"
|
||||
echo "$out"
|
||||
echo "$out" | grep -q 'KEYCLOAK_CLIENT_SECRETS_AUDIT_OK' \
|
||||
|| { echo "FAIL: the audit did not report every client secret matching git"; exit 1; }
|
||||
echo "$out" | grep -q 'UNRESOLVED IMPORT PLACEHOLDER' \
|
||||
&& { echo "FAIL: a client still holds the literal import placeholder"; exit 1; }
|
||||
# The URL half. A realm still holding PRE-migration redirect URIs is the
|
||||
# state prod was measured in: every OIDC login fails with a redirect_uri
|
||||
# mismatch while the KeycloakRealmImport CR still reads Done=True. Asserting
|
||||
# only the secret sentinel would pass on exactly that realm.
|
||||
echo "$out" | grep -q 'KEYCLOAK_CLIENT_URLS_AUDIT_OK' \
|
||||
|| { echo "FAIL: the audit did not report every client URL matching git"; exit 1; }
|
||||
echo "$out" | grep -q 'DRIFT -- realm has' \
|
||||
&& { echo "FAIL: a client URL still differs from the declared one"; exit 1; }
|
||||
echo KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK
|
||||
check:
|
||||
(contains($stdout, 'KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK')): true
|
||||
catch:
|
||||
- describe:
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
namespace: keycloak
|
||||
name: keycloak-client-secret-audit
|
||||
- podLogs:
|
||||
namespace: keycloak
|
||||
selector: app.kubernetes.io/name=keycloak-client-secret-audit
|
||||
tail: 60
|
||||
finally:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n keycloak delete job keycloak-client-secret-audit \
|
||||
--ignore-not-found --wait=false
|
||||
@@ -11,18 +11,6 @@
|
||||
#
|
||||
# Goes GREEN once Keycloak is up + the realm client-secret hand-off (keycloak-realm-clients) is filled
|
||||
# so the placeholder resolves and the import completes. chainsaw v0.2.12: script + block contains check.
|
||||
#
|
||||
# NOT PROVEN HERE — READ THIS BEFORE TRUSTING A GREEN RUN (added 2026-09-06).
|
||||
# Done=True means the import JOB EXITED 0. It does NOT mean the placeholders resolved: when the
|
||||
# operator cannot resolve one it stores the LITERAL `(env:...)` text as the client secret and the
|
||||
# import still reports success, then rests at Done=True / Started=False and never re-runs. THIS
|
||||
# TEST PASSED THROUGHOUT the 2026-09-06 outage in which all 12 prod clients held the literal
|
||||
# placeholder and every OIDC login on the cluster failed. Done=True is a necessary, badly
|
||||
# insufficient condition.
|
||||
# The sufficient check is tests/l2-behavioral/keycloak-client-secrets-match-git, which compares
|
||||
# each client's STORED secret against the SOPS-declared one; the static half is
|
||||
# tests/l0-static/keycloak-client-secret-reconciler. Do not "simplify" by folding those into this
|
||||
# test: the whole lesson is that the import's own success signal is not evidence.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
|
||||
@@ -10,31 +10,8 @@
|
||||
# egress-gateway node is STOPPED (fault injection) —
|
||||
# THEN the vpn-class workload reaches NOTHING on the plain internet: its curl FAILS
|
||||
# and, critically, does NOT return the WAN IP (a WAN IP = LEAK = test FAILS).
|
||||
# vpn-class egress is blackholed while tun0 is absent (ADR-0015 invariant 3/4:
|
||||
# vpn fails CLOSED).
|
||||
#
|
||||
# MECHANISM CORRECTED 2026-09-12: this comment used to credit "the host nftables
|
||||
# kill-switch". That table never sees this traffic — Cilium's eBPF egress-gateway
|
||||
# datapath (bpf.masquerade) bpf_redirect()s the steered skbs to the egress device
|
||||
# without invoking netfilter POSTROUTING, measured with counters on the gateway
|
||||
# node (0 packets for the pod CIDR and the PIA range in forward and postrouting,
|
||||
# against 45166 on the same hook for WAN traffic). What fails closed is CILIUM's
|
||||
# own drop when it cannot use the cached egress interface; tcpdump on both the WAN
|
||||
# and tun0 during the stop caught 0 packets. The ASSERTION below is unchanged and
|
||||
# still the right one — only the explanation was wrong.
|
||||
#
|
||||
# PREREQUISITE, AND IT IS NOT OPTIONAL: this test cannot run at all unless a node
|
||||
# carries homelab.keithjackson.dev/egress-gateway=true (the fault Job is pinned to
|
||||
# it by nodeSelector, and with no gateway there is no tunnel to stop).
|
||||
#
|
||||
# RECOVERY DEPENDS ON THE RE-STEER PATH. Stopping and starting the tunnel gives
|
||||
# tun0 a NEW ifindex, and Cilium does not re-resolve `interface: tun0` on its own,
|
||||
# so before 2026-09-12 the recovery step could not pass: measured, the map stayed
|
||||
# on the old ifindex and vpn-class egress never came back. roles/vpn_egress_gateway
|
||||
# now publishes the live uplink as a node label on every connect, which makes
|
||||
# Cilium re-resolve; with it, stop->start recovers unaided (map 1989 -> 1990, PIA IP
|
||||
# returns). If this test's recovery step ever fails again, check
|
||||
# `journalctl -u vpn-egress-resteer` on the gateway node first.
|
||||
# The host nftables kill-switch blackholes egress-class=vpn traffic while tun0
|
||||
# is absent (ADR-0015 invariant 3/4: vpn fails CLOSED).
|
||||
# AND the `internet`-class workload is UNAFFECTED (invariant 3: internet fails OPEN).
|
||||
# WHEN the tunnel RECOVERS (service restarted), THEN vpn-class egress returns (PIA IP).
|
||||
#
|
||||
|
||||
@@ -1,347 +0,0 @@
|
||||
# Bulk NFS storage — the media library and the legacy Nextcloud archive
|
||||
|
||||
The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`.
|
||||
Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is
|
||||
assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the
|
||||
case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk
|
||||
provisioning unimplemented, because nothing needs it.
|
||||
|
||||
---
|
||||
|
||||
## 1. The storage host, and why prod cannot reach it
|
||||
|
||||
The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface:
|
||||
|
||||
```
|
||||
vmbr0 192.168.1.9/24 default via 192.168.1.1 (pfSense LAN)
|
||||
```
|
||||
|
||||
| | nodes | reaches 192.168.1.9? |
|
||||
|---|---|---|
|
||||
| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) |
|
||||
| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. |
|
||||
|
||||
Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give
|
||||
different values for the `BULK_NFS_SERVER` cluster-var:
|
||||
|
||||
| outcome | `BULK_NFS_SERVER` for prod |
|
||||
|---|---|
|
||||
| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` |
|
||||
| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` |
|
||||
|
||||
Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder
|
||||
behaves differently from the repo's other ones.**
|
||||
|
||||
### It only needs ONE port
|
||||
|
||||
The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that
|
||||
must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed
|
||||
four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The
|
||||
server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the
|
||||
conservative floor.
|
||||
|
||||
### The export's client list is the SAME under both outcomes — measured, not assumed
|
||||
|
||||
**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation
|
||||
(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`:
|
||||
|
||||
```
|
||||
SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22
|
||||
ss -tn -> 192.168.1.9:22 192.168.2.132:39768
|
||||
```
|
||||
|
||||
The original WAN-side source address survives the translation (return traffic finds its way back via
|
||||
the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's
|
||||
**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either
|
||||
way. **It does not have to wait for the routing decision.**
|
||||
|
||||
---
|
||||
|
||||
## 2. The layout, as found
|
||||
|
||||
### `Storage/plex` → `/mnt/Storage/plex` — 8.33 T used, 6.91 T avail
|
||||
|
||||
```
|
||||
/mnt/Storage/plex/ drwxr-xr-x 1000:3000
|
||||
├── downloads/ drwxr-xr-x 1000:1000
|
||||
│ ├── complete/ drwxr-xr-x 1000:3000
|
||||
│ │ ├── radarr/ drwxr-xr-x 1000:3000
|
||||
│ │ └── tv-sonarr/ drwxr-xr-x 1000:3000
|
||||
│ ├── incomplete/ drwxrwxr-x 1000:3000
|
||||
│ ├── downloads -> /shared/downloads (symlink OUTSIDE the export — see §8)
|
||||
│ └── <in-flight release dirs> drwxr-xr-x 1000:1000
|
||||
├── movies/ 1165 entries drwxr-xr-x 1000:3000
|
||||
├── tv/ 107 entries drwxr-xr-x 1000:3000
|
||||
└── photos/ drwxr-xr-x 1000:3000
|
||||
```
|
||||
|
||||
**The single most important fact about this layout: `downloads/` is a SIBLING of `tv/` and `movies/`
|
||||
inside the same dataset.** Sonarr's import source and its library are on one filesystem on the
|
||||
server, and so are Radarr's. That is what makes hardlinked / atomically-renamed imports possible at
|
||||
all — and §4 is about not throwing it away on the client side.
|
||||
|
||||
Note the download client's completed folders are already named `radarr` and `tv-sonarr`, i.e. this
|
||||
tree has been driven by an arr stack for a long time. The k3s arrs are joining an existing layout,
|
||||
not defining one.
|
||||
|
||||
### `Storage/nextcloud` → `/mnt/Storage/nextcloud` — 270 G used
|
||||
|
||||
```
|
||||
/mnt/Storage/nextcloud/ drwxr-xr-x root:root
|
||||
└── nextcloud/ drwxr-xr-x 1000:3000
|
||||
├── config/ 103 M drwxr-xr-x 1000:3000
|
||||
├── config_old/ 3.4 G drwxr-xr-x 1000:3000
|
||||
└── config_backup/ 267 G drwxr-xr-x 1000:3000
|
||||
```
|
||||
|
||||
**There is no `data/` directory.** 99% of the dataset is `config_backup`. This is the old, non-k3s
|
||||
Nextcloud's configuration plus a backup blob — an **archive**, not a live user-data tree. That is
|
||||
why it is mounted read-only; see §6.
|
||||
|
||||
---
|
||||
|
||||
## 3. What each app mounts, and why
|
||||
|
||||
| app | export | mountPath | subPath | mode | access mode |
|
||||
|---|---|---|---|---|---|
|
||||
| plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany |
|
||||
| sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
|
||||
| radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
|
||||
| nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany |
|
||||
|
||||
In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr
|
||||
Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from
|
||||
`/media/downloads/complete/{tv-sonarr,radarr}`.
|
||||
|
||||
**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually
|
||||
given the same export (it is not in this change), the arrs and the download client will already agree
|
||||
on absolute paths, which is the second precondition for hardlinked imports — otherwise every import
|
||||
needs a Remote Path Mapping and quietly falls back to copying when one is missing.
|
||||
|
||||
**`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the
|
||||
renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the
|
||||
config volume entirely. See §8 for the separate latent bug that discovery exposed.
|
||||
|
||||
---
|
||||
|
||||
## 4. Why the arrs mount the export ROOT, not two subPaths
|
||||
|
||||
This is the design decision most likely to be "simplified" later by someone who has not read this.
|
||||
|
||||
Sonarr and Radarr import a finished download into the library by **hardlinking** it (preserving the
|
||||
seeding copy) or by an **atomic rename**. Both are same-filesystem operations, and **to the kernel,
|
||||
"same filesystem" means "same mount."**
|
||||
|
||||
The tempting narrower shape is two subPath volumes — `subPath: tv` and `subPath: downloads`. Those
|
||||
are **two separate mounts** of the same export. `link()` across them returns **EXDEV** no matter that
|
||||
the server knows perfectly well they are one dataset. Sonarr does not fail loudly at that: it
|
||||
degrades to copy-then-delete. Every import then rewrites the entire file over NFS and destroys the
|
||||
seeding copy. The symptom is "imports are slow and my ratio is gone", weeks later.
|
||||
|
||||
The smallest subtree containing both `tv/` and `downloads/` **is the export root**. So the root mount
|
||||
is the *minimal* mount that preserves hardlinks: least privilege bounded below by a hard requirement,
|
||||
not a shortcut.
|
||||
|
||||
The cost is stated rather than hidden: Sonarr can also write `/media/movies` and Radarr `/media/tv`.
|
||||
Narrowing that would cost hardlinks, which is the worse trade.
|
||||
|
||||
---
|
||||
|
||||
## 5. The export change the operator must apply — STRICTLY ADDITIVE
|
||||
|
||||
**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the
|
||||
exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and
|
||||
192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the
|
||||
live non-k3s prod stack.
|
||||
|
||||
Current values (`zfs get sharenfs Storage`):
|
||||
|
||||
```
|
||||
Storage/plex [email protected]:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash
|
||||
Storage/nextcloud [email protected]:@192.168.1.5,no_subtree_check,no_root_squash
|
||||
```
|
||||
|
||||
### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list
|
||||
|
||||
```sh
|
||||
zfs set sharenfs='[email protected]:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex
|
||||
```
|
||||
|
||||
Every pre-existing host is preserved verbatim and no option changes; only clients are added.
|
||||
|
||||
### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=`
|
||||
|
||||
```sh
|
||||
zfs set sharenfs='[email protected]:@192.168.1.5,[email protected]:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud
|
||||
```
|
||||
|
||||
This is the one place read-only can be enforced **server-side**, and it is worth taking: only
|
||||
Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a
|
||||
misconfigured client cannot then damage the archive.
|
||||
|
||||
**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but
|
||||
all three may be scheduled on any node, so the server sees the same source addresses for all of
|
||||
them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a
|
||||
client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an
|
||||
honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a
|
||||
dedicated node pool for Plex, which is a much larger change.
|
||||
|
||||
Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host
|
||||
(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive
|
||||
adding a node; it also grants the whole subnet. Per-node is the tighter default.
|
||||
|
||||
### Verifying, without changing anything
|
||||
|
||||
```sh
|
||||
exportfs -v # the new client lines appear, existing ones unchanged
|
||||
zfs get sharenfs Storage/plex Storage/nextcloud
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. uid / gid — the failure this design exists to avoid
|
||||
|
||||
The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through
|
||||
verbatim** and the server's on-disk ownership is the entire access story. The media tree is
|
||||
`0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**.
|
||||
|
||||
**`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS
|
||||
volume plugin performs **no ownership management**, so `podSecurityContext.fsGroup` is a **no-op on
|
||||
an NFS volume**. There is no recursive chown, no `g+rwX`, nothing. The container's own uid has to be
|
||||
right.
|
||||
|
||||
| app | container uid | writes? | outcome |
|
||||
|---|---|---|---|
|
||||
| plex | 1000 (`PLEX_UID`/`PLEX_GID`, already set) | no | reads fine — the tree is 0755, so world-readable anyway |
|
||||
| sonarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
|
||||
| radarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
|
||||
| nextcloud | 33 (`www-data`) | no | reads and traverses 0755 fine |
|
||||
|
||||
**Without the added `PUID`/`PGID`, sonarr and radarr would have failed exactly the way this tier
|
||||
always fails: the mount succeeds, the pod is Ready, and every import fails EACCES.** LinuxServer
|
||||
images default to uid/gid **911**, which owns nothing on that tree.
|
||||
|
||||
`PGID: 3000` is chosen over 1000 so files these write are group-owned by the dominant group already
|
||||
on the tree, keeping them readable to the live non-k3s consumers.
|
||||
|
||||
`podSecurityContext.fsGroup: 3000` is also added to sonarr and radarr — **for the Longhorn config
|
||||
PVC, not for the NFS mount.** fsGroup does work on a Longhorn filesystem, and it makes `/data/config`
|
||||
writable by the now-uid-1000 process without depending on the image re-chowning it after `PUID`
|
||||
changed.
|
||||
|
||||
Nextcloud needs no uid work at all, which is one more reason its mount is read-only.
|
||||
|
||||
---
|
||||
|
||||
## 7. The prod placeholder does NOT fail at apply time — read this before merging
|
||||
|
||||
Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation
|
||||
(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.**
|
||||
|
||||
A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts
|
||||
`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure
|
||||
control-plane and never touches the server. The failure appears only when a pod mounts:
|
||||
|
||||
```
|
||||
MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER
|
||||
```
|
||||
|
||||
so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.**
|
||||
|
||||
**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a
|
||||
revision that carries these bulk volumes.** Dev renders a working value and is where these manifests
|
||||
are meant to be proven first.
|
||||
|
||||
### The static guard this convention leans on is missing from `main` — but the gate now refuses the value anyway
|
||||
|
||||
`bootstrap/standup.sh` — which carries the `*_is_placeholder` guard family — **does not exist on
|
||||
`main`** (verified: `git ls-tree main bootstrap/` lists only `README.md`, `flux/`, `gitea/`, `sops/`;
|
||||
the file lives on unmerged prod branches, and there is no deletion commit for it anywhere — it never
|
||||
landed rather than having been removed).
|
||||
|
||||
**UPDATED 2026-09-13.** This section used to end "so
|
||||
`conformance/tests/l0-static/placeholder-guard-coupling/check.sh` exits **3 SKIPPED**, and nothing in
|
||||
the gate currently refuses a placeholder-shaped overlay value." That is **no longer true.** That check
|
||||
was restructured (see DECISIONS-LOG, branch `fix/placeholder-guard-coupling-unskipped`): it never
|
||||
exits 3, and its HALF A screens **every environment overlay's effective cluster-vars value** against
|
||||
an independent placeholder detector with no guard required. A `BULK_NFS_SERVER` of
|
||||
`TBD-PROD-BULK-NFS-SERVER` in `clusters/prod/` is now a **merge-gate FAIL** — verified by deliberately
|
||||
authoring exactly that value and watching the check fail, which is the one hazard on this page that
|
||||
CRD validation cannot catch. HALF A also refuses a LOUD base sentinel that an overlay inherits by
|
||||
*omitting* its patch op, which is the other way this knob can silently ship wrong.
|
||||
|
||||
The guard is now **discovered by content**, not by path, so when `standup.sh` (or its predicate in a
|
||||
sourceable library) lands, the coupling half activates with no edit to the test — and it then covers
|
||||
this knob **provided** the guard gains an arm for `BULK_NFS_SERVER` and a consumer that calls it.
|
||||
Without the arm, HALF A is what holds the line, by design.
|
||||
|
||||
---
|
||||
|
||||
## 8. Loose ends and things deliberately not done
|
||||
|
||||
1. **`/data/config` is very likely the wrong home for the arrs' config PVC — a pre-existing latent
|
||||
bug, not fixed here.** LinuxServer's arr images run with `-data=/config`, so the app writes
|
||||
`/config`, which is the container's **ephemeral layer**, while the Longhorn PVC mounted at
|
||||
`/data/config` sits unused. This is the exact failure `catalog/plex.yaml`'s `mountPath` comment
|
||||
records having already been bitten by ("the PVC sat unused and Plex wrote to the container's
|
||||
EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it
|
||||
means those apps start from an empty config volume, so it belongs in its own change with the
|
||||
operator present. **This change does not make it worse** — that is why the media mount is `/media`.
|
||||
2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy
|
||||
non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s
|
||||
qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were
|
||||
chosen so it needs no Remote Path Mapping.
|
||||
3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside
|
||||
the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log
|
||||
it while scanning `downloads/`.
|
||||
4. **`nfsvers=4.2` was not chosen**, though the server advertises it. Nothing in these workloads uses
|
||||
what 4.2 adds (server-side copy, sparse-file `SEEK`/`ALLOCATE`), and 4.1 is the version the
|
||||
existing `bulk` StorageClass stub already named.
|
||||
5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning
|
||||
change that should be measured, not guessed, and it changes client behaviour under server
|
||||
failure.
|
||||
6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this
|
||||
workstation (browser-OIDC only), and this change is manifest-authoring under an explicit
|
||||
do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and
|
||||
mounting* is reasoned from the manifests.
|
||||
7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.**
|
||||
`ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the
|
||||
in-cluster conformance test for the bulk tier is still a gap, so typing it would select a
|
||||
requirement the gate cannot satisfy — the hazard `catalog/qbittorrent.yaml` already warns about.
|
||||
Scaffolding that test needs a cluster and is the correct follow-up.
|
||||
|
||||
---
|
||||
|
||||
## 9. Validating on dev, once the export is extended
|
||||
|
||||
Dev can prove all of this before prod's network path exists. Read-only checks first:
|
||||
|
||||
```sh
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive'
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide
|
||||
```
|
||||
|
||||
Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three
|
||||
of them naming the *same* export, one per namespace, which is the cross-namespace model) — each
|
||||
`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`.
|
||||
|
||||
Then the two things that actually matter, from inside a pod:
|
||||
|
||||
```sh
|
||||
# (a) the writer can write — this is the uid check, and it is the one that fails in practice
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
|
||||
sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK'
|
||||
|
||||
# (b) hardlinks work BETWEEN the library and the download path — the root-mount check
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
|
||||
sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link;
|
||||
echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK'
|
||||
|
||||
# (c) the reader really is read-only
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \
|
||||
sh -c 'touch /media/.probe 2>&1 | head -1' # expect "Read-only file system"
|
||||
```
|
||||
|
||||
`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it
|
||||
prints `EXDEV`, something has split the single mount back into two.
|
||||
@@ -1,319 +0,0 @@
|
||||
# Decommission Proxmox `192.168.2.102` → TrueNAS SCALE on the N5 Air
|
||||
|
||||
**Status:** plan. **Drafted** 2026-09-06, **revised** 2026-09-06 after the operator settled the
|
||||
target architecture.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **N5 Air** | TrueNAS SCALE. **Storage only** — no VMs, no hypervisor. |
|
||||
| **Compute** | k3s **prod** cluster (`192.168.2.10/.11/.12`). Plex, sonarr, radarr, jackett, qbittorrent, seerr and nextcloud are already modelled in `apps/wave-0/`. |
|
||||
| **Dev k3s** | Does not move to the NAS. |
|
||||
| **Proxmox host** | **Decommissioned.** |
|
||||
| **Drives** | Physically relocated. None purchased. |
|
||||
|
||||
All figures measured on the live host, not inferred.
|
||||
|
||||
---
|
||||
|
||||
## Already done — 2026-09-06
|
||||
|
||||
**Reclaimed 3.56 TB.** `Storage/plex@pre-cleanup-20260828` destroyed after confirming no holds and
|
||||
no clones. Async free drained in ~90 seconds.
|
||||
|
||||
| | before | after |
|
||||
|---|---|---|
|
||||
| `Storage` allocated | 12.3 T (75%) | **8.76 T (53%)** |
|
||||
| `Storage` free | 4.04 T | **7.60 T** |
|
||||
| `Storage/plex` used | 11.9 T | 8.33 T (0 B in snapshots) |
|
||||
|
||||
That free space is not a nicety — **it is what lets this whole migration happen with no network
|
||||
transfer at all.** See §3.
|
||||
|
||||
---
|
||||
|
||||
## 1. The plan changed shape
|
||||
|
||||
The first draft carried the VMs across on their own zvols and put Proxmox on the NAS. With no
|
||||
hypervisor on the far side, that inverts:
|
||||
|
||||
> **Every zvol is now dead weight, and every service's data has to be drained onto a plain dataset
|
||||
> before the Proxmox box dies.**
|
||||
|
||||
Dead-weight zvols, once their contents are extracted:
|
||||
|
||||
| zvol | size | after extraction |
|
||||
|---|---|---|
|
||||
| `Storage:vm-101/102/105-disk-0,-disk-1` (dev k3s) | 585 G | delete |
|
||||
| `Storage:vm-112-disk-0` (docker-services) | 65 G | delete |
|
||||
| `Storage:vm-103-disk-0` (nextcloud boot) | 81 G | delete |
|
||||
| `Storage2:vm-103-disk-0` (**nextcloud data**) | 4.06 T | extract, then delete |
|
||||
|
||||
The good news is the shape of the drain. Everything that must be rescued is either already on
|
||||
`Storage`, or can be copied onto `Storage` **locally, inside the same chassis, onto the very drives
|
||||
that are moving.** With 7.60 T free there is room. **Nothing crosses the network.**
|
||||
|
||||
---
|
||||
|
||||
## 2. Findings
|
||||
|
||||
### 2a. "The path lies" — two confirmed instances, assume a third
|
||||
|
||||
**Games — 422 GB, not on the Storage pool.** Challenged and re-verified; the test is decisive:
|
||||
|
||||
```
|
||||
findmnt -T /mnt/Storage/Games → TARGET / SOURCE rpool/ROOT/pve-1
|
||||
findmnt -T /mnt/Storage/plex → TARGET /mnt/Storage/plex SOURCE Storage/plex
|
||||
```
|
||||
|
||||
`Games` is a plain directory on the Proxmox **root** filesystem that merely sits inside the Storage
|
||||
mount tree. The pool accounting closes on it exactly:
|
||||
|
||||
```
|
||||
rpool ALLOC 540 G = ROOT 431 G + data 105 G + var-lib-vz 3.75 G
|
||||
ROOT 431 G = Games 422 G + /usr 3.5 G + /var 5 G + /boot 0.4 G
|
||||
```
|
||||
|
||||
The two Microns are a **mirror**, so the pool is ~888 G usable, not ~1.9 T — 422 G is 48% of it, and
|
||||
`zpool list` agrees at 60% capacity overall. Surprising, but the numbers do match.
|
||||
|
||||
**Gitea — 13 MB in an anonymous Docker volume, not where its compose file points.**
|
||||
`/root/docker-compose.yml` in LXC 106 bind-mounts `/opt/gitea/data:/var/lib/data`. That path holds
|
||||
**512 bytes**. The real Gitea state is in an unnamed Docker volume:
|
||||
|
||||
```
|
||||
.../volumes/8a1d2616…/_data/git/repositories/ 12 M home-network.git, home-network-uds.git, jakman85/…
|
||||
.../volumes/8a1d2616…/_data/data/gitea.db 11 M SQLite, mtime today 19:00 — live
|
||||
```
|
||||
|
||||
Copying `/opt/gitea` would migrate an empty directory and silently lose all three repositories.
|
||||
|
||||
**Rule for the rest of the drain:** for every service, verify where the bytes actually are before
|
||||
trusting the path in a compose file, an export list or a mount table. Two for two so far.
|
||||
|
||||
### 2b. Retiring pfSense is easier than rehoming it
|
||||
|
||||
`pfsense` is VM 100 and **is** `192.168.1.1`. Decommissioning the host takes the entire
|
||||
`192.168.1.0/24` with it:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `.1` pfSense · `.4` iDRAC · `.9` the Proxmox host itself | die with the box |
|
||||
| `.5` nextcloud · `.6` plex · `.7` gitea · `.18` docker-services/Pi-hole | data drains, service moves to k3s |
|
||||
| `.8` unifi · `.20` monitoring · `.21` auth · `.10` semaphore | rehome or retire (§2e) |
|
||||
| `.49` kube-vip VIP · `.50/.51/.52` dev k3s | dev cluster is not moving; retire with the box |
|
||||
|
||||
**But pfSense exists only to contain that lab LAN.** Prod k3s is already on `192.168.2.0/24` —
|
||||
pfSense's *WAN* side — and the NAS will be too. Once the lab is gone, nothing lives behind pfSense.
|
||||
**Retire it; do not rehome it.** That is only clean because everything else is leaving `1.0/24`
|
||||
anyway, which is exactly what this migration does.
|
||||
|
||||
**A documented blocker dissolves with it.** `infra/PROD-STANDUP.md` records that prod k3s cannot
|
||||
reach the S3 backup endpoint at `192.168.1.9:9000` because prod sits on pfSense's WAN side and the
|
||||
crossing rule does not exist. With the NAS on `192.168.2.0/24`, prod reaches its storage directly
|
||||
and no pfSense rule is ever needed.
|
||||
|
||||
**Three things to close before pfSense goes:**
|
||||
|
||||
1. The SOHO router NATs inbound **443** to `192.168.2.102` for the legacy stack. It must be
|
||||
repointed at prod's Gateway (`192.168.2.20`). `infra/PROD-PROVISIONING.md` §10a already flags
|
||||
that the apex flip and the legacy cutover are the same event — **this migration is that event.**
|
||||
2. **VLANs 5 and 123** (minecraft; the arr VM, which reaches Plex media as `10.123.1.10`) are
|
||||
pfSense-side constructs. They disappear; the k3s apps need no VLAN.
|
||||
3. The **OpenVPN split-tunnel egress gateway** currently runs on `k3s-dev-3` (ADR-0015). It is
|
||||
already modelled for prod in `platform/vpn-egress/`, but confirm prod's gateway is live before
|
||||
the dev node goes away.
|
||||
|
||||
### 2c. Nextcloud is the critical path
|
||||
|
||||
Its user data is **inside a filesystem, inside LVM, inside a 4 TB zvol, inside a VM**:
|
||||
|
||||
```
|
||||
Storage2/vm-103-disk-0 → zd224 → PV /dev/zd224 → VG vg_storage → LV lv_storage (<4.00 T)
|
||||
```
|
||||
|
||||
With no hypervisor on the far side this must be copied out **at the file level, while the VM is
|
||||
still running** — it gates the decommission and it is the longest single task in the plan.
|
||||
|
||||
**One number is missing and I could not get it.** VM 103's QEMU guest agent is not running
|
||||
(`qm agent 103 ping` → *"QEMU guest agent is not running"*) and its SSH does not accept the
|
||||
`home_network` key. The zvol shows `referenced 3.94 T`, but it is attached **without `discard=on`**
|
||||
and is thick-provisioned (`refreservation 4.06 T`), so deletions inside the guest were never
|
||||
returned to ZFS — 3.94 T is an **upper bound**, possibly far above the truth.
|
||||
|
||||
**Run this inside the Nextcloud VM and the last open question closes:**
|
||||
|
||||
```bash
|
||||
df -h / # or wherever lv_storage is mounted
|
||||
lsblk -f /dev/mapper/vg_storage-lv_storage
|
||||
```
|
||||
|
||||
Separately: `Storage/nextcloud` is 270 G, and **267 G of it is a directory called `config_backup`**
|
||||
(`config` itself is 103 M). Almost certainly a stale dump — likely another easy reclaim, but confirm
|
||||
before deleting.
|
||||
|
||||
### 2d. Storage2 — and the two oldest drives — can probably be retired
|
||||
|
||||
After the drain and the zvol deletions, `Storage` would hold roughly:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| plex | 8.33 T |
|
||||
| nextcloud data | ≤ 3.94 T (real number pending §2c) |
|
||||
| Games | 0.42 T |
|
||||
| docker-volumes, other_containers, k8s-volumes, gitea | ~0.02 T |
|
||||
| **total** | **≤ 12.7 T of 16.4 T (≤ 77%)** |
|
||||
|
||||
If the real Nextcloud figure comes in under ~2 T — which the missing-TRIM situation makes plausible
|
||||
— everything fits on the 18 TB mirror at a comfortable ~65%, and **`Storage2` can be retired
|
||||
entirely.** That would remove the two **6.0-year-old** Seagates, by a wide margin the most
|
||||
failure-prone hardware in the estate, and free two bays. This is the single best safety outcome
|
||||
available, and it costs nothing.
|
||||
|
||||
### 2e. Services that simply end
|
||||
|
||||
Decommissioning kills these; none has a home in the target architecture unless you give it one:
|
||||
|
||||
| guest | note |
|
||||
|---|---|
|
||||
| `docker-services` Pi-hole (`192.168.1.18`) | **LAN DNS.** Confirm what `192.168.2.0/24` uses for DNS before the box dies. |
|
||||
| `unifi` (VM 121) | Network controller — needs a home or a cloud key. |
|
||||
| `auth` (VM 122) | Superseded by Keycloak on prod k3s. |
|
||||
| `monitoring` (VM 120) | Superseded by the k3s platform monitoring stack. |
|
||||
| `semaphore` (LXC 110) | 432 MB resident of an 8 GB reservation; low value. |
|
||||
| `minecraft` (LXC 108) | Operator call. Its 3.5 G subvol is on `Storage` and rides along. |
|
||||
| **versitygw S3** on `192.168.1.9:9000` | **Velero + CNPG backup target.** Dies with the box. TrueNAS must take this role (MinIO app) or prod loses its backup destination — which `infra/PROD-STANDUP.md` already records as uncovered. |
|
||||
|
||||
### 2f. Still true from the first draft
|
||||
|
||||
- **No VM backups exist.** `/etc/pve/jobs.cfg` empty, no PBS, newest dump log April 2025.
|
||||
- **`ata-WDC_…3FGHAB2T` has 36 ZFS read errors** on pristine SMART (0 reallocated / pending /
|
||||
uncorrectable, `PASSED`) — points at cable or controller, not platter. Re-verify after the move.
|
||||
- Drive ages: WD 18 TB pair **4.75 y**, Seagate 8 TB pair **6.0 y**.
|
||||
|
||||
---
|
||||
|
||||
## 3. TrueNAS SCALE specifics
|
||||
|
||||
**ZFS compatibility: verified, not assumed.** The only non-default features *active* on either pool
|
||||
are `block_cloning`, `head_errlog`, `vdev_zaps_v2`, `zilsaxattr`, `zstd_compress`, `log_spacemap`,
|
||||
`project_quota`, `spacemap_v2` — all OpenZFS **2.2** era or older. Nothing 2.3- or 2.4-only is
|
||||
active, even though the pools were last written by `zfs-2.4.3-pve1`. Any TrueNAS SCALE from 24.10
|
||||
onward imports them cleanly.
|
||||
|
||||
**Mountpoints mostly already match TrueNAS convention** — a lucky accident worth knowing:
|
||||
|
||||
| dataset | current mountpoint | TrueNAS wants | |
|
||||
|---|---|---|---|
|
||||
| `Storage/plex`, `/nextcloud`, `/docker-volumes`, `/other_containers`, `/k8s-volumes`, `/k3s-backups` | `/mnt/Storage/…` | `/mnt/Storage/…` | already correct |
|
||||
| `Storage` (pool root) | `/Storage` | `/mnt/Storage` | fix on import |
|
||||
| `Storage2` (pool root) | `/Storage2` | `/mnt/Storage2` | fix on import |
|
||||
| `Storage/subvol-108-disk-0` | `/Storage/subvol-108-disk-0` | — | rename or delete |
|
||||
|
||||
**Clear the ZFS `sharenfs` properties.** `Storage/plex`, `Storage/nextcloud` and
|
||||
`Storage/docker-volumes` carry exports as ZFS properties; `Storage/k8s-volumes` is in
|
||||
`/etc/exports`. TrueNAS generates exports from its own middleware database and will fight
|
||||
properties it did not set. Wipe them and recreate the shares in the UI.
|
||||
|
||||
**Fix the export targets while you are there.** Today's lists are stale: `/mnt/k3s` →
|
||||
`192.168.1.25/.26/.27` and `k8s-volumes` → `192.168.1.40/.41/.42` are hosts that do not answer ARP.
|
||||
The new exports target **`192.168.2.10/.11/.12`**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Sequence
|
||||
|
||||
### Phase A — drain, locally, no downtime, no network
|
||||
|
||||
Everything below is a copy from somewhere else in the chassis onto `Storage`, which is moving.
|
||||
|
||||
1. **Games** — `rpool` → new dataset.
|
||||
```bash
|
||||
zfs create Storage/games
|
||||
rsync -aHAX --info=progress2 /mnt/Storage/Games/ /mnt/Storage/games/
|
||||
```
|
||||
2. **Gitea** — copy the *Docker volume*, not `/opt/gitea` (§2a). Stop the container first so
|
||||
`gitea.db` is quiesced.
|
||||
```bash
|
||||
pct exec 106 -- docker stop gitea
|
||||
zfs create Storage/gitea
|
||||
rsync -aHAX /rpool/data/subvol-106-disk-0/var/lib/docker/volumes/8a1d2616*/_data/ /mnt/Storage/gitea/
|
||||
```
|
||||
3. **Nextcloud** — the long one. Copy `lv_storage`'s contents to a new dataset at file level from
|
||||
inside the running VM, over its existing NFS mount of the host. Verify §2c's size first so you
|
||||
know whether it fits (it will, in 7.60 T free).
|
||||
4. **Confirm `config_backup`** (267 G) is disposable; drop it if so.
|
||||
5. **Verify each service's real data location before copying it** — the §2a rule.
|
||||
|
||||
### Phase B — prove the compute side before the storage moves
|
||||
|
||||
Bring the `apps/wave-0` manifests up on prod k3s while the old box is still alive, so the cutover is
|
||||
a storage swap against known-good workloads rather than two migrations at once.
|
||||
|
||||
Wrinkle: prod cannot reach `192.168.1.9` today (§2b). Either add a temporary pfSense rule for the
|
||||
test, or accept that app bring-up happens after the NAS is live on `192.168.2.0/24` — in which case
|
||||
stage the app configs now and keep the window short.
|
||||
|
||||
### Phase C — cutover
|
||||
|
||||
1. Stop all guests; reverse `startup:` order, pfSense last.
|
||||
2. Export cleanly — **never just cut power:**
|
||||
```bash
|
||||
zpool export Storage && zpool export Storage2
|
||||
zpool status # both must be gone before power-off
|
||||
```
|
||||
3. **Label every drive** with pool and serial before it leaves the chassis
|
||||
(`3FGJ6M9T`/`3FGHAB2T` = Storage; `WKD1S0TR`/`WKD1S10M` = Storage2).
|
||||
4. Move the four HDDs — cold, upright. The 6-year-old Seagates are the fragile ones.
|
||||
5. Install TrueNAS SCALE on the shipped 64 GB M.2 (Slot A). The five bays take the four HDDs with
|
||||
one spare; the Micron SSDs are **not needed** in the target — leave them in the old chassis.
|
||||
6. **Import read-only first**, verify, then import for real:
|
||||
```bash
|
||||
zpool import # both pools listed
|
||||
zpool import -o readonly=on Storage
|
||||
zfs list # verify datasets, sizes
|
||||
zpool export Storage && zpool import Storage
|
||||
```
|
||||
|
||||
### Phase D — repoint, reclaim, retire
|
||||
|
||||
1. Recreate NFS shares in the TrueNAS UI targeting `192.168.2.10/.11/.12`; clear the old `sharenfs`
|
||||
properties.
|
||||
2. Point the `apps/wave-0` PVs at the NAS; bring Plex, the arr stack and Nextcloud up on prod k3s.
|
||||
3. Stand up MinIO on TrueNAS to replace versitygw, and re-enable Velero + CNPG backups (§2e).
|
||||
4. **Only once everything is verified from k3s:** delete the dead zvols (§1) — ~731 G on `Storage`,
|
||||
4.06 T on `Storage2`.
|
||||
5. Re-evaluate §2d. If it fits, evacuate `Storage2`, retire the 6-year-old Seagates, keep two bays
|
||||
free.
|
||||
6. Repoint the SOHO 443 NAT, then power off the Proxmox host.
|
||||
7. `zpool status` — confirm the 36 read errors did **not** follow `3FGHAB2T` into the new chassis.
|
||||
Full scrub on the new hardware.
|
||||
|
||||
---
|
||||
|
||||
## 5. Rollback, and the one-way door
|
||||
|
||||
Until Phase D step 4, the old chassis is intact: `rpool`, the Proxmox install, both Microns and
|
||||
every `/etc/pve` config untouched. Put the four HDDs back and it boots into the previous world.
|
||||
|
||||
> ### Do not run `zpool upgrade`
|
||||
>
|
||||
> `zpool status Storage2` already nags — *"Some supported and requested features are not enabled…
|
||||
> action: Enable all features using 'zpool upgrade'"* — and TrueNAS will offer it in the UI after
|
||||
> import. It is the only irreversible step here: run it and the pools can never go back to the
|
||||
> Proxmox host. Leave it until the NAS has been serving k3s for weeks.
|
||||
|
||||
Note the asymmetry that makes this safe: TrueNAS's OpenZFS **reads** these pools happily, because
|
||||
newer ZFS always reads older pools. The reverse is not true.
|
||||
|
||||
---
|
||||
|
||||
## 6. Open items
|
||||
|
||||
1. **The Nextcloud number** (§2c) — one `df -h` inside VM 103. It decides whether `Storage2` and the
|
||||
two oldest drives can be retired.
|
||||
2. **Is `config_backup` (267 G) disposable?**
|
||||
3. **What serves DNS on `192.168.2.0/24`** once Pi-hole dies?
|
||||
4. **UniFi controller** — rehome, cloud key, or drop?
|
||||
5. **Minecraft** — keep or retire?
|
||||
6. **Gitea** — its whole state is 3 repos and an 11 MB SQLite DB, and prod k3s already runs its own
|
||||
Gitea. Is the legacy one worth migrating at all, or does it just end?
|
||||
@@ -0,0 +1,315 @@
|
||||
# Off-cluster backup on Cloudflare R2 — operator procedure
|
||||
|
||||
**Owner:** WS-BACKUP · **Serves:** R-STATE-03 (DR restore), R-IDP-05 (Keycloak DB), R-ENV-02 (one
|
||||
environment-agnostic path) · **Status:** authored and staged, **NOT switched on**.
|
||||
|
||||
> **Read this first.** Velero-against-R2 and CNPG/barman-against-R2 are **UNPROVEN in this repo**.
|
||||
> Nothing here claims they work. What *is* proven is in [§7](#7-what-is-proven-and-what-is-not).
|
||||
> R-STATE-03 is currently PROVEN **against MinIO only** and must be re-proven against R2 before this
|
||||
> migration can be called done.
|
||||
|
||||
## Why R2 at all
|
||||
|
||||
The off-cluster target used to be a MinIO on the Proxmox host (`http://192.168.1.9:9000`). Two things
|
||||
killed it, both recorded on 2026-09-05:
|
||||
|
||||
1. **Prod physically cannot reach it.** Prod is on the SOHO network `192.168.2.0/24`, which is
|
||||
pfSense's **WAN**; the MinIO sits inside pfSense's LAN `192.168.1.0/24`. Measured *from a pod inside
|
||||
prod* — the workstation's own reachability proves nothing about pod egress and has misled this
|
||||
project once already — `curl http://192.168.1.9:9000/minio/health/live` timed out after 15 s.
|
||||
Reaching it would need a WAN→LAN rule that does not exist.
|
||||
2. **Dev's "off-cluster" backup was not off-cluster.** That MinIO runs on the *same Proxmox host* as
|
||||
dev's VMs, so it shared a failure domain with the thing it was backing up.
|
||||
|
||||
R2 needs no firewall change, is genuinely off-site, and is already trusted with the OpenTofu remote
|
||||
state (`infra/tofu/REMOTE-STATE.md`) — the same account, the same kind of token. It is explicitly an
|
||||
interim decision: *"that's where backups will go for the time being until a better on-prem backup
|
||||
solution is created."*
|
||||
|
||||
---
|
||||
|
||||
## 1. Cloudflare: buckets, token, account id
|
||||
|
||||
All of this is in the Cloudflare dashboard; none of it touches the cluster.
|
||||
|
||||
1. **R2 → Create bucket** → `velero`. Location hint: whatever is nearest; jurisdiction default.
|
||||
2. **R2 → Create bucket** → `cnpg`.
|
||||
Both names are fixed by the manifests: `bucket: velero` in `platform/velero/helmrelease.yaml`, and
|
||||
`cnpg` via `CNPG_BACKUP_PATH` (`s3://cnpg/dev`, `s3://cnpg/prod`). Use exactly those names or change
|
||||
the manifests — do not improvise.
|
||||
3. **R2 → Manage API Tokens → Create API Token**
|
||||
- Permission: **Object Read & Write** (not "Admin Read & Write" — the cluster never needs to create
|
||||
or delete buckets, and a token that can is a token that can delete your backups).
|
||||
- **Specify bucket(s):** `velero` **and** `cnpg`. One token scoped to both — the manifests already
|
||||
assume one shared key, and `s3check` proves both buckets with it.
|
||||
- Copy the **Access Key ID** and **Secret Access Key**. They are shown **once**. Put them in the
|
||||
password manager before you do anything else.
|
||||
4. **Account id** — 32 hex characters, visible in the dashboard URL and in the R2 S3 endpoint the
|
||||
token page shows you.
|
||||
|
||||
**This repo already knows it.** `infra/tofu/backend.r2.hcl` commits
|
||||
`https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com` for the OpenTofu state bucket,
|
||||
with the (correct) note that an account id is not a credential. **If the backup buckets live in the
|
||||
same Cloudflare account as the tofu state — they should; there is no reason for a second account —
|
||||
then the endpoint is already known and you do not need to look anything up:**
|
||||
|
||||
```
|
||||
https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com
|
||||
```
|
||||
|
||||
Confirm it matches the endpoint on the token page. If it does not, you have two accounts and the
|
||||
`velero`/`cnpg` buckets are in the other one — use that one's id, and say so in the DECISIONS-LOG.
|
||||
|
||||
The endpoint form is always `https://<account-id>.r2.cloudflarestorage.com` — **no bucket in the
|
||||
hostname** (path-style; see [§6](#6-r2-specific-gotchas)).
|
||||
|
||||
---
|
||||
|
||||
## 2. Repo values to set
|
||||
|
||||
Only **one** non-secret value is missing, and it is not in a secret file:
|
||||
|
||||
| File | Key | From | To |
|
||||
|---|---|---|---|
|
||||
| `clusters/base/cluster-vars.yaml` | `S3_BACKUP_ENDPOINT` | `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com` | `https://<account-id>.r2.cloudflarestorage.com` |
|
||||
|
||||
Already authored, **do not change**: `S3_BACKUP_REGION: "auto"`, `VELERO_BACKUP_PREFIX` (dev `dev` /
|
||||
prod `prod`), `CNPG_BACKUP_PATH` (`s3://cnpg/dev` / `s3://cnpg/prod`), and `checksumAlgorithm: ""` on
|
||||
the Velero BackupStorageLocation.
|
||||
|
||||
The placeholder is deliberately loud, and the guards match it as a **substring** — `bootstrap/standup.sh`
|
||||
refuses to probe it and refuses to resume `velero` while it is there.
|
||||
|
||||
---
|
||||
|
||||
## 3. Credentials: four Secrets, one command
|
||||
|
||||
The R2 key is read by **four** committed SOPS Secrets, in **two** different shapes. No manifest changes
|
||||
are needed — only the encrypted values change:
|
||||
|
||||
| File | Consumer | Shape |
|
||||
|---|---|---|
|
||||
| `platform/velero/velero-cloud.sops.yaml` | Velero (`velero-cloud-credentials`, ns `velero`) | `stringData.cloud` — an **AWS INI file**: `[default]` + `aws_access_key_id=` / `aws_secret_access_key=` |
|
||||
| `platform/keycloak/keycloak-db-backup-credentials.sops.yaml` | CNPG `keycloak-db` (ns `keycloak`) | `ACCESS_KEY_ID` / `ACCESS_SECRET_KEY` |
|
||||
| `apps/wave-0/nextcloud-db-backup-credentials.sops.yaml` | CNPG `nextcloud-db` | `ACCESS_KEY_ID` / `ACCESS_SECRET_KEY` |
|
||||
| `apps/wave-0/semaphore-db-backup-credentials.sops.yaml` | CNPG `semaphore-db` | `ACCESS_KEY_ID` / `ACCESS_SECRET_KEY` |
|
||||
|
||||
A Secret is namespace-scoped and a Pod reads only its own namespace's, which is why there are four
|
||||
copies of one key rather than one.
|
||||
|
||||
### The one command
|
||||
|
||||
```bash
|
||||
./infra/devcontainer/dev.sh run platform/backup/set-r2-credentials.sh
|
||||
```
|
||||
|
||||
It prompts for the two values (never echoed, never in shell history, never written to a file), writes
|
||||
them into all four Secrets with `sops set` — which encrypts in place, so **no decrypted copy of any
|
||||
Secret is ever created** — then decrypts each file back and proves all four now hold the same key by
|
||||
comparing 32-bit fingerprints. It never prints a credential.
|
||||
|
||||
There is deliberately **no plaintext template to fill in and encrypt**. The
|
||||
`cp x.example x.yaml && $EDITOR && sops -e -i` pattern leaves a plaintext Kubernetes Secret sitting in
|
||||
the working tree, one `git add -A` away from being in git forever; `sops set` removes that window
|
||||
entirely. If you would rather edit by hand, `sops platform/velero/velero-cloud.sops.yaml` decrypts to
|
||||
`$EDITOR` and re-encrypts on save — also with no plaintext on disk — but then you must repeat it four
|
||||
times and keep them identical yourself.
|
||||
|
||||
To check the four files agree without changing anything:
|
||||
|
||||
```bash
|
||||
./infra/devcontainer/dev.sh run platform/backup/set-r2-credentials.sh --check
|
||||
```
|
||||
|
||||
### What is NOT changed
|
||||
|
||||
`platform/backup/minio-credentials.sops.yaml` keeps the **MinIO** key. That is not an oversight — it
|
||||
is what you need to read the pre-migration backups ([§8](#8-going-back-to-minio)). Its
|
||||
`.yaml.example` sibling documents the MinIO shape and stays accurate for the MinIO target.
|
||||
|
||||
Review the diff before committing: you should see **only** `ENC[...]` blobs and `lastmodified`/`mac`
|
||||
change in four files. If a key name or a `metadata.name` moved, stop.
|
||||
|
||||
---
|
||||
|
||||
## 4. Prove the target BEFORE merging
|
||||
|
||||
```bash
|
||||
K=infra/ansible/.cache/dev.kubeconfig # or prod.kubeconfig
|
||||
./infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig $K --expect-server https://192.168.2.50:6443 \
|
||||
--s3-endpoint https://<account-id>.r2.cloudflarestorage.com s3check
|
||||
```
|
||||
|
||||
`--s3-endpoint` probes a **candidate** endpoint instead of the live `cluster-vars` value, so the target
|
||||
is proven before it is authored. Run it against **both** clusters — prod's egress is the whole reason
|
||||
this migration exists, and it is a different network position from dev's.
|
||||
|
||||
`s3check` runs a throwaway pod per credential and, for each, proves:
|
||||
|
||||
1. the committed credential **loads and authenticates**, in the exact shape its consumer reads it
|
||||
(Velero's INI file mounted from the `cloud` key; CNPG's discrete `ACCESS_KEY_ID`/`ACCESS_SECRET_KEY`);
|
||||
2. the bucket **exists and is listable** under this environment's own prefix (`ListObjectsV2`, the
|
||||
operation Velero's BSL sync and Barman's WAL scan both perform);
|
||||
3. **PUT → GET → byte-compare (sha256 + length) → DELETE → confirm-gone** of one throwaway object at
|
||||
`<prefix>/.s3check/<pod>.probe` — a genuine write, not a HEAD. A HEAD passes with a read-only token
|
||||
and with R2 rejecting every write;
|
||||
4. whether the **R2 checksum trap** ([§6](#6-r2-specific-gotchas)) is present, probed on purpose by
|
||||
repeating the PUT with checksums *enabled*, and named explicitly either way.
|
||||
|
||||
It **fails closed**: a pod that cannot be created, an empty log, a missing verdict line, a short
|
||||
assertion count, an OK-line count that disagrees with the claimed count, or any single failed
|
||||
assertion all exit non-zero. There is no "reachability unknown, carrying on". Each failure names its
|
||||
own cause and fix — wrong key, unscoped token, missing bucket, no pod egress, checksum rejection.
|
||||
|
||||
It writes and deletes exactly one object per bucket and leaves no pod behind. It applies nothing and
|
||||
does not enable `velero`.
|
||||
|
||||
---
|
||||
|
||||
## 5. The verification sequence — and it ends in a RESTORE
|
||||
|
||||
**A backup that has never been restored is not a backup.** R-STATE-03 is PROVEN against MinIO; that
|
||||
proof does not transfer to a different object store with a different S3 implementation.
|
||||
|
||||
1. **`s3check` on dev, then on prod.** Both must pass. (§4)
|
||||
2. Merge the endpoint + the four re-encrypted Secrets. Let Flux reconcile.
|
||||
3. **Resume `velero`** — `bootstrap/standup.sh … resume` does it once the endpoint is no longer a
|
||||
placeholder, or `flux resume kustomization velero`.
|
||||
4. `velero backup-location get` → `default` **Available**. *Available only means the BSL could be
|
||||
read;* it is not a backup.
|
||||
5. `velero backup create --from-schedule daily` → wait for **Completed**, then
|
||||
`velero backup logs <name>`. Watch specifically for `PodVolumeBackup` failures: the PVC-data path
|
||||
is kopia, a **different S3 client** from the one the BSL uses (see §7), so metadata can succeed
|
||||
while volume data fails.
|
||||
6. **Restore.** `velero restore create --from-backup <name> --include-namespaces <scratch-ns>` into a
|
||||
*scratch* namespace, and verify the PVC contents actually arrived — not just that the Restore says
|
||||
Completed. This is the R-STATE-03 re-proof.
|
||||
7. **CNPG PITR.** Confirm WAL is arriving:
|
||||
`kubectl -n keycloak get cluster keycloak-db -o jsonpath='{.status.conditions}'` (look for
|
||||
`ContinuousArchiving`), and that objects exist under `s3://cnpg/<env>/keycloak-db/`. Then do the
|
||||
restore that matters: a `Cluster` with `bootstrap.recovery` from the barman object store into a
|
||||
scratch namespace, and check the recovered database actually contains data. The existing
|
||||
`conformance/tests/l2-behavioral/keycloak-db-dr` and `l3-fault-injection/dr-restore` tests are the
|
||||
harness for this; they must be re-run **against R2**, and their previous PROVEN state records MinIO,
|
||||
not R2.
|
||||
8. Only after 6 and 7 pass may this migration be described as done, and the ledger updated. Until
|
||||
then the honest status is "R2 accepts objects; recovery from R2 is untested".
|
||||
|
||||
---
|
||||
|
||||
## 6. R2-specific gotchas
|
||||
|
||||
**`region: auto`.** R2 accepts nothing else, and S3 clients require *some* region. Authored as
|
||||
`S3_BACKUP_REGION: "auto"`. This repo already learned it — `infra/tofu/backend.tf` carries
|
||||
`region = "auto" # R2 ignores region, but the S3 backend requires a value.`
|
||||
|
||||
**`checksumAlgorithm: ""` — do not delete this line.** `velero-plugin-for-aws`'s own
|
||||
`backupstoragelocation.md` for **v1.14.2** (the pinned version) documents:
|
||||
|
||||
> `checksumAlgorithm` — The checksum algorithm to use for uploading objects to S3. Supported values
|
||||
> "CRC32", "CRC32C", "SHA1", "SHA256". If the value is set as empty string "", no checksum will be
|
||||
> calculated and attached to the request headers. **Optional (defaults to "CRC32")**
|
||||
|
||||
So *omitting* the key means CRC32 **is** sent: `x-amz-sdk-checksum-algorithm` +
|
||||
`x-amz-checksum-crc32`. This repo has already measured that R2 rejects AWS checksums — that is exactly
|
||||
why `infra/tofu/backend.tf` needs `skip_s3_checksum = true # R2 rejects AWS trailing (streaming)
|
||||
checksums.` for the state bucket. The plugin's own compatibility table lists `checksumAlgorithm=""` as
|
||||
the remedy for the same failure on Ceph S3, Backblaze B2 and IBM COS. If backups start failing with
|
||||
**HTTP 400 `InvalidArgument`** after a plugin bump, look here first. `s3check` probes for this on
|
||||
purpose and names it.
|
||||
|
||||
**Path-style.** `s3ForcePathStyle: "true"` is authored and correct for R2 as well as MinIO. Never put
|
||||
the bucket in the hostname.
|
||||
|
||||
**No bucket auto-creation.** Velero does not create buckets and the token deliberately cannot. A
|
||||
missing bucket is a `NoSuchBucket` at runtime; `s3check` catches it first.
|
||||
|
||||
**Token scope is per-bucket.** A token scoped only to `velero` authenticates fine and then 403s on
|
||||
`cnpg` — the two consumers fail independently and quietly. `s3check` proves both.
|
||||
|
||||
**Both clusters share one endpoint.** Isolation is by path only (`VELERO_BACKUP_PREFIX`,
|
||||
`CNPG_BACKUP_PATH`), because R2 is one environment-independent URL. Those two knobs are load-bearing:
|
||||
without them prod would adopt and TTL-delete dev's Velero backups, and both clusters' identically-named
|
||||
CNPG Clusters would interleave their WAL into one Barman stream, destroying PITR for both.
|
||||
|
||||
---
|
||||
|
||||
## 7. What is proven, and what is not
|
||||
|
||||
**Proven (2026-09-05, WS-BACKUP, against the live dev cluster and its MinIO at
|
||||
`http://192.168.1.9:9000`):**
|
||||
|
||||
- the rewritten `s3check` itself: 7 assertions per credential path, both credential shapes, from
|
||||
in-cluster pods — credential load, bucket list, PUT/GET/byte-compare/DELETE/confirm-gone, no object
|
||||
and no pod left behind;
|
||||
- its failure paths: the `TBD` placeholder is refused; an unreachable endpoint (`:9099`) produces a
|
||||
named no-egress failure and exit 1, not a pass;
|
||||
- MinIO accepts a checksum-bearing PUT, i.e. the trap is a *non-*MinIO problem, and
|
||||
`checksumAlgorithm: ""` is harmless there.
|
||||
|
||||
**Established from the pinned versions (documentation and image inspection, not measured against R2):**
|
||||
|
||||
- `velero-plugin-for-aws` **v1.14.2** pairs with Velero **v1.18.x** (upstream compatibility table) and
|
||||
defaults `checksumAlgorithm` to `CRC32` — so the authored `""` is load-bearing, not decoration.
|
||||
- That plugin's "Non-AWS S3 compatible provider known issues with plugin v1.14.x" table **does not
|
||||
mention Cloudflare R2 at all** — neither as working nor as broken. There is no upstream statement to
|
||||
lean on in either direction.
|
||||
- **The PVC-data path is a different S3 client.** `checksumAlgorithm` is a BackupStorageLocation
|
||||
*object-store plugin* setting. Velero's File System Backup uploads through the unified repository's
|
||||
kopia S3 backend (`pkg/repository/udmrepo/kopialib/backend/s3.go` in Velero v1.18.0), which
|
||||
references only endpoint/region-style options and has **no checksum knob at all**. So R2
|
||||
compatibility for backup *metadata* and for *volume data* are two separate questions, and only step
|
||||
5 of §5 answers the second.
|
||||
- **CNPG/barman is likely clear of the checksum trap, by version.** Inspected inside the live
|
||||
`ghcr.io/cloudnative-pg/postgresql:16.4` image on dev: `barman-cloud-wal-archive 3.11.1`,
|
||||
`botocore 1.35.60`, `boto3 1.35.60`. That botocore predates the 1.36 line in which botocore began
|
||||
attaching CRC32 request checksums by default (the change that introduced
|
||||
`AWS_REQUEST_CHECKSUM_CALCULATION`), so this image should not send them.
|
||||
**Forward risk:** barman-cloud exposes **no** `checksumAlgorithm` equivalent. If the CNPG image is
|
||||
ever bumped to one carrying botocore ≥ 1.36, WAL archiving to R2 could start failing with 400
|
||||
`InvalidArgument` and there is no config knob to turn it off — only `AWS_REQUEST_CHECKSUM_CALCULATION=when_required`
|
||||
in the Pod environment, which CNPG does not currently set. Treat a CNPG image bump as a change that
|
||||
must be re-verified against R2.
|
||||
|
||||
**Not proven, at all:**
|
||||
|
||||
- that Velero can write a usable backup to R2 (metadata or kopia volume data);
|
||||
- that CNPG/barman can archive WAL to R2 and that a `bootstrap.recovery` from R2 succeeds;
|
||||
- that a restore from R2 yields correct data. R-STATE-03's PROVEN state is **MinIO-only**.
|
||||
|
||||
---
|
||||
|
||||
## 8. Going back to MinIO
|
||||
|
||||
The MinIO copy is **not deleted** and stays restorable. To read pre-migration backups:
|
||||
|
||||
1. `clusters/base/cluster-vars.yaml` → `S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"`
|
||||
2. `VELERO_BACKUP_PREFIX` → `""` (the MinIO backups are at the **bucket root**, not under `dev/`) and
|
||||
`CNPG_BACKUP_PATH` → `s3://cnpg` (Barman objects are at `s3://cnpg/<clusterName>/`).
|
||||
3. Restore the MinIO credential into the four Secrets. `platform/backup/minio-credentials.sops.yaml`
|
||||
still holds it: `sops -d --extract '["stringData"]["ACCESS_KEY_ID"]'` it and feed the two values to
|
||||
`platform/backup/set-r2-credentials.sh` (the script cares about the *shape*, not the provider).
|
||||
4. `S3_BACKUP_REGION` may stay `auto` — MinIO ignores the region.
|
||||
5. **Dev only.** Prod cannot reach that MinIO at all (§Why R2); pointing prod back is not an option.
|
||||
|
||||
Reverting for a restore and then returning to R2 costs another PITR reset. Prefer restoring from MinIO
|
||||
into a *scratch* namespace on dev with a second, temporary BackupStorageLocation over flipping the
|
||||
whole cluster's endpoint back and forth.
|
||||
|
||||
---
|
||||
|
||||
## 9. Merging this resets dev's PITR window
|
||||
|
||||
Say it out loud before merging, because it is not recoverable by retrying:
|
||||
|
||||
- Dev's R2 prefix starts **empty**. Velero writes a **fresh kopia repository** — the first backup is a
|
||||
full upload of every included PVC, and no earlier restore point exists in R2.
|
||||
- Every CNPG Cluster takes **new base backups** (`ScheduledBackup … immediate: true`), and the 30-day
|
||||
Barman retention window (`retentionPolicy: "30d"`) rebuilds **from zero**. For roughly a month, dev's
|
||||
point-in-time recovery reaches back only as far as the merge.
|
||||
- The MinIO copy still holds the old window (§8), but reaching it means reverting the endpoint — so it
|
||||
is a deliberate excursion, not a fallback that happens automatically.
|
||||
|
||||
**Do not merge this while dev's proven MinIO restore is the only recovery path you are relying on.**
|
||||
The same applies to prod the day it first backs up anywhere.
|
||||
@@ -102,12 +102,10 @@ check runs instead of silently skipping.
|
||||
> `KUBECONFIG` to `dev.kubeconfig`, leaving an undocumented — and misleadingly named —
|
||||
> `DEV_KUBECONFIG=<prod path>` as the only route to prod. Select with **`CLUSTER=dev|prod`**;
|
||||
> **dev remains the default**, so every existing invocation means what it always meant. Inside the
|
||||
> container `KUBECONFIG` is a chain of a tiny per-invocation selector file over a merged config
|
||||
> carrying both clusters as contexts **`k3s-dev`** and **`k3s-prod`** (never `default`), regenerated
|
||||
> each run by [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
|
||||
> `infra/ansible/.cache/clusters.kubeconfig`. The merged file deliberately carries **no**
|
||||
> `current-context`: it is shared by every container, and stamping the current cluster into it made
|
||||
> concurrent `CLUSTER=dev` and `CLUSTER=prod` runs silently swap clusters mid-command. Naming matters: k3s writes *both* source kubeconfigs
|
||||
> container `KUBECONFIG` is a merged config carrying both clusters as contexts **`k3s-dev`** and
|
||||
> **`k3s-prod`** (never `default`), regenerated each run by
|
||||
> [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
|
||||
> `infra/ansible/.cache/clusters.kubeconfig`. Naming matters: k3s writes *both* source kubeconfigs
|
||||
> with context, cluster and user all called `default`, and kubectl's merge keeps the first
|
||||
> duplicate — so a plain `KUBECONFIG=dev:prod` would silently resolve `default` to **dev** while you
|
||||
> believed you were on prod. Requesting a cluster whose kubeconfig is absent is a hard error, never
|
||||
|
||||
+370
-137
@@ -10,6 +10,11 @@ hardware until the blockers in §7 are closed and `make -C ci gate` is green.
|
||||
> **R-ENV-01/02/03** — the very requirements this document serves. We proceed with prep work
|
||||
> knowingly; see `DECISIONS-LOG.md` (2026-08-31 Phase-7 entry).
|
||||
|
||||
> **Looking for the commands?** This document is the *decision record* — why prod is built the way it
|
||||
> is, what was measured, and which hypotheses are dead. The **executable sequence** lives in
|
||||
> [`PROD-STANDUP.md`](PROD-STANDUP.md): every command, in order, driven by `bootstrap/standup.sh`.
|
||||
> Start there; come back here for the reasoning behind any step.
|
||||
|
||||
Prod is **3× mini-PC bare metal** ([spec docs/02](../../home-network-k3s/docs/02-environments-and-provisioning.md)).
|
||||
The dev path (OpenTofu → Proxmox VM → cloud-init) **does not apply**: there is no hypervisor API to
|
||||
call. Stage 1 "Create" is a human imaging a box. Stages 2–4 (baseline, cluster-join, manage) reuse
|
||||
@@ -28,9 +33,9 @@ Fill a row per node as it lands. Node 1 is as-found on 2026-08-31.
|
||||
| Model | HP EliteDesk 800 G5 Desktop Mini | HP EliteDesk 800 G5 Desktop Mini | HP EliteDesk 800 G5 Desktop Mini |
|
||||
| CPU | i7-9700T, 8c/8t, VT-x | i7-9700T, 8c/8t, VT-x | i7-9700T, 8c/8t, VT-x |
|
||||
| RAM | 31 GiB (2×16 GB Micron) | **2.6 GiB usable of 16 GB — see §0b** | 31.1 GiB usable of 32 GB (2×16 GB Crucial `CT16G4SFD8266.C16FE`, **a matched pair**) |
|
||||
| Disks | 1× 466 GB SATA SSD (`Samsung SSD 850 EVO 500GB`) — **replaced 2026-09-07, was `C400-MTFDDAC512M` (~2011); see §0c** | 1× 477 GB **NVMe** (`PM9A1 Samsung`) | 1× 466 GB SATA SSD (`WDC WDS500G2B0A-00SM50`) |
|
||||
| Drive health | PASSED, **0 reallocated / 0 grown bad blocks**, `Wear_Leveling_Count` raw 106, but **77,047 power-on hours** (~8.8 y) over only 219 power cycles | **96% life**, 2142 h, PASSED | **100% life**, 3213 h, PASSED, 0 grown bad blocks / 0 erase / 0 program fails, 0 CRC |
|
||||
| OS | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** |
|
||||
| Disks | 1× 477 GB SATA SSD (`C400-MTFDDAC512M`, ~2011) | 1× 477 GB **NVMe** (`PM9A1 Samsung`) | 1× 466 GB SATA SSD (`WDC WDS500G2B0A-00SM50`) |
|
||||
| Drive health | 69% life, 15 grown bad blocks, 187 erase fails | **96% life**, 2142 h, PASSED | **100% life**, 3213 h, PASSED, 0 grown bad blocks / 0 erase / 0 program fails, 0 CRC |
|
||||
| OS | Debian 13 (trixie), kernel 6.12 | Debian 13 (trixie), kernel 6.12 | Debian 13 (trixie), kernel 6.12.107+deb13-amd64 |
|
||||
| Firmware | HP R21 Ver. 02.26.00 | HP R21 Ver. 02.26.00 | HP R21 Ver. 02.26.00 |
|
||||
| `/dev/kvm` | present | present | present |
|
||||
| Serial | MXL1052NGN | MXL0073949 | 8CC015307N |
|
||||
@@ -48,47 +53,6 @@ modern NVMe; node 3 a third distinct model, a WD Blue 3D SATA SSD in excellent h
|
||||
an identical disk layout across all three because Longhorn replica placement assumes symmetry — so
|
||||
this is a live input to the disk decision, not a footnote. Node 2's NVMe is the drive node 1 should
|
||||
have had; node 3 does not have it either. Node 3 is also the only node with **two matched DIMMs**
|
||||
|
||||
### §0c — node 1's drive was replaced (2026-09-07)
|
||||
|
||||
**The paragraph above is history, not current state.** On 2026-09-07 node 1's kubelet stopped
|
||||
posting status while the host stayed up and SSH-able. Root cause was **etcd starved of disk I/O**,
|
||||
not memory: `slow fdatasync` at 3.5–6.7 s against an expected 1 s, `apply request took too long` at
|
||||
8.3 s against 100 ms, then `etcdserver: too many requests`. No OOM, MCE or hung task in the kernel
|
||||
log. All three nodes place etcd's WAL (`/data/k3s`) and every Longhorn replica (`/data/longhorn`) on
|
||||
the SAME partition; node 1's was the 2011 C400 with 61,440 reallocated sectors and ~31% of its
|
||||
endurance consumed, and its `/proc/pressure/io` `full avg300` ran **11.36** against ~1% steady state
|
||||
on the other two. The failure is self-reinforcing: the node dropping out degrades Longhorn volumes,
|
||||
whose rebuilds add write load to the very disk etcd needs.
|
||||
|
||||
The operator replaced the drive with a **Samsung SSD 850 EVO 500GB** and reinstalled Debian.
|
||||
Measured under the same load class — an active 18-replica Longhorn rebuild, with node 1 as the
|
||||
*write target*, i.e. the heavier side:
|
||||
|
||||
| node | drive | `full avg300` during rebuild |
|
||||
|---|---|---|
|
||||
| `k3s-prod-1` | Samsung 850 EVO (new) | **3.79** |
|
||||
| `k3s-prod-2` | Samsung PM9A1 NVMe | 0.33 |
|
||||
| `k3s-prod-3` | WDC WDS500G2B0A | 2.47 |
|
||||
| *`k3s-prod-1` before* | *C400-MTFDDAC512MAM* | ***11.36*** |
|
||||
|
||||
Roughly a **3× improvement**, moving node 1 from ~4× worse than its peers into the same band as
|
||||
node 3, and it stayed `Ready` throughout. **The 850 EVO is not new**: 77,047 power-on hours over 219
|
||||
power cycles — it evidently idled in an always-on system — but its wear tells the opposite story to
|
||||
the C400's (0 reallocated vs 61,440; wear-leveling 106 vs 949).
|
||||
|
||||
**§2's exit condition is NOT met by this replacement.** etcd and Longhorn still share one SATA
|
||||
spindle. The contention is reduced, not eliminated; giving etcd its own device remains the fix, and
|
||||
node 1 still has exactly one disk. The three nodes also remain asymmetric — now 850 EVO / PM9A1
|
||||
NVMe / WD Blue.
|
||||
|
||||
The rebuild itself surfaced two things worth carrying forward. Longhorn refused the new disk with
|
||||
`DiskFilesystemChanged` — "record diskUUID doesn't match the one on the disk" — because its node CR
|
||||
still held the old drive's `diskUUID`; the remedy is disable → remove the disk entry → re-add, in
|
||||
that order (the validating webhook enforces it). And the Ansible inventory still listed node 1 in
|
||||
`[k3s_prod_primary]`, which decides both which play gets `--cluster-init` *and* what `k3s_api_host`
|
||||
resolves to — so re-provisioning it unchanged would have pointed a rebuilt, empty node at itself and
|
||||
founded a second cluster beside the live one. The founder now sits on node 2.
|
||||
(2× Crucial 2Rx8 DDR4-2666) and the only one with its **full** RAM complement of 32 GB actually
|
||||
usable. **That is luck, not immunity:** node 3's firmware publishes an EFI Memory Attributes Table
|
||||
too (`MEMATTR=0xbeece018`), the kernel simply parsed it cleanly on this boot, so node 3 is exposed to
|
||||
@@ -820,74 +784,6 @@ disk each, already fully partitioned, no free slot verified populated.
|
||||
physically) and flip `node_baseline_etcd_source` back to `device` — the role still
|
||||
supports that path unchanged, and it is the dev shape.
|
||||
|
||||
#### THE MEASUREMENT CAME BACK — 2026-09-07. It does not hold on node 1.
|
||||
|
||||
Not under a deliberate test: in production, 2 days 7 hours after the cluster came up.
|
||||
**06:05:21Z `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) while the
|
||||
host stayed up, pingable and SSH-able. From the k3s journal:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task in the kernel log — **this is not §0b and not memory**. Recovery
|
||||
required a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt, and I/O pressure
|
||||
only fell once the rebuilds finished. The failure is **self-reinforcing**: the node dropping
|
||||
out degrades volumes → Longhorn rebuilds → more write load on the disk etcd needs → etcd
|
||||
stalls further.
|
||||
|
||||
`/proc/pressure/io` **full avg300**, all three nodes, during the incident:
|
||||
|
||||
| node | device | drive | IO PSI full avg300 |
|
||||
|---|---|---|---|
|
||||
| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM (2011 consumer SATA SSD, no PLP) | **11.36** |
|
||||
| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
|
||||
| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
|
||||
|
||||
The prediction in the paragraph above — "every prod node is SSD or NVMe end-to-end, where
|
||||
concurrent rebuild I/O costs latency rather than collapsing it" — was **right about node 2 and
|
||||
wrong about node 1**. A 2011 consumer SATA SSD with 61,440 reallocated sectors, 31% of its
|
||||
endurance consumed and no power-loss protection is not the "SSD" that reasoning assumed. §0's
|
||||
"three nodes, three different drive models" is the fault line, exactly where §0 said it would
|
||||
be.
|
||||
|
||||
**The exit condition is therefore OPEN and the second bullet applies.** Replacing node 1's
|
||||
drive, or giving etcd its own device, is the fix, and it is the operator's.
|
||||
|
||||
Pending that, three SOFTWARE mitigations shipped on 2026-09-07 (branch
|
||||
`fix/etcd-io-priority`). **None of them is the fix**, and each says so where it lives:
|
||||
|
||||
1. **Observability first** — `NodeIOPressureHigh` / `NodeIOPressureCritical` in
|
||||
`platform/monitoring/alerts.yaml`, plus an I/O row on the node-pressure dashboard. The
|
||||
stack alerted on CPU and memory PSI and had **no I/O alert at all**, so the first signal
|
||||
of this outage was a node already out of the cluster. Thresholds (5% / 10% full IO PSI)
|
||||
come from the numbers in the table above. Note the etcd histograms this section's first
|
||||
bullet asks for are **not available**: kube-prometheus-stack's `kubeEtcd` scrape job is
|
||||
deliberately disabled for k3s (`platform/monitoring/kube-prometheus-stack.yaml`), so PSI
|
||||
is the signal that actually exists. Enabling the etcd scrape would make that bullet
|
||||
directly measurable and is worth doing.
|
||||
2. **Longhorn concurrency ceilings** — `platform/longhorn/helmrelease.yaml`. Honestly:
|
||||
`concurrentReplicaRebuildPerNodeLimit: 1` was **already in force during this outage** and
|
||||
held; the 18 rebuilds ran one at a time and the node stalled anyway. Longhorn offers no
|
||||
further rebuild lever for the v1 data engine (`replicaRebuildingBandwidthLimit` is
|
||||
v2/SPDK only; `rebuildConcurrentSyncLimit` is already at its floor of 1). What was
|
||||
tightened is every *other* Longhorn I/O source — restore, snapshot purge, backup, engine
|
||||
upgrade — each of which defaulted to 5 concurrent operations per node and each of which
|
||||
could reproduce this on its own.
|
||||
3. **Block-layer arbitration** — `roles/etcd_io_priority`, enabled for `[k3s_prod]`, applied
|
||||
only when the operator runs `playbooks/etcd-io-priority.yml`. BFQ + `IOWeight` so etcd's
|
||||
cgroup outweighs `kubepods.slice`. Its README is explicit that most of the obvious knobs
|
||||
here (`IOSchedulingClass`, `io.latency`, bare `IOWeight`) are **inert on these nodes**,
|
||||
and that even this one cannot arbitrate the ext4 journal both tenants share.
|
||||
|
||||
The last point is the one that matters for this section: **no cgroup or scheduler control
|
||||
separates two tenants on one filesystem.** etcd's `fdatasync` waits on a jbd2 journal commit,
|
||||
jbd2 is a kernel thread in the root cgroup, and Longhorn's writes dirty the same journal. The
|
||||
software mitigations raise the load level at which this happens. They do not remove it.
|
||||
|
||||
**~~Known, separate consequence — `/var` is only 12 GiB.~~ RESOLVED 2026-09-04 by moving
|
||||
k3s's data dir.** This was recorded here on 2026-09-03 as a capacity item "to watch or
|
||||
repartition". Re-measuring it against dev turned it from a watch item into a blocker: the
|
||||
@@ -1369,6 +1265,45 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
`inventory/group_vars/k3s_prod/vars.yml` overrides it to `clusters/prod` — the same
|
||||
mechanism as `kube_vip_address` / `k3s_data_dir` / `cluster_kubeconfig_name`. Dev has no
|
||||
override anywhere, so dev resolves `clusters/dev` exactly as before.
|
||||
- [x] **The Kyverno CRDs had to be applied BY HAND, and nothing in the repo did it** — AUTHORED
|
||||
2026-09-04 (`bootstrap/flux/preapply-crds.sh`), **not yet run against prod**. This is a hard
|
||||
**prerequisite of a reconcilable cluster**, not a nice-to-have: `platform/kyverno/admission/
|
||||
helmrelease.yaml` sets `crds: {install: false}` — deliberately, because the CRD bundle is
|
||||
**5,636,155 bytes across 22 CRDs** at chart 3.9.0 (measured 2026-09-04 with `helm template
|
||||
kyverno kyverno --repo https://kyverno.github.io/kyverno/ --version 3.9.x --include-crds`), far
|
||||
past kubectl's 256 KiB `last-applied-configuration` annotation limit, so Helm's client-side apply
|
||||
fails `metadata.annotations: Too long` and **stalls** the HelmRelease. The consequence nobody
|
||||
had automated: with CRD install off, **nothing installs those CRDs at all**. On a cold cluster
|
||||
Kyverno's Deployments come up with no API types, `platform` never goes Ready, and all 9
|
||||
downstream Kustomizations stall behind it. Dev only ever had them because the command was once
|
||||
run in somebody's shell — the exact undocumented manual step this project exists to delete.
|
||||
- **The fix, in git:** `bootstrap/flux/bootstrap.sh` now runs `./preapply-crds.sh` as step
|
||||
**[2/4]** — after `flux install`, before `kubectl apply -k ${FLUX_ENV_PATH}`. It renders the
|
||||
chart, keeps only `kind: CustomResourceDefinition` documents, applies them
|
||||
`--server-side --force-conflicts` (no annotation, so no size limit), then waits for
|
||||
`condition=Established`. The chart name, version constraint, repo URL and namespace are
|
||||
**derived from that same HelmRelease/HelmRepository** — there is no second copy of the version
|
||||
to drift, and a Renovate bump of the one pinned line carries this step with it.
|
||||
- **No environment fork (R-ENV-02):** the script names no environment and takes no overlay
|
||||
input; it reads repo files and acts on `$KUBECONFIG`. Dev and prod run the identical step, so
|
||||
`roles/flux_bootstrap` needed **no change** — it already calls `bootstrap.sh`, the single
|
||||
entrypoint.
|
||||
- **It cannot silently skip.** Three guards, and deliberately no opt-out flag: a derived
|
||||
coordinate that is empty/`null` aborts; a render that yields **0** CRDs aborts (an empty
|
||||
stream would otherwise `kubectl apply` cleanly and exit 0 — the silent-success failure mode
|
||||
that has bitten this repo repeatedly); and every applied CRD must reach `Established`.
|
||||
- **Toolchain:** needs `helm` and `yq` on PATH in addition to `flux`/`kubectl` — both are baked
|
||||
into `infra/devcontainer` (`HELM_VERSION=3.16.0`, `YQ_VERSION=4.44.6`), and `bootstrap.sh`
|
||||
now fails at step 0 if either is missing. It also needs **outbound network to
|
||||
`kyverno.github.io`** from wherever the play runs (the Ansible role delegates to localhost).
|
||||
- **PM to verify in the container** (the authoring agent ran it only with a stubbed `kubectl`,
|
||||
never against a cluster): pinned `helm` 3.16.0 renders the same 22 CRDs, and the run is a
|
||||
clean no-op the second time.
|
||||
- **Unblocking the already-bootstrapped prod cluster** does not need a full re-bootstrap:
|
||||
```bash
|
||||
# inside infra/devcontainer, KUBECONFIG=.cache/prod.kubeconfig
|
||||
bash bootstrap/flux/preapply-crds.sh
|
||||
```
|
||||
- [ ] Confirm prod's backup target: dev's S3 (`versitygw` on the Proxmox host, `192.168.1.9`) is on
|
||||
the dev side of the network. Off-cluster backup is DR-critical (R-STATE-03).
|
||||
|
||||
@@ -1381,7 +1316,11 @@ single-server-etcd window, and avoids a later member-add dance.
|
||||
1. Per-node §1–§5 on each of the three boxes.
|
||||
2. §7 repo work lands on `main` with the gate green.
|
||||
3. `node-baseline` → `k3s` (all three) → `kube-vip` → `cilium`.
|
||||
4. `flux bootstrap ... --path=clusters/prod`.
|
||||
4. `flux bootstrap ... --path=clusters/prod` — i.e. `playbooks/flux-bootstrap.yml -e k3s_hosts=k3s_prod`,
|
||||
which runs `bootstrap/flux/bootstrap.sh` with `FLUX_ENV_PATH=clusters/prod`. That script's step
|
||||
**[2/4]** pre-applies the 22 Kyverno CRDs server-side; **without it `platform` cannot go Ready**
|
||||
and the 9 tiers below it never reconcile (§7). It needs `helm` + `yq` on PATH and outbound network
|
||||
to the chart repo.
|
||||
5. Conformance against prod (§9).
|
||||
|
||||
## 9. Exit criteria (Phase 7)
|
||||
@@ -1397,7 +1336,122 @@ single-server-etcd window, and avoids a later member-add dance.
|
||||
distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and
|
||||
prod's own Keycloak realm + per-service client secrets.
|
||||
|
||||
## 10. DNS/TLS environment split — **the 2026-09-04 incident, and phase 1 of the fix**
|
||||
## 9a. PROD BRING-UP STATE (2026-09-04) — what came up, and the TWO blockers that need the operator
|
||||
|
||||
Prod's `platform` tier was unsuspended after the Kyverno CRDs were pre-applied (see §7). Result:
|
||||
**8 of 9 HelmReleases Ready, 76/76 pods Running** — cert-manager, CNPG, Cilium, Longhorn,
|
||||
kube-prometheus-stack, Loki, Alloy, external-dns. `platform` itself is **not** Ready, and the 10
|
||||
downstream tiers remain blocked behind it. Two distinct problems, both needing an operator decision.
|
||||
|
||||
### The CRD pre-apply itself: WORKED, and is now proven against a live API server
|
||||
|
||||
22 CRDs applied server-side and all reached `Established`; a second run was a clean no-op (`rc=0`),
|
||||
so the idempotency claim is proven, not asserted. Reproduced under the **pinned** toolchain
|
||||
(helm 3.16.0, yq 4.44.6) matching an independent helm 3.20.0 run byte-for-byte: `3.9.x` → 3.9.0,
|
||||
22 CRDs, 5,636,155 bytes.
|
||||
|
||||
**The CRDs were the real cause of a 4½-hour crash loop.** Kyverno's four Deployments had **55
|
||||
restarts each** and stabilised within minutes of the CRD apply — its controllers were coming up with
|
||||
no API types to watch. That is the failure mode §7 predicted, observed.
|
||||
|
||||
### BLOCKER 1 — Kyverno's Helm release is stuck in `uninstalling`
|
||||
|
||||
Diagnosis chain, from the HelmRelease conditions:
|
||||
|
||||
```
|
||||
Released: InstallFailed — Helm install failed ...: context deadline exceeded
|
||||
Remediated: UninstallFailed — Helm uninstall remediation ... timed out waiting for the condition
|
||||
Ready: StateError — unable to determine state for release with status 'uninstalling'
|
||||
```
|
||||
|
||||
The install timed out **because** the pods were crash-looping without CRDs. `install.remediation.
|
||||
retries: 3` then triggered an uninstall, and that uninstall itself timed out — leaving
|
||||
`sh.helm.release.v1.kyverno.v1` with `status=uninstalling`. helm-controller cannot proceed from that
|
||||
state, so `platform` (which has `wait: true`) never goes Ready and all 10 tiers below it stay blocked.
|
||||
|
||||
**The Deployments are healthy and at the right chart version** (kyverno-3.9.0), so this is purely a
|
||||
stale Helm bookkeeping record, not a broken workload.
|
||||
|
||||
**Remediation requires a destructive command and was deliberately NOT run unattended:**
|
||||
|
||||
```
|
||||
helm uninstall kyverno -n kyverno --no-hooks --wait=false # clear the stuck release
|
||||
flux reconcile helmrelease kyverno -n kyverno --force # let Flux install cold, CRDs now present
|
||||
```
|
||||
|
||||
This is safe on prod specifically because prod carries no real workloads yet, and a clean cold
|
||||
install is the stronger outcome: it *proves* the bootstrap path this repo now automates. Note the
|
||||
`failurePolicy: Fail` webhooks (`kyverno-policy-*`, `kyverno-exception-*`, `kyverno-cleanup-*`,
|
||||
`kyverno-global-context-*`) gate only **Kyverno's own** resource types, so a brief absence does not
|
||||
block ordinary workload admission. The pre-applied CRDs live **outside** the Helm release and
|
||||
survive the uninstall, which is exactly why the pre-apply had to be a separate step.
|
||||
|
||||
### BLOCKER 2 — prod's external-dns is an ARMED footgun: the installed release predates the fix
|
||||
|
||||
**This is the more dangerous of the two.** Prod's live external-dns Deployment renders:
|
||||
|
||||
```
|
||||
--policy=sync --registry=txt --domain-filter=keithjackson.dev --provider=cloudflare
|
||||
```
|
||||
|
||||
**and NO `--txt-owner-id`.** Absent that flag external-dns falls back to the chart default `default`
|
||||
— which is precisely the owner under which **all 36 of dev's TXT registry entries are recorded**.
|
||||
Scaling this Deployment to 1 would make prod claim dev's records and, under `policy: sync`,
|
||||
**delete them: the 2026-09-04 incident, exactly.**
|
||||
|
||||
The repo is CORRECT and is not the problem. `platform/externaldns/helmrelease.yaml` sets
|
||||
`txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}"`, prod's applied `cluster-vars` resolves it to `prod`, and
|
||||
the **live HelmRelease `spec.values` carries `txtOwnerId: "prod"`**. Verified against the real chart:
|
||||
`helm template external-dns --version 1.21.1` with `txtOwnerId: prod` renders `--txt-owner-id=prod`,
|
||||
so the values key is right and an upgrade *would* fix it.
|
||||
|
||||
**The fault is Helm/Flux state, not authoring.** `helm get values external-dns` shows the INSTALLED
|
||||
release carries `policy` and `domainFilters` but **no `txtOwnerId`** — it predates `c8c1d29`. Yet
|
||||
helm-controller reports `Ready=True`, `observedGeneration=3`, and a history entry whose
|
||||
`configDigest` **equals** `lastAttemptedConfigDigest`, i.e. **it believes it has already deployed the
|
||||
current config.** So it will never upgrade on its own, and the stale release persists indefinitely.
|
||||
|
||||
**The only thing preventing the incident was `replicas=0` — manual drift from the incident recovery,
|
||||
which any Helm upgrade would reset to the chart default of 1.** That is not a guard; it is luck.
|
||||
|
||||
Containment applied (reversible, and it costs nothing because prod publishes no hostnames yet — the
|
||||
app tiers are suspended): **`flux suspend helmrelease external-dns -n external-dns`**, with
|
||||
`replicas` left at 0. Flux can now neither upgrade the release nor reset the replica count.
|
||||
|
||||
**Verified remediation, in this order — the ordering is the whole point:**
|
||||
|
||||
1. Force the upgrade so the release picks up `txtOwnerId`, but **do not let the pod run before the
|
||||
flag is confirmed present.** The naive `resume` + `reconcile --force` resets `replicas` to 1 and
|
||||
starts the pod in the same operation, so there is no window in which to check — a race against a
|
||||
process whose failure mode is deleting dev's DNS.
|
||||
2. Confirm the live Deployment args contain **`--txt-owner-id=prod`**.
|
||||
3. Only then allow it to scale to 1, and immediately diff the Cloudflare zone against a
|
||||
pre-recorded snapshot.
|
||||
|
||||
**Dev was verified unharmed throughout this work:** the Cloudflare zone was snapshotted before
|
||||
unsuspending `platform` and re-queried after — **57 records, byte-for-byte identical, all 36 TXT
|
||||
entries still `owner=default`.** Snapshots are taken against the **Cloudflare API**, never a local
|
||||
resolver (a pfSense override produced a false healthy reading during the incident).
|
||||
|
||||
> ### THE APEX INVERSION IS NOT ON THIS BRANCH
|
||||
>
|
||||
> Sections 10 and 10a below describe the **target** DNS layout — prod on the bare apex, dev on
|
||||
> `dev.home.` — and the ordered runbook for getting there. **That change is deliberately not authored
|
||||
> on this branch.** It is one commit, `9137bbd`, kept on branch **`migration/dns-apex-inversion`**,
|
||||
> because merging it *is* dev's migration: `clusters/dev/sync.yaml` reconciles `./clusters/dev` from
|
||||
> `main` every 10 minutes, so a merge executes the cutover on the live dev cluster unattended.
|
||||
>
|
||||
> On this branch **prod is authored on `prod.home.keithjackson.dev`**, which is what prod actually
|
||||
> runs. Read what follows as the plan, not as a description of the current authored state. The
|
||||
> inversion is applied as step **D3** of the §10a runbook, operator present.
|
||||
|
||||
## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture**
|
||||
|
||||
> **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's
|
||||
> `platform` tier and its 10 downstream tiers are SUSPENDED. Dev is **live on the apex**. The
|
||||
> ordered cutover is **§10a**, and merging the authored change to `main` is step **D3** of it,
|
||||
> not step one — dev self-reconciles `main`, so a merge outside the runbook window executes
|
||||
> the migration unattended.
|
||||
|
||||
### What happened
|
||||
|
||||
@@ -1518,26 +1572,205 @@ Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working S
|
||||
Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change.
|
||||
Required, roughly in order:
|
||||
|
||||
1. **`BASE_DOMAIN` → `dev.home.keithjackson.dev`** in `clusters/dev`, plus `k3s_base_domain` for the
|
||||
dev inventory group.
|
||||
2. **A new `*.dev.home.keithjackson.dev` wildcard cert** issued and Ready *before* cutover.
|
||||
3. **Keycloak issuer change.** The realm's issuer URL is baked into every token; every OIDC client
|
||||
(`grafana`, `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`,
|
||||
`qbittorrent`, `radarr`, `sonarr`, and the `kubernetes` public client) needs its redirect URIs
|
||||
updated in the same change, and the k3s apiserver `oidc-issuer-url` with it — which means a
|
||||
**rolling k3s restart** on the dev servers.
|
||||
4. **kubectl SSO re-issue** for every user (issuer changed).
|
||||
5. **`gitea.dev.home.keithjackson.dev`** — this is the GitOps source URL in
|
||||
`clusters/dev/kustomization.yaml`. Flux must be able to resolve and trust the new host *before*
|
||||
the URL is switched, or dev stops reconciling and the change cannot be pushed to fix itself.
|
||||
Sequence this last, and keep the bootstrap Gitea available as the recovery path (docs/05).
|
||||
6. **`EXTERNALDNS_TXT_OWNER_ID` for dev.** Once dev's names move, its existing `default`-owned
|
||||
records are for names dev no longer serves; retire them deliberately rather than letting a
|
||||
renamed owner orphan them.
|
||||
7. **Apex CNAMEs** `<svc>.home.keithjackson.dev` → `<svc>.prod.home.keithjackson.dev`. This is what
|
||||
actually **moves user traffic to prod**, and it must not land before prod serves those services.
|
||||
8. **The L2 behavioural conformance tests hardcode `home.keithjackson.dev`** (`dns-exposure`,
|
||||
`dns-internal-resolution`, `gateway-ingress-reachable`, `oidc-sso-grafana`, `sso-forward-auth`,
|
||||
`trusted-tls`). They are test inputs, not reconciled manifests, so they were left alone in phase 1
|
||||
— but they will fail against prod and must be parameterized before §9's "suite passes green
|
||||
against the prod cluster" can be claimed.
|
||||
**(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root
|
||||
Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main`
|
||||
too. **A merge to `main` therefore *executes* this migration on the live dev cluster, unattended.**
|
||||
The authored change is a *target*, not a deployment. **Do not merge these files to `main` outside the
|
||||
runbook window.** Park them on a branch until the operator is at the keyboard and step **D1** has
|
||||
been done.
|
||||
|
||||
### D — dev vacates the apex (operator present, in this order)
|
||||
|
||||
- [ ] **D0. Snapshot the escape routes before touching anything.** Confirm a working break-glass
|
||||
**cert-based** admin kubeconfig for dev (not the OIDC one — OIDC breaks in D6). Confirm the
|
||||
retained bootstrap Gitea can be started (`bootstrap/gitea/bootstrap-gitea.sh`; dev's historical
|
||||
recovery host was `192.168.2.103:3000`, prod's current bootstrap is `192.168.2.132:3000`).
|
||||
Record dev's current Cloudflare record set (names + TXT registry owners) so D7 has a checklist
|
||||
and so an accidental deletion is detectable.
|
||||
- [ ] **D1. Break the automatic trigger.** Either `flux suspend kustomization flux-system -n
|
||||
flux-system` on dev (stops the root from self-applying the merge), **or** re-point dev's
|
||||
`GitRepository` url at the bootstrap Gitea for the window. Do this **before** the merge, not
|
||||
after. This is what converts "merging strands dev" into "merging is inert until you say go".
|
||||
- [ ] **D2. Publish and prove the new host set *while dev still serves the apex*.** The new names
|
||||
must exist before anything depends on them:
|
||||
- a `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev` **wildcard Certificate issued
|
||||
and `Ready`** (LE prod, DNS-01) — see the ACME note below before ordering it;
|
||||
- public DNS for `gitea.dev.home.keithjackson.dev` resolving to dev's Gateway VIP
|
||||
(192.168.1.53), **checked against Cloudflare / a public resolver, never through the pfSense
|
||||
override** — that mistake was made on 2026-09-04 and produced a false healthy reading;
|
||||
- `curl -sS https://gitea.dev.home.keithjackson.dev/api/v1/version` succeeding with a
|
||||
publicly-trusted cert from a host on the dev LAN.
|
||||
Until D2 is green, D3 is not safe.
|
||||
- [ ] **D3. Merge the authored change to `main`** (`clusters/base/cluster-vars.yaml`,
|
||||
`clusters/dev/`, `clusters/prod/`, the two Ansible files, `platform/externaldns`,
|
||||
`catalog/render.py`). With D1 in place nothing moves yet.
|
||||
- [ ] **D4. Release dev's reconciliation deliberately.** Resume the root Kustomization (or restore
|
||||
the in-cluster url, now pointing at `gitea.dev.home.keithjackson.dev`) and watch
|
||||
`flux get kustomizations -A` converge. Dev's Gateway listeners, HTTPRoutes, coredns-custom zone
|
||||
and Homepage links all move here. Dev's external-dns publishes the new record set under owner
|
||||
`dev`.
|
||||
- [ ] **D5. Keycloak: issuer + every client.** Dev's realm issuer becomes
|
||||
`https://keycloak.dev.home.keithjackson.dev/realms/homelab`. The issuer is baked into every
|
||||
token, and **~12 OIDC clients** need their redirect URIs moved in the same change: `grafana`,
|
||||
`gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, `qbittorrent`,
|
||||
`radarr`, `sonarr`, plus the `kubernetes` public client. Most are reconciled from
|
||||
`platform/keycloak/keycloak-realm-import.yaml` and move with D4; **headlamp's is the exception**
|
||||
— its issuer/callback live in a SOPS-encrypted Secret and must be re-encrypted by hand.
|
||||
- [ ] **D6. Rolling k3s restart + kubectl SSO re-issue.** `k3s_base_domain` now renders a different
|
||||
`oidc-issuer-url` into `/etc/rancher/k3s/config.yaml`, so the `k3s_server` role is **no longer
|
||||
a no-op on dev**: it rewrites the file and fires the Restart handler on all three servers. Run
|
||||
it **one server at a time**, confirming `kubectl get nodes` between each. Every user's OIDC
|
||||
kubeconfig is invalidated and must be re-issued; the cert-based admin kubeconfig keeps working
|
||||
throughout, which is what makes this step recoverable. **Do not reboot nodes** — restart the
|
||||
`k3s` service only.
|
||||
- [ ] **D7. Retire dev's old apex records.** They are owned by `default`, which no environment now
|
||||
claims, so nothing removes them automatically. Delete the `<svc>.home.keithjackson.dev` A
|
||||
records, the apex record, **and their TXT registry entries** (`external-dns-*` TXT), plus any
|
||||
stale `_acme-challenge.home.keithjackson.dev`. *Alternative, if you prefer external-dns to do
|
||||
its own cleanup:* set dev's `EXTERNALDNS_TXT_OWNER_ID` back to `default` for exactly one
|
||||
reconcile after D4 — `policy: sync` will then garbage-collect the apex set it no longer has
|
||||
sources for — and only then switch it to `dev`. That is two commits and needs watching; the
|
||||
direct deletion is preferred because it is verifiable in one place.
|
||||
- [ ] **D8. Verify the apex is EMPTY, against Cloudflare.** `dig +short <svc>.home.keithjackson.dev
|
||||
@1.1.1.1` returns nothing for every service name, and no TXT registry entry for those names
|
||||
remains. **This is the gate on the whole P sequence.** A pfSense/PiHole override is not
|
||||
evidence about public DNS.
|
||||
|
||||
### D0/D1 AS EXECUTED (2026-09-04) — and the self-reference that would have undone it
|
||||
|
||||
D1 was taken via the **repoint** option, not the suspend option, because it is strictly stronger:
|
||||
suspending only defers the strand, while repointing removes the dependency that causes it. Dev's
|
||||
`GitRepository` now reads **`http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git`,
|
||||
branch `dev-live`** — IP-addressed, plain HTTP, so it depends on **neither DNS nor TLS**, which is
|
||||
exactly the property needed during a DNS/TLS migration. **A merge to `main` is now inert for dev.**
|
||||
|
||||
**THE TRAP, and it is not obvious: a bare copy of `main@4809199` on the bootstrap Gitea would NOT
|
||||
have held.** The root Kustomization renders `./clusters/dev`, and `clusters/dev/kustomization.yaml`
|
||||
*contains the patch that sets `GitRepository.spec.url`*. So Flux would have fetched from the
|
||||
bootstrap Gitea, applied the manifests it found there, and those manifests would have set the url
|
||||
straight back to `gitea.home.keithjackson.dev` — reverting the repoint on the next reconcile and
|
||||
re-arming the strand. **The branch must declare its own source.** `dev-live` (`a4a3c2e`) therefore
|
||||
carries exactly one content change on top of `4809199`: `clusters/dev/kustomization.yaml`'s url +
|
||||
`ref.branch`. Proven by `kubectl kustomize clusters/dev` at both revisions differing in **exactly
|
||||
two lines** out of 357. The same property makes the revert self-sustaining in reverse, since
|
||||
`main@4809199` carries the `gitea.home…` url. Revert command: see `REVERT.txt` in the run scratchpad.
|
||||
|
||||
Verified after the cutover: **13/13 Kustomizations Ready** on `dev-live@a4a3c2e`, `cluster-vars`
|
||||
byte-identical to the D0 snapshot (27/27 keys, `BASE_DOMAIN` still `home.keithjackson.dev`),
|
||||
**18/18 Longhorn volumes attached+healthy**, all 16 HTTPRoute hostnames still on the apex,
|
||||
`wildcard-home` still Ready, and the Cloudflare record set re-queried and diffed **byte-for-byte
|
||||
against the D0 snapshot — identical** (57 records, 36 TXT all `owner=default`).
|
||||
|
||||
> **NEW DEPENDENCY, AND IT IS THE WEAK LINK.** `192.168.2.132` is a **DHCP lease on the
|
||||
> workstation's wifi** (`wlp0s20f3`, NetworkManager `ipv4.method=auto` — confirmed, not assumed). A
|
||||
> lease change, a wifi drop, or the workstation going offline **stalls dev's fetching**. The cluster
|
||||
> keeps running and serving; only reconciliation pauses, and the fix is a `kubectl patch` of the url.
|
||||
> Two consequences: (a) a **static reservation** for this host is worth having before any repeat of
|
||||
> this procedure, and (b) **keep the window short** — the exit is D4, which returns dev to its own
|
||||
> in-cluster Gitea at `gitea.dev.home.keithjackson.dev`.
|
||||
>
|
||||
> Pointing Flux at dev's Gitea **Service** instead (`gitea.gitea.svc.cluster.local:3000`) would drop
|
||||
> the DNS, TLS *and* workstation dependencies all at once, and is the better long-term shape. It was
|
||||
> **not** done here: namespace `gitea` carries a `default-deny` NetworkPolicy plus a
|
||||
> `gitea-allow-gateway-ingress` CiliumNetworkPolicy admitting **only** the Gateway, so
|
||||
> source-controller cannot reach the Service without a new authored policy. Worth doing deliberately,
|
||||
> not mid-migration.
|
||||
|
||||
### P — prod takes the apex
|
||||
|
||||
- [ ] **P1. Unsuspend prod**, in dependency order: `flux resume kustomization platform`, then the
|
||||
10 downstream tiers.
|
||||
**CORRECTION (2026-09-04) — the D8 gate is on prod taking the APEX, not on prod coming up.**
|
||||
This step originally read "only now unsuspend prod", which silently assumed prod would be
|
||||
serving `*.home.keithjackson.dev`. It is not: prod runs revision `c8c1d29`, where
|
||||
`BASE_DOMAIN=prod.home.keithjackson.dev` and `EXTERNALDNS_TXT_OWNER_ID=prod`. Prod's names are
|
||||
therefore a **disjoint subtree** under a **distinct registry owner**, so neither incident mode
|
||||
of §10a applies: it cannot claim dev's `default`-owned records, and its wildcard
|
||||
`*.prod.home.keithjackson.dev` is a **different name set** from dev's, so it has its own
|
||||
Let's Encrypt duplicate-certificate bucket and races no `_acme-challenge` with dev.
|
||||
**Prod may therefore be brought up in full while dev still holds the apex** — and should be,
|
||||
because it proves the entire platform stack end-to-end *before* the apex flip, turning the
|
||||
cutover into a DNS/TLS change on a known-good cluster instead of a bring-up and a cutover at
|
||||
once. What D8 gates is **only** the later move of prod's `BASE_DOMAIN` to the bare apex.
|
||||
Verify the two guard values live in prod's applied `cluster-vars` before unsuspending; if
|
||||
`EXTERNALDNS_TXT_OWNER_ID` is not `prod`, STOP — that was the root cause of the incident.
|
||||
- [ ] **P2. Prod's external-dns publishes under owner `prod`.** Check the logs for a matching hosted
|
||||
zone (F-DNS1 fail-closed mode: no zone selected ⇒ nothing created — which is safe, but means
|
||||
the filter was narrowed against instruction). Confirm the new TXT registry entries read owner
|
||||
`prod` and that no record owned by `dev` was touched.
|
||||
- [ ] **P3. Prod's wildcard cert `*.home.keithjackson.dev` + `home.keithjackson.dev` is `Ready`** and
|
||||
solved DNS-01 without colliding with any `_acme-challenge` dev still holds.
|
||||
- [ ] **P4. Prod's kubectl SSO works without an `/etc/hosts` pin.** Prod nodes resolve
|
||||
`keycloak.home.keithjackson.dev` through **public** upstreams (§4), so this only becomes true
|
||||
once P2 has published that name at prod's Gateway VIP (192.168.2.20). Before then a prod node
|
||||
resolving its own issuer reaches **dev**. Use the break-glass cert kubeconfig until P2 is done.
|
||||
- [ ] **P5. Cut prod's GitOps source over** from the workstation bootstrap Gitea
|
||||
(`http://192.168.2.132:3000/...`) to `https://gitea.home.keithjackson.dev/...` once prod runs
|
||||
its own Gitea — the block in `clusters/prod/kustomization.yaml` owns that procedure. Note the
|
||||
inversion **simplified** this: prod's apex Gitea name and dev's `gitea.dev.home...` are now
|
||||
different hosts, so prod pointing at the apex means "prod's own Gitea", never dev's.
|
||||
|
||||
### ACME: the duplicate-certificate limit
|
||||
|
||||
Let's Encrypt rate-limits **duplicate certificates — 5 per week for an identical set of names.**
|
||||
Prod's wildcard is `*.home.keithjackson.dev` + `home.keithjackson.dev`, which is **exactly the name
|
||||
set dev holds today**. Every prod order therefore counts against the same bucket as dev's historical
|
||||
renewals of that pair, and a retry loop during P3 can exhaust it and leave prod with **no trusted
|
||||
cert for up to a week**. Mitigations:
|
||||
|
||||
- Order prod's cert **after** D8, so dev is no longer renewing the same pair.
|
||||
- Let dev's old `*.home.keithjackson.dev` `Certificate`/Secret be **deleted** in D4/D7 rather than
|
||||
left to renew in the background.
|
||||
- Rehearse against **letsencrypt-staging** first (`ACME_CLUSTER_ISSUER`) — staging has its own,
|
||||
far looser limits — and only switch prod to `letsencrypt-prod` when the DNS-01 solve is proven.
|
||||
- Do **not** delete-and-recreate prod's `Certificate` to "retry"; each recreation is a fresh order.
|
||||
|
||||
### The L2 tests hardcode the apex
|
||||
|
||||
Six behavioural (L2) chainsaw tests embed `home.keithjackson.dev` as a literal:
|
||||
|
||||
| test | file |
|
||||
|---|---|
|
||||
| `dns-exposure` | `conformance/tests/l2-behavioral/dns-exposure/chainsaw-test.yaml` |
|
||||
| `dns-internal-resolution` | `conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml` |
|
||||
| `gateway-ingress-reachable` | `conformance/tests/l2-behavioral/gateway-ingress-reachable/chainsaw-test.yaml` (+ `fixtures/app-and-probe.yaml`) |
|
||||
| `oidc-sso-grafana` | `conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml` |
|
||||
| `sso-forward-auth` | `conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml` |
|
||||
| `trusted-tls` | `conformance/tests/l2-behavioral/trusted-tls/chainsaw-test.yaml` |
|
||||
|
||||
They are **test inputs, not reconciled manifests**, so they are deliberately left alone by the
|
||||
authoring change: they are the only thing still asserting the apex, and while dev is live on the apex
|
||||
they are *correct*. Under the new direction the literal happens to become right for **prod** and
|
||||
wrong for **dev**, so they must be **parameterized per environment** (not merely edited) before
|
||||
either §9's *"suite passes green against the prod cluster"* or a green L2 run against migrated dev
|
||||
can be claimed. Until then, expect the dev L2 suite to fail between D4 and that parameterization —
|
||||
a known, scheduled failure, not a regression.
|
||||
|
||||
The L1 *rendering* tests need no change: their goldens carry `${BASE_DOMAIN}` as a token, and the
|
||||
literal hostnames in them come from each test's own fixture `ingressRoutes[].host`, which is
|
||||
environment-independent by construction.
|
||||
|
||||
### Offline proof carried out for the authoring change (no cluster touched)
|
||||
|
||||
All 12 `substituteFrom` paths were rendered for **both** overlays through the real toolchain
|
||||
(`kubectl kustomize --load-restrictor LoadRestrictionsNone <spec.path> | flux envsubst --strict`,
|
||||
with each overlay's effective `cluster-vars` taken from `kubectl kustomize clusters/<env>`), before
|
||||
and after the change.
|
||||
|
||||
- **Dev's diff is *only* the hostname move.** Proven by inverse substitution rather than by reading
|
||||
the diff: applying `s/dev.home.keithjackson.dev/home.keithjackson.dev/` to the new render restores
|
||||
the baseline **byte-for-byte** in all 12 paths, plus 2 declared `txtOwnerId: default → dev` lines.
|
||||
145 changed line pairs, all classified: 143 hostname-move, 2 declared owner-id, **0 unexplained**,
|
||||
and **0 bare-apex remnants**. The 2 paths with no change (`cilium`, `velero`) provably contain no
|
||||
hostname at all (`grep -c keithjackson.dev` = 0).
|
||||
- **Prod lands on the bare apex:** **0** occurrences of `prod.home.keithjackson.dev` and **0** of
|
||||
`dev.home.keithjackson.dev` anywhere in prod's rendered output; 143 apex names.
|
||||
- **Certs:** dev `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev`; prod
|
||||
`*.home.keithjackson.dev` + `home.keithjackson.dev`.
|
||||
- **Positive controls** (the harness is provably not blind): perturbing an unrelated knob
|
||||
(`GATEWAY_VIP`) surfaces 1 unexplained line; perturbing `BASE_DOMAIN` to a wrong value fails 10 of
|
||||
12 paths; injecting a single un-moved hostname is caught by the independent bare-apex scan (which
|
||||
the byte-identity check alone cannot see — which is why both exist).
|
||||
- Gates: `ci/l0.sh`, `ci/l1.sh`, `ci/render-golden.sh`, `ci/lint.sh`,
|
||||
`flux-postbuild-substitution/check.sh` (254 assertions), `catalog/render.py --selftest`,
|
||||
`ci/spec-immutability.sh` — all **PASS**.
|
||||
@@ -0,0 +1,519 @@
|
||||
# Prod cluster standup — the complete command sequence
|
||||
|
||||
**Every command below is copy-pasteable and runs from the repo root**, through the pinned dev
|
||||
container. Nothing here needs a command invented on the spot; that is the point. If a step is not in
|
||||
this file it is not part of standup.
|
||||
|
||||
State as of **2026-09-06**: phases **A**, **B** and **C** are DONE. The cluster is live — k3s
|
||||
v1.36.4 HA (3 etcd members), kube-vip, Cilium 1.20.1, Longhorn — with **12/13 Flux tiers Ready**
|
||||
(`velero` deliberately suspended), **112 pods Running**, an issued Let's Encrypt wildcard, and
|
||||
**all three nodes addressing their full 32 GB** (node 2's DIMM upgrade completed 2026-09-06 and is
|
||||
confirmed at ~`32638256Ki`, matching the other two to within 4 KiB — the §0b memattr bug is not
|
||||
biting on the backports kernel).
|
||||
|
||||
Storage survived two unplanned node events (an ungraceful power-down of prod-3, then prod-2's memory
|
||||
upgrade) and converged to **17/17 volumes healthy, replicas symmetric across all three nodes** —
|
||||
though only after a leftover cordon was cleared; read
|
||||
[§Node maintenance](#node-maintenance--draining-rebooting-and-the-cordon-that-stops-self-healing)
|
||||
before doing node work, that cordon silently prevents Longhorn from rebuilding.
|
||||
|
||||
What remains is in [§Blockers](#blockers), and every item there needs an operator decision:
|
||||
no backup coverage (R2 credentials), prod still on `prod.home.` rather than the apex, and — found
|
||||
2026-09-05 — **prod's UniFi VM has no disk** because Kyverno blocks CDI's generated importer pod.
|
||||
|
||||
> **The one guard that matters.** Dev is LIVE and shares the `keithjackson.dev` Cloudflare zone with
|
||||
> prod. On 2026-09-04 prod deleted every one of dev's public DNS records. `bootstrap/standup.sh`
|
||||
> therefore **requires** `--expect-server` and compares it against the live API server before running
|
||||
> anything. There is no override flag. Do not add one.
|
||||
|
||||
## Constants
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| kube-API (kube-vip VIP) | `https://192.168.2.9:6443` |
|
||||
| Gateway / ingress VIP | `192.168.2.20` |
|
||||
| Nodes | `k3s-prod-1/2/3` — `192.168.2.10 / .11 / .12` |
|
||||
| Break-glass kubeconfig | `infra/ansible/.cache/prod.kubeconfig` (cert-based; survives OIDC/DNS failure) |
|
||||
| GitOps source (interim) | `http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` branch `main` |
|
||||
| `BASE_DOMAIN` | `prod.home.keithjackson.dev` — **not** the apex; see §Blockers |
|
||||
|
||||
Container paths: the repo is mounted at `/work`, so kubeconfig paths passed to the script are
|
||||
**repo-relative** (`infra/ansible/.cache/prod.kubeconfig`).
|
||||
|
||||
---
|
||||
|
||||
## Phase A — bare metal to a joined cluster (Ansible)
|
||||
|
||||
Prerequisites that are **not** commands: firmware/BIOS (§1), disk layout (§2), OS install (§3),
|
||||
network (§4), and SSH + the vaulted `ansible_become_password` in
|
||||
`inventory/group_vars/k3s_prod/vault.yml` (§5). Ansible cannot escalate on prod without that vault
|
||||
file. See `infra/PROD-PROVISIONING.md`.
|
||||
|
||||
**Install the backports kernel first.** All three prod nodes need ≥6.13 because HP's R21 firmware
|
||||
publishes a corrupt EFI Memory Attributes Table and Linux 6.12's `efi_memattr_init()` does not
|
||||
validate it, reserving everything above the table (fixed upstream in 6.13 by `8fbe4c49c0cc`, absent
|
||||
from `linux-6.12.y`). Prod runs `7.1.8+deb13-amd64` from trixie-backports.
|
||||
|
||||
```bash
|
||||
timeout 3600 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
|
||||
ansible-playbook /work/infra/ansible/playbooks/node-baseline.yml \
|
||||
-i /work/infra/ansible/inventory/hosts.ini -e k3s_hosts=k3s_prod --tags kernel
|
||||
```
|
||||
|
||||
Then the full chain — baseline, k3s HA, Cilium, verify, Flux hand-off. **All three `-e` flags are
|
||||
required**: a play's `hosts:` is templated before any host exists, so `group_vars` cannot retarget it.
|
||||
|
||||
```bash
|
||||
timeout 3600 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
|
||||
ansible-playbook /work/infra/ansible/playbooks/site.yml \
|
||||
-i /work/infra/ansible/inventory/hosts.ini \
|
||||
-e k3s_hosts=k3s_prod -e k3s_primary_hosts=k3s_prod_primary -e k3s_secondary_hosts=k3s_prod_secondary
|
||||
```
|
||||
|
||||
kube-vip is a separate play (not imported by `site.yml`):
|
||||
|
||||
```bash
|
||||
timeout 1800 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
|
||||
ansible-playbook /work/infra/ansible/playbooks/kube-vip.yml \
|
||||
-i /work/infra/ansible/inventory/hosts.ini -e k3s_hosts=k3s_prod
|
||||
```
|
||||
|
||||
Hardware health (read-only; `node_health` asserts the memattr fix took and that all RAM is addressed).
|
||||
|
||||
> **The flag is `node_health_hosts`, NOT `k3s_hosts`.** Diagnostic plays deliberately keep their own
|
||||
> target var so they stay independently pointable while a bring-up run is scoped elsewhere
|
||||
> (`node-health.yml` → `node_health_hosts`, `memattr-survey.yml` → `memattr_survey_hosts`,
|
||||
> `vpn.egress.gateway.yml` → `vpn_egress_gateway_hosts`). The consequence is a **silent** one: passing
|
||||
> `k3s_hosts=k3s_prod` here does not error, it falls through to the `k3s_servers` default and runs
|
||||
> against **DEV's three nodes**. Verified — this exact mistake was made while writing this file and
|
||||
> caught only by `--list-hosts`. **Always dry-run the target first:**
|
||||
>
|
||||
> ```bash
|
||||
> timeout 200 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
|
||||
> ansible-playbook /work/infra/ansible/playbooks/node-health.yml \
|
||||
> -i /work/infra/ansible/inventory/hosts.ini -e node_health_hosts=k3s_prod --list-hosts
|
||||
> ```
|
||||
|
||||
```bash
|
||||
timeout 900 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
|
||||
ansible-playbook /work/infra/ansible/playbooks/node-health.yml \
|
||||
-i /work/infra/ansible/inventory/hosts.ini -e node_health_hosts=k3s_prod
|
||||
```
|
||||
|
||||
## Phase B — Flux bootstrap
|
||||
|
||||
`flux-bootstrap.yml` (last play of `site.yml`) shells to `bootstrap/flux/bootstrap.sh`, the single
|
||||
documented entrypoint. That script now runs the CRD pre-apply as step **[2/4]**, so a cold bootstrap
|
||||
no longer strands `platform`. The **age private key Secret must exist first** — bootstrap verifies it
|
||||
and refuses to continue rather than silently skipping decryption
|
||||
(`bootstrap/sops/age-key-secret.md`).
|
||||
|
||||
## Phase C — bring the reconciliation graph up
|
||||
|
||||
One script, six steps, each independently runnable and idempotent. **Run them in this order.**
|
||||
|
||||
```bash
|
||||
K=infra/ansible/.cache/prod.kubeconfig
|
||||
S=https://192.168.2.9:6443
|
||||
```
|
||||
|
||||
### C1 — preflight (read-only, changes nothing)
|
||||
|
||||
Asserts the guards that prevent a repeat of the DNS incident and reports every tier's state. **Fails
|
||||
closed** if `EXTERNALDNS_TXT_OWNER_ID` is empty, still the base sentinel, or `default` (dev's owner).
|
||||
|
||||
```bash
|
||||
timeout 300 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 preflight
|
||||
```
|
||||
|
||||
### C2 — pre-apply chart CRDs, and break the CDI cold-start deadlock (idempotent)
|
||||
|
||||
```bash
|
||||
timeout 600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 crds
|
||||
```
|
||||
|
||||
Two cold-start problems, same root cause: **Flux server-side dry-runs every object in a Kustomization
|
||||
before applying any of them, so one unknown `kind` aborts the entire tier.**
|
||||
|
||||
1. **Kyverno** — its ~5.6 MiB CRD bundle exceeds kubectl's 256 KiB annotation limit, so the chart sets
|
||||
`crds.install: false` and nothing else installs them. `bootstrap/flux/preapply-crds.sh` applies them
|
||||
server-side, deriving the version from the HelmRelease Flux reconciles (no second pin).
|
||||
|
||||
2. **CDI — a true deadlock.** `platform-config` applies the CDI CR *and*
|
||||
`platform/kubevirt/cdi-storageprofiles.yaml` in the **same pass**. But `StorageProfile`'s CRD does
|
||||
not exist until `cdi-operator` has reconciled that CR. So the dry-run fails on the unknown kind →
|
||||
the CR is never applied → the CRD is never created → `platform-config` can never converge, and
|
||||
every tier below it stays blocked. **Invisible on an established cluster** (dev's CRDs have been
|
||||
there for months) **and fatal on a cold one.** Observed on prod 2026-09-05: `cdi-operator`
|
||||
Available for 12h with **no CDI CR** and no `storageprofiles` CRD.
|
||||
|
||||
The step breaks it by applying **only the CR** — the minimum — and leaving everything else to
|
||||
Flux. It carries no `${}` tokens, so applying it here cannot diverge from what Flux would render.
|
||||
The CRD appeared in 10s.
|
||||
|
||||
**REPAIRED IN THE GRAPH (2026-09-05).** `cdi-storageprofiles.yaml` now lives in its own tier,
|
||||
`./platform-config-cdi`, with `dependsOn: platform-config` and `wait: true`. Because `dependsOn` is
|
||||
**revision**-gated and `platform-config` runs `wait: true`, the CDI CR must be applied *and healthy*
|
||||
— which is precisely the event that creates the `StorageProfile` CRD — before the profiles are
|
||||
applied. A cold cluster now orders itself. The `crds` step is retained as a fallback for a cluster
|
||||
already stuck in the broken state, and because it is idempotent.
|
||||
|
||||
**The generalisable rule, now recorded in `clusters/README.md`:** a CR whose CRD is created by a
|
||||
*controller at runtime* is **second-order** and needs its own pass. `cdi-operator.yaml` registers
|
||||
exactly one CRD (`cdis.cdi.kubevirt.io`); `storageprofiles`, `datavolumes`, `datasources` and
|
||||
`dataimportcrons` appear only after cdi-operator reconciles the CDI CR. An audit of
|
||||
`platform-config` found `StorageProfile` is the only second-order kind there.
|
||||
|
||||
`apps-vms` gained `dependsOn: platform-config-cdi` — load-bearing, not cosmetic:
|
||||
`apps/vms/unifi/20-datavolume.yaml` provisions on `storageClassName: longhorn-vm`, and its comment
|
||||
previously claimed `platform-config` guaranteed the profiles, which the move would have silently
|
||||
falsified.
|
||||
|
||||
#### Merge precondition and the one-time transient
|
||||
|
||||
`platform-config` has `prune: true`, so on the first reconcile after merge it **deletes** both
|
||||
StorageProfiles, CDI recreates them within seconds with an empty spec, and `platform-config-cdi`
|
||||
re-applies the overrides ~30–60s later (kustomize-controller's dependency requeue).
|
||||
|
||||
**Measured on dev, so the exposure is known rather than assumed** — comparing our override against
|
||||
what CDI infers unaided for the same Longhorn provisioner:
|
||||
|
||||
| profile | `spec` (ours) | CDI-derived `status` |
|
||||
|---|---|---|
|
||||
| `longhorn-vm` (we override) | RWX / Block, `snapshot` | **ReadWriteMany** / Block, `snapshot` |
|
||||
| `longhorn-static` (no override) | *unset* | **ReadWriteOnce** / Block, `snapshot` |
|
||||
|
||||
So **`cloneStrategy` is not at risk** — CDI infers `snapshot` for Longhorn anyway. **`accessModes`
|
||||
is:** unaided, CDI produces **RWO**. A DataVolume provisioned on `longhorn-vm` *inside the window*
|
||||
would be stamped RWO/Block and be **permanently non-live-migratable**. Flux can no longer cause that
|
||||
(`apps-vms` now gates on the new tier); **do not start a VM disk import, clone or DataVolume by hand
|
||||
during the window.**
|
||||
|
||||
- [ ] **Before merging, confirm `flux get kustomization platform-config` is Ready on the target.** If
|
||||
it is not, the profiles are pruned and sit at CDI defaults until it recovers — and prod was in
|
||||
exactly that state hours earlier. The fallback is the two-step the Cilium move used: merge with
|
||||
`platform-config.prune: false`, confirm the ownership label transferred, restore `prune: true` in
|
||||
a second commit.
|
||||
- [ ] After: both profiles carry `spec.cloneStrategy: snapshot`, `longhorn-vm` carries
|
||||
`claimPropertySets` RWX/Block, and both are labelled
|
||||
`kustomize.toolkit.fluxcd.io/name=platform-config-cdi`.
|
||||
|
||||
### C3 — unwedge stuck Helm releases
|
||||
|
||||
A release left in `uninstalling` / `pending-*` cannot be advanced by helm-controller: it reports
|
||||
`StateError` forever, so `platform` (`wait: true`) never goes Ready and all 10 tiers below it stall.
|
||||
**Dry run by default** — it prints what it would do and changes nothing:
|
||||
|
||||
```bash
|
||||
timeout 600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 unwedge
|
||||
```
|
||||
|
||||
To act (this **deletes** the release's Kubernetes objects and lets Flux install cold). Pre-applied
|
||||
CRDs live **outside** the Helm release and survive, which is exactly why C2 comes first:
|
||||
|
||||
```bash
|
||||
timeout 900 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 \
|
||||
--confirm-destructive unwedge
|
||||
```
|
||||
|
||||
### C4 — external-dns safe upgrade
|
||||
|
||||
**The most dangerous step in the repo, and the ordering is the entire mitigation.** A Helm upgrade
|
||||
resets `replicas` to the chart default (1) and applies the new args in ONE operation — there is no
|
||||
window in which to check the result first. If `--txt-owner-id` were missing, the pod starts as owner
|
||||
`default`, claims dev's TXT registry entries, and under `policy: sync` **deletes the records behind
|
||||
them**.
|
||||
|
||||
So this step **proves the outcome before causing it**: it renders the chart with the *live*
|
||||
HelmRelease values and asserts `--txt-owner-id=<owner>` appears, aborting if not. Rendering is
|
||||
deterministic and touches nothing, which converts a race into a precondition. It then snapshots the
|
||||
Cloudflare zone, upgrades, re-verifies the live args (scaling to 0 and failing if they are wrong),
|
||||
and diffs the zone.
|
||||
|
||||
```bash
|
||||
timeout 900 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 externaldns
|
||||
```
|
||||
|
||||
Expected diff: **additions only, all under `prod.home.keithjackson.dev`, all TXT entries owner
|
||||
`prod`.** Any change to a name outside that subtree, or to a TXT entry owned by another cluster, is a
|
||||
regression — stop and investigate.
|
||||
|
||||
### C5 — resume the tiers, in dependency order
|
||||
|
||||
`platform → platform-config → {oauth2-proxy, headlamp, homepage, crafty, apps-wave-0, apps-vms} →
|
||||
apps-wave-1 → apps-wave-2`. Waits for each to go Ready and **stops at the first that does not** — a
|
||||
partial, understood bring-up beats a forced green. `velero` is skipped while `S3_BACKUP_ENDPOINT` is
|
||||
a placeholder (see §Blockers). Long-running; expect it to background.
|
||||
|
||||
```bash
|
||||
timeout 3600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 resume
|
||||
```
|
||||
|
||||
### C6 — verify
|
||||
|
||||
```bash
|
||||
timeout 900 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 verify
|
||||
```
|
||||
|
||||
### Everything at once
|
||||
|
||||
Runs C1→C6 with `unwedge` in **dry-run** mode (no `--confirm-destructive`), so it stops short of the
|
||||
destructive step by design:
|
||||
|
||||
```bash
|
||||
timeout 3600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
|
||||
--kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 all
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## VERIFIED WORKING — 2026-09-05, standup complete
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Kustomizations | **12/13 Ready** (`velero` deliberately suspended — see §Blockers) |
|
||||
| Pods | **113 total, 0 not Running/Completed** |
|
||||
| Wildcard cert | `wildcard-home` **Ready**, Let's Encrypt **prod**, `*.prod.home.keithjackson.dev` |
|
||||
| Gateway VIP | `192.168.2.20` · crafty L4 `192.168.2.22` · unifi L4 `192.168.2.21` |
|
||||
| Published DNS | **36 records** under `prod.home.keithjackson.dev`, every TXT registry entry `owner=prod` |
|
||||
| Dev's records | **57, byte-identical to the pre-standup baseline — nothing added, changed or removed** |
|
||||
|
||||
End-to-end HTTPS **through the workstation's own resolver — no `--resolve` override, no `-k`** (after
|
||||
installing the dnsmasq profile below). Every request landed on `192.168.2.20`:
|
||||
|
||||
```
|
||||
prod.home.keithjackson.dev 200 headlamp. 200 keycloak. 302 (login) nextcloud. 302
|
||||
gitea. 200 semaphore. 200 grafana. 302 (SSO) pihole. 302
|
||||
```
|
||||
|
||||
The served certificate is prod's own, confirmed by SNI — **not** dev's wildcard, which is what the
|
||||
resolver bug previously made it look like:
|
||||
|
||||
```
|
||||
subject = CN=*.prod.home.keithjackson.dev
|
||||
issuer = C=US, O=Let's Encrypt, CN=YR1 (LE production)
|
||||
SAN = *.prod.home.keithjackson.dev, prod.home.keithjackson.dev
|
||||
valid = 2026-09-05 .. 2026-12-04
|
||||
```
|
||||
|
||||
Dev, checked in the same pass, still serves `CN=*.home.keithjackson.dev` — **unchanged**.
|
||||
|
||||
> Ignore `curl: (91) No OCSP response received` if you pass `--cert-status`: Let's Encrypt retired
|
||||
> OCSP responders in 2025, so there is nothing to staple. It is not a certificate fault, and the plain
|
||||
> requests above all validated normally.
|
||||
|
||||
### Workstation DNS — install the prod dnsmasq profile
|
||||
|
||||
**CORRECTION (operator, 2026-09-05): this is the workstation's own dnsmasq, NOT pfSense.** Earlier
|
||||
notes in this repo called it a "pfSense override"; that was wrong and the topology below is why.
|
||||
|
||||
`/etc/NetworkManager/dnsmasq.d/10-home.keithjackson.dev.conf` (the dev profile) contains:
|
||||
|
||||
```
|
||||
address=/.home.keithjackson.dev/192.168.2.50
|
||||
```
|
||||
|
||||
dnsmasq's `address=` matches the named domain **and every subdomain**, so that one line also captures
|
||||
the whole `prod.home.keithjackson.dev` tree and answers it with **dev's** address, while Cloudflare
|
||||
correctly returns prod's `192.168.2.20`.
|
||||
|
||||
**Fix — one command, already authored in this repo:**
|
||||
|
||||
```bash
|
||||
sudo infra/workstation/dnsmasq/install.sh
|
||||
```
|
||||
|
||||
It installs `20-prod.home.keithjackson.dev.conf` and reloads the live resolver. dnsmasq resolves
|
||||
`address=` by **longest match**, so the prod rules win without editing or reordering the dev profile.
|
||||
Verified against a throwaway dnsmasq loaded with both profiles:
|
||||
|
||||
| query | answer |
|
||||
|---|---|
|
||||
| `prod.home.keithjackson.dev`, `gitea.prod…`, `keycloak.prod…` | `192.168.2.20` (Gateway) |
|
||||
| `unifi.prod.home.keithjackson.dev` | `192.168.2.21` (L4) |
|
||||
| `crafty-mc.prod.home.keithjackson.dev` | `192.168.2.22` (L4) |
|
||||
| `gitea.home.keithjackson.dev`, `home.keithjackson.dev` | `192.168.2.50` — **dev, unchanged** |
|
||||
|
||||
> **Install to `/etc/NetworkManager/dnsmasq.d/`, never `/etc/dnsmasq.d/`.** The standalone
|
||||
> `dnsmasq.service` is **inactive**; the live resolver is the one NetworkManager spawns
|
||||
> (`dns=dnsmasq`) with `--conf-dir=/etc/NetworkManager/dnsmasq.d`. A file in the other directory
|
||||
> changes nothing and looks like it should. `install.sh` pins the destination for exactly this reason.
|
||||
|
||||
**Until it is installed, verify prod only with an explicit resolve** — and note *why* this trap is
|
||||
convincing: dev serves a `*.home.keithjackson.dev` wildcard cert, which matches
|
||||
`prod.home.keithjackson.dev` (one label), so the prod **apex returns 200 over valid TLS** while
|
||||
`keycloak.prod.home.keithjackson.dev` (two labels) fails with a certificate-name error. It reads like
|
||||
a broken prod certificate. It is the wrong server answering.
|
||||
|
||||
```bash
|
||||
curl -sS --resolve gitea.prod.home.keithjackson.dev:443:192.168.2.20 \
|
||||
https://gitea.prod.home.keithjackson.dev/
|
||||
```
|
||||
|
||||
## NODE MAINTENANCE — draining, rebooting, and the cordon that stops self-healing
|
||||
|
||||
Two unplanned node events on 2026-09-05/06 (an ungraceful power-down of prod-3, then a memory upgrade
|
||||
on prod-2) produced one finding worth more than either incident.
|
||||
|
||||
### A failed `kubectl drain` still cordons — and on this cluster a cordon stops storage self-healing
|
||||
|
||||
`kubectl drain k3s-prod-2` fails on this cluster with the usual two objections (emptyDir pods,
|
||||
DaemonSet pods) — **but it cordons the node first, and the cordon persists after the failure.**
|
||||
Longhorn's `disable-scheduling-on-cordoned-node` is **`true`**, so a cordoned node receives no replica
|
||||
scheduling. Observed consequence: after prod-2's reboot, **all 17 volumes sat `attached/degraded` with
|
||||
prod-2 holding 2 of 18 replicas and ZERO rebuilds in flight.** Nothing was driving recovery, every pod
|
||||
was `Running`, and no alert fired. `kubectl uncordon` started the rebuilds immediately; they converged
|
||||
17/17 healthy in ~6 minutes.
|
||||
|
||||
**So: after any drain — successful or not — `kubectl uncordon <node>` is not tidying up, it is what
|
||||
re-enables storage redundancy.** Check `kubectl get nodes -o custom-columns=N:.metadata.name,\
|
||||
U:.spec.unschedulable` before believing a green pod list.
|
||||
|
||||
### `drain` cannot complete on this cluster, and the two usual flags do not fix it
|
||||
|
||||
`--ignore-daemonsets --delete-emptydir-data` clears the two reported objections and then **hangs
|
||||
indefinitely**: all three `instance-manager-*` PDBs report **`ALLOWED DISRUPTIONS = 0`** and
|
||||
`node-drain-policy` is `block-if-contains-last-replica`. Longhorn is refusing to let an
|
||||
instance-manager be evicted while it runs volume engines. That is data protection, not a bug.
|
||||
|
||||
**Requesting replica eviction does not rescue it either:** with **3 nodes and 3 replicas** there is no
|
||||
fourth node to move a replica to, so eviction can never complete. This is structural, not a setting.
|
||||
|
||||
Options, in order of preference:
|
||||
|
||||
1. **Just reboot it (cordon, reboot, uncordon).** Proven twice on this cluster: Longhorn resyncs on
|
||||
return, and `replica-replenishment-wait-interval` is `600`s, so a reboot completed inside ~10
|
||||
minutes does not even trigger full rebuilds. This is the normal path for firmware, memory or kernel
|
||||
work.
|
||||
2. **If the node must be genuinely empty**, first scale to 0 the workloads holding engines on it —
|
||||
that detaches them, the PDBs go to 1 allowed disruption, and the drain completes. Find them with:
|
||||
`kubectl -n longhorn-system get engines.longhorn.io -o json` filtered on `.spec.nodeID`.
|
||||
3. Changing `node-drain-policy` to `always-allow` is **not** advised at 3 nodes/3 replicas — it trades
|
||||
the protection for exactly the data risk the default exists to prevent.
|
||||
|
||||
### Check the kube-vip leader before rebooting a server
|
||||
|
||||
`kube-vip` holds the API VIP (`192.168.2.9`) via a leader lease, and it is a **DaemonSet** — so
|
||||
`--ignore-daemonsets` leaves it running and a *drain* does not move the VIP. A **reboot** does: the
|
||||
lease fails over and `kubectl` blips for a few seconds. Harmless, but do not mistake it for a fault.
|
||||
Identify the holder: `kubectl -n kube-system logs -l app.kubernetes.io/name=kube-vip-ds | grep -i leader`.
|
||||
|
||||
### Post-maintenance checklist
|
||||
|
||||
- [ ] `kubectl uncordon <node>` — **and confirm** `.spec.unschedulable` is clear on all three.
|
||||
- [ ] Longhorn converged: `kubectl -n longhorn-system get volumes.longhorn.io` shows **0 degraded**.
|
||||
Rebuilds are serial, so allow minutes, and watch rather than assume.
|
||||
- [ ] Replica symmetry: each node runs the same count
|
||||
(`get replicas.longhorn.io` grouped by `.spec.nodeID` / `.status.currentState`).
|
||||
- [ ] All three nodes report full memory — prod's HP firmware publishes a corrupt EFI memattr table and
|
||||
a wrong kernel silently strands RAM (§0b). `k3s-prod-*` should each show ~`32638256Ki`.
|
||||
- [ ] Pods: none `Running`-but-not-`Ready`.
|
||||
|
||||
> **KNOWN, NOT FIXED — pod distribution does not rebalance.** After prod-2's cordon+reboot the spread
|
||||
> was **52 / 10 / 50**. Kubernetes does not move running pods back, and for the many SINGLE-REPLICA
|
||||
> apps here a spread constraint would change nothing (one replica lives on one node either way). It
|
||||
> self-corrects as pods restart. It is recorded because it concentrates failure: at the time of writing
|
||||
> **all three CNPG databases run on prod-1**, so losing that node takes Keycloak, Nextcloud and
|
||||
> Semaphore together. The real fix is database HA (`instances: 2`), which is a resource decision the
|
||||
> operator owns — see §Blockers.
|
||||
|
||||
## Blockers
|
||||
|
||||
### 1. Backups: endpoint DECIDED, but prod cannot reach it — **prod has no backup coverage**
|
||||
|
||||
**Operator ruling 2026-09-05: "the S3 backup location is the same as Dev."** Prod's
|
||||
`S3_BACKUP_ENDPOINT` is now `http://192.168.1.9:9000`, and the `TBD-PROD-*` placeholder is retired.
|
||||
|
||||
**It is not reachable from prod.** Measured from a throwaway pod **inside the prod cluster** — not
|
||||
from the workstation, which proves nothing about pod egress:
|
||||
|
||||
```
|
||||
curl http://192.168.1.9:9000/minio/health/live → connection timed out after 15s
|
||||
```
|
||||
|
||||
**This is expected topology, not a misconfiguration** (operator, 2026-09-05): the k3s prod cluster is
|
||||
**not behind pfSense**. It sits on the SOHO router network `192.168.2.0/24`, which is pfSense's
|
||||
**WAN** side; pfSense *contains* `192.168.1.0/24`, the dev LAN where MinIO lives. Prod reaching
|
||||
`192.168.1.9` therefore means crossing **into** pfSense from its WAN — which requires a deliberate
|
||||
pfSense rule or port-forward that does not exist. **Operator decision required**, and worth weighing:
|
||||
a prod backup that lands inside the dev LAN is arguably not off-cluster DR at all.
|
||||
|
||||
Re-test with `standup.sh --s3-endpoint http://192.168.1.9:9000 s3check` — it fails closed and says so.
|
||||
|
||||
> **Do not confuse this with the SOHO router's inbound NAT.** That router forwards inbound **443** to
|
||||
> the pfSense WAN address `192.168.2.102` to serve the **existing, non-k3s production environment**,
|
||||
> which must stay live on 443 until the k3s cutover is deliberately made. Nothing in this runbook
|
||||
> touches that NAT, and prod's Gateway on `192.168.2.20:443` does not contend with it.
|
||||
|
||||
Until that routing exists, `velero` stays suspended and **`apps-wave-0` still applies GREEN** — the
|
||||
token *is* defined, so strict substitution succeeds — while every write to that endpoint fails at
|
||||
**runtime**. A green `apps-wave-0` does not mean backups work. **Treat prod as having no backup
|
||||
coverage.**
|
||||
|
||||
#### Sharing one MinIO is only safe because of two new knobs
|
||||
|
||||
Nothing about the backup *paths* was per-environment before 2026-09-05. Enabling prod against dev's
|
||||
MinIO without these would have been the external-dns incident again, in a worse place:
|
||||
|
||||
| | before | after |
|
||||
|---|---|---|
|
||||
| Velero | `bucket: velero`, **no prefix**, both clusters run schedules named `daily` | `prefix: ${VELERO_BACKUP_PREFIX}` — dev `""`, prod `prod` |
|
||||
| CNPG | `destinationPath: s3://cnpg`, serverName defaults to the **Cluster name** | `${CNPG_BACKUP_PATH}` — dev `s3://cnpg`, prod `s3://cnpg/prod` |
|
||||
|
||||
- **Velero** syncs backups it finds in its BSL into itself as `Backup` CRs and **garbage-collects them
|
||||
when their TTL expires**. Prod would have adopted dev's backups and deleted them on schedule.
|
||||
- **CNPG is the worse one.** Both environments have `keycloak-db`, `nextcloud-db` and `semaphore-db`,
|
||||
so prod's Postgres would archive WAL into the *exact prefix* dev's does. Two servers interleaving
|
||||
WAL into one Barman stream **corrupts the chain and destroys PITR** — and dev's restore is
|
||||
**proven** (R-STATE-03).
|
||||
|
||||
**The base values are dev's current layout on purpose and must stay that way.** Dev's existing
|
||||
backups live at the bucket root and at `s3://cnpg`; giving dev a prefix would **orphan the very
|
||||
backups its proven restore depends on** — the same reasoning that keeps dev's
|
||||
`EXTERNALDNS_TXT_OWNER_ID` at `default`. Prod has no history to preserve, so it takes the nested paths.
|
||||
|
||||
> **Delivery note.** Prod reconciles the bootstrap Gitea's `main` at `c8c1d29`. This change sits on a
|
||||
> later branch that also carries the DNS inversion, which must **not** reach prod (it would move prod
|
||||
> to the apex and collide with dev). Delivering it needs a `c8c1d29`-based branch carrying only these
|
||||
> commits. No urgency: velero cannot be enabled until the routing above is fixed.
|
||||
|
||||
### 2. Prod is on `prod.home.keithjackson.dev`, not the apex
|
||||
|
||||
The operator's target layout is dev under `*.dev.home.keithjackson.dev` and prod on the **apex**.
|
||||
Prod cannot take the apex until dev vacates it, because both clusters would request a
|
||||
`letsencrypt-prod` wildcard for the *same name set* and race each other's `_acme-challenge` TXT
|
||||
records, so neither reliably solves DNS-01.
|
||||
|
||||
**This does not block phase C.** Prod on its own subtree with `txtOwnerId: prod` collides with
|
||||
nothing, and standing it up first is strictly better: it proves the whole stack before the flip, so
|
||||
the cutover becomes a DNS/TLS change on a known-good cluster instead of a bring-up and a cutover at
|
||||
once. The apex move is `infra/PROD-PROVISIONING.md` §10a, sequence **D** (dev vacates) then **P**.
|
||||
|
||||
> **THE APEX FLIP IS ALSO THE LEGACY-PROD CUTOVER — they are the same event.** The SOHO router NATs
|
||||
> inbound **443** to the pfSense WAN address `192.168.2.102`, serving the **existing, non-k3s
|
||||
> production environment**, which the operator requires to stay live until the switch is deliberately
|
||||
> made. So §10a sequence **P** is not merely a DNS/TLS change on the k3s side: taking
|
||||
> `home.keithjackson.dev` means public traffic must also stop resolving to / being forwarded to the
|
||||
> legacy stack. Sequence **P** as written does **not** cover repointing that NAT, and the two must be
|
||||
> planned together or the apex will resolve to k3s while inbound 443 still lands on the legacy box.
|
||||
>
|
||||
> Nothing in phase C touches it: prod's Gateway is `192.168.2.20:443`, a different address from the
|
||||
> NAT target, and prod's wildcard cert is issued by **DNS-01**, which needs no inbound 443 at all.
|
||||
|
||||
---
|
||||
|
||||
## Not covered here
|
||||
|
||||
- **Dev's migration off the apex** — §10a sequence D. Touches Keycloak's issuer, ~12 OIDC client
|
||||
redirect URIs, and a rolling k3s restart that invalidates every OIDC kubeconfig. Operator-present.
|
||||
- **Prod's GitOps cutover** (P5) from the workstation bootstrap Gitea to prod's own in-cluster Gitea.
|
||||
`clusters/prod/kustomization.yaml` owns that procedure.
|
||||
- **Six L2 conformance tests hardcode the apex** and must be parameterized per environment before
|
||||
§9's "suite passes green against the prod cluster" can be claimed.
|
||||
Loaded 100 of 216 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user